[6.3.0] Wire credential helper to repository fetching.

src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java

@@ -148,11 +148,13 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures Credential Helpers to use for retrieving credentials for the provided scope"
-              + " (domain).\n\n"
-              + "Credentials from Credential Helpers take precedence over credentials from"
-              + " <code>--google_default_credentials</code>, `--google_credentials`, or"
-              + " <code>.netrc</code>.\n\n"
+          "Configures a credential helper to use for retrieving authorization credentials for "
+              + " repository fetching, remote caching and execution, and the build event"
+              + " service.\n\n"
+              + "Credentials supplied by a helper take precedence over credentials supplied by"
+              + " --google_default_credentials, --google_credentials, a .netrc file, or the auth"
+              + " parameter to repository_ctx.download and repository_ctx.download_and_extract.\n\n"
+              + "May be specified multiple times to set up multiple helpers.\n\n"
               + "See https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md"
               + " for details.")
   public List<UnresolvedScopedCredentialHelper> credentialHelpers;
@@ -164,8 +166,8 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures the timeout for the Credential Helper.\n\n"
-              + "Credential Helpers failing to respond within this timeout will fail the"
+          "Configures the timeout for a credential helper.\n\n"
+              + "Credential helpers failing to respond within this timeout will fail the"
               + " invocation.")
   public Duration credentialHelperTimeout;
 
@@ -176,13 +178,13 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures the duration for which credentials from Credential Helpers are cached.\n\n"
+          "The duration for which credentials supplied by a credential helper are cached.\n\n"
               + "Invoking with a different value will adjust the lifetime of preexisting entries;"
               + " pass zero to clear the cache. A clean command always clears the cache, regardless"
               + " of this flag.")
   public Duration credentialHelperCacheTimeout;
 
-  /** One of the values of the `--credential_helper` flag. */
+  /** One of the values of the `--experimental_credential_helper` flag. */
   @AutoValue
   public abstract static class UnresolvedScopedCredentialHelper {
     /** Returns the scope of the credential helper (if any). */
@@ -192,15 +194,19 @@ public abstract static class UnresolvedScopedCredentialHelper {
     public abstract String getPath();
   }
 
-  /** A {@link Converter} for the `--credential_helper` flag. */
+  /** A {@link Converter} for the `--experimental_credential_helper` flag. */
   public static final class UnresolvedScopedCredentialHelperConverter
       extends Converter.Contextless<UnresolvedScopedCredentialHelper> {
     public static final UnresolvedScopedCredentialHelperConverter INSTANCE =
         new UnresolvedScopedCredentialHelperConverter();
 
     @Override
     public String getTypeDescription() {
-      return "An (unresolved) path to a credential helper for a scope.";
+      return "Path to a credential helper. It may be absolute, relative to the PATH environment"
+          + " variable, or %workspace%-relative. The path be optionally prefixed by a scope "
+          + " followed by an '='. The scope is a domain name, optionally with a single leading '*'"
+          + " wildcard component. A helper applies to URIs matching its scope, with more specific"
+          + " scopes preferred. If a helper has no scope, it applies to every URI.";
     }
 
     @Override

src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java

@@ -372,7 +372,6 @@ static Optional<Credentials> newCredentialsFromNetrc(
     }
   }
 
-  @VisibleForTesting
   public static CredentialHelperProvider newCredentialHelperProvider(
       CredentialHelperEnvironment environment,
       CommandLinePathFactory pathFactory,

src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD

@@ -17,6 +17,7 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib:runtime",
         "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/common/options",
         "//third_party:caffeine",
         "//third_party:guava",
     ],

src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialModule.java

@@ -21,6 +21,7 @@
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.common.options.OptionsBase;
 import java.net.URI;
 import java.time.Duration;
 
@@ -37,6 +38,11 @@ public Cache<URI, ImmutableMap<String, ImmutableList<String>>> getCredentialCach
     return credentialCache;
   }
 
+  @Override
+  public Iterable<Class<? extends OptionsBase>> getCommonCommandOptions() {
+    return ImmutableList.of(AuthAndTLSOptions.class);
+  }
+
   @Override
   public void beforeCommand(CommandEnvironment env) {
     // Update the cache expiration policy according to the command options.

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -26,6 +26,9 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection_impl",
@@ -59,6 +62,7 @@ java_library(
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/protobuf:failure_details_java_proto",
         "//third_party:guava",
+        "//third_party:jsr305",
     ],
 )
 

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -15,6 +15,7 @@
 
 package com.google.devtools.build.lib.bazel;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
@@ -25,6 +26,13 @@
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
+import com.google.devtools.build.lib.authandtls.StaticCredentials;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
 import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
@@ -110,6 +118,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 
 /** Adds support for fetching external code. */
 public class BazelRepositoryModule extends BlazeModule {
@@ -146,6 +155,8 @@ public class BazelRepositoryModule extends BlazeModule {
   private List<String> allowedYankedVersions = ImmutableList.of();
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
 
+  @Nullable private CredentialModule credentialModule;
+
   public BazelRepositoryModule() {
     this.starlarkRepositoryFunction = new StarlarkRepositoryFunction(downloadManager);
     this.repositoryHandlers = repositoryRules();
@@ -263,6 +274,8 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction());
     filesystem = runtime.getFileSystem();
+
+    credentialModule = Preconditions.checkNotNull(runtime.getBlazeModule(CredentialModule.class));
   }
 
   @Override
@@ -373,6 +386,41 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
                 Code.BAD_DOWNLOADER_CONFIG));
       }
 
+      try {
+        AuthAndTLSOptions authAndTlsOptions = env.getOptions().getOptions(AuthAndTLSOptions.class);
+        var credentialHelperEnvironment =
+            CredentialHelperEnvironment.newBuilder()
+                .setEventReporter(env.getReporter())
+                .setWorkspacePath(env.getWorkspace())
+                .setClientEnvironment(env.getClientEnv())
+                .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout)
+                .build();
+        CredentialHelperProvider credentialHelperProvider =
+            GoogleAuthUtils.newCredentialHelperProvider(
+                credentialHelperEnvironment,
+                env.getCommandLinePathFactory(),
+                authAndTlsOptions.credentialHelpers);
+
+        downloadManager.setCredentialFactory(
+            headers -> {
+              Preconditions.checkNotNull(headers);
+
+              return new CredentialHelperCredentials(
+                  credentialHelperProvider,
+                  credentialHelperEnvironment,
+                  credentialModule.getCredentialCache(),
+                  Optional.of(new StaticCredentials(headers)));
+            });
+      } catch (IOException e) {
+        env.getReporter().handle(Event.error(e.getMessage()));
+        env.getBlazeModuleEnvironment()
+            .exit(
+                new AbruptExitException(
+                    detailedExitCode(
+                        "Error initializing credential helper", Code.CREDENTIALS_INIT_FAILURE)));
+        return;
+      }
+
       if (repoOptions.experimentalDistdir != null) {
         downloadManager.setDistdir(
             repoOptions.experimentalDistdir.stream()

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java

@@ -61,6 +61,12 @@ public class DownloadManager {
   private int retries = 0;
   private boolean urlsAsDefaultCanonicalId;
   @Nullable private Credentials netrcCreds;
+  private CredentialFactory credentialFactory = StaticCredentials::new;
+
+  /** Creates {@code Credentials} from a map of per-{@code URI} authentication headers. */
+  public interface CredentialFactory {
+    Credentials create(Map<URI, Map<String, List<String>>> authHeaders);
+  }
 
   public DownloadManager(RepositoryCache repositoryCache, Downloader downloader) {
     this.repositoryCache = repositoryCache;
@@ -92,6 +98,10 @@ public void setNetrcCreds(Credentials netrcCreds) {
     this.netrcCreds = netrcCreds;
   }
 
+  public void setCredentialFactory(CredentialFactory credentialFactory) {
+    this.credentialFactory = credentialFactory;
+  }
+
   /**
    * Downloads file to disk and returns path.
    *
@@ -257,7 +267,7 @@ public Path download(
       try {
         downloader.download(
             rewrittenUrls,
-            new StaticCredentials(rewrittenAuthHeaders),
+            credentialFactory.create(rewrittenAuthHeaders),
             checksum,
             canonicalId,
             destination,
@@ -338,7 +348,7 @@ public byte[] downloadAndReadOneUrl(
     for (int attempt = 0; attempt <= retries; ++attempt) {
       try {
         return httpDownloader.downloadAndReadOneUrl(
-            rewrittenUrls.get(0), new StaticCredentials(authHeaders), eventHandler, clientEnv);
+            rewrittenUrls.get(0), credentialFactory.create(authHeaders), eventHandler, clientEnv);
       } catch (ContentLengthMismatchException e) {
         if (attempt == retries) {
           throw e;

src/main/protobuf/failure_details.proto

@@ -249,6 +249,7 @@ message ExternalRepository {
     OVERRIDE_DISALLOWED_MANAGED_DIRECTORIES = 1 [(metadata) = { exit_code: 2 }];
     BAD_DOWNLOADER_CONFIG = 2 [(metadata) = { exit_code: 2 }];
     REPOSITORY_MAPPING_RESOLUTION_FAILED = 3 [(metadata) = { exit_code: 37 }];
+    CREDENTIALS_INIT_FAILURE = 4 [(metadata) = { exit_code: 2 }];
   }
   Code code = 1;
   // Additional data could include external repository names.

src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD

@@ -50,6 +50,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:top_level_artifact_context",
         "//src/main/java/com/google/devtools/build/lib/analysis:transitive_info_collection",
         "//src/main/java/com/google/devtools/build/lib/analysis:workspace_status_action",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
         "//src/main/java/com/google/devtools/build/lib/bazel:main",
         "//src/main/java/com/google/devtools/build/lib/bazel:repository_module",
         "//src/main/java/com/google/devtools/build/lib/bugreport",

src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java

@@ -53,6 +53,7 @@
 import com.google.devtools.build.lib.analysis.util.AnalysisMock;
 import com.google.devtools.build.lib.analysis.util.AnalysisTestUtil;
 import com.google.devtools.build.lib.analysis.util.AnalysisTestUtil.DummyWorkspaceStatusActionContext;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
 import com.google.devtools.build.lib.bazel.BazelRepositoryModule;
 import com.google.devtools.build.lib.bugreport.BugReport;
 import com.google.devtools.build.lib.bugreport.BugReporter;
@@ -530,7 +531,8 @@ protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
             .addBlazeModule(new BuildIntegrationTestCommandsModule())
             .addBlazeModule(new OutputFilteringModule())
             .addBlazeModule(connectivityModule)
-            .addBlazeModule(getMockBazelRepositoryModule());
+            .addBlazeModule(getMockBazelRepositoryModule())
+            .addBlazeModule(new CredentialModule());
     getSpawnModules().forEach(builder::addBlazeModule);
     builder
         .addBlazeModule(getBuildInfoModule())

src/test/shell/bazel/remote_helpers.sh

@@ -38,8 +38,7 @@ function serve_file() {
   cd -
 }
 
-# Serves $1 as a file on localhost:$nc_port insisting on authentication (but
-# accepting any credentials.
+# Serves $1 as a file on localhost:$nc_port expecting authentication.
 #   * nc_port - the port nc is listening on.
 #   * nc_log - the path to nc's log.
 #   * nc_pid - the PID of nc.