Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@
import com.google.devtools.build.lib.runtime.InfoItem;
import com.google.devtools.build.lib.runtime.ProcessWrapper;
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory;
import com.google.devtools.build.lib.runtime.RepositoryRemoteHelpersFactory;
import com.google.devtools.build.lib.runtime.ServerBuilder;
import com.google.devtools.build.lib.runtime.WorkspaceBuilder;
import com.google.devtools.build.lib.server.FailureDetails.ExternalRepository;
Expand Down Expand Up @@ -588,11 +588,11 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
moduleMirrors = DEFAULT_MODULE_MIRRORS;
}

RepositoryRemoteExecutorFactory remoteExecutorFactory =
env.getRuntime().getRepositoryRemoteExecutorFactory();
RepositoryRemoteHelpersFactory repositoryRemoteHelpersFactory =
env.getRuntime().getRepositoryHelpersFactory();
RepositoryRemoteExecutor remoteExecutor = null;
if (remoteExecutorFactory != null) {
remoteExecutor = remoteExecutorFactory.create();
if (repositoryRemoteHelpersFactory != null) {
remoteExecutor = repositoryRemoteHelpersFactory.createExecutor();
}
repositoryFetchFunction.setRepositoryRemoteExecutor(remoteExecutor);
singleExtensionEvalFunction.setRepositoryRemoteExecutor(remoteExecutor);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hasher;
import com.google.devtools.build.lib.bazel.repository.cache.RepoContentsCache;
import com.google.devtools.build.lib.bazel.repository.cache.LocalRepoContentsCache;
import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.profiler.Profiler;
Expand Down Expand Up @@ -87,7 +87,7 @@ public void vendorRepos(Path externalRepoRoot, ImmutableList<RepositoryName> rep
Path cacheRepoDir = repoUnderExternal.resolveSymbolicLinks();
actualMarkerFile =
cacheRepoDir.replaceName(
cacheRepoDir.getBaseName() + RepoContentsCache.RECORDED_INPUTS_SUFFIX);
cacheRepoDir.getBaseName() + LocalRepoContentsCache.RECORDED_INPUTS_SUFFIX);
} else {
actualMarkerFile = markerUnderExternal;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
import com.google.devtools.build.lib.bazel.bzlmod.NonRegistryOverride;
import com.google.devtools.build.lib.bazel.bzlmod.VendorFileValue;
import com.google.devtools.build.lib.bazel.repository.RepositoryFunctionException.AlreadyReportedRepositoryAccessException;
import com.google.devtools.build.lib.bazel.repository.cache.RepoContentsCache;
import com.google.devtools.build.lib.bazel.repository.cache.RepoContentsCache.CandidateRepo;
import com.google.devtools.build.lib.bazel.repository.cache.LocalRepoContentsCache;
import com.google.devtools.build.lib.bazel.repository.cache.LocalRepoContentsCache.CandidateRepo;
import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
import com.google.devtools.build.lib.bazel.repository.starlark.NeedsSkyframeRestartException;
import com.google.devtools.build.lib.bazel.repository.starlark.RepoMetadata;
Expand Down Expand Up @@ -92,7 +92,7 @@
public final class RepositoryFetchFunction implements SkyFunction {

private final BlazeDirectories directories;
private final RepoContentsCache repoContentsCache;
private final LocalRepoContentsCache repoContentsCache;
private final Supplier<Map<String, String>> clientEnvironmentSupplier;

private double timeoutScaling = 1.0;
Expand All @@ -104,7 +104,7 @@ public final class RepositoryFetchFunction implements SkyFunction {
public RepositoryFetchFunction(
Supplier<Map<String, String>> clientEnvironmentSupplier,
BlazeDirectories directories,
RepoContentsCache repoContentsCache) {
LocalRepoContentsCache repoContentsCache) {
this.clientEnvironmentSupplier = clientEnvironmentSupplier;
this.directories = directories;
this.repoContentsCache = repoContentsCache;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ java_library(
name = "cache",
srcs = [
"DownloadCache.java",
"RepoContentsCache.java",
"LocalRepoContentsCache.java",
"RepositoryCache.java",
],
deps = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
* up-to-date), the recorded inputs file has its mtime updated. Cached repos whose recorded inputs
* file is older than {@code --repo_contents_cache_gc_max_age} are garbage collected.
*/
public final class RepoContentsCache {
public final class LocalRepoContentsCache {
public static final String RECORDED_INPUTS_SUFFIX = ".recorded_inputs";

/**
Expand Down Expand Up @@ -131,7 +131,7 @@ public ImmutableList<CandidateRepo> getCandidateRepos(String predeclaredInputHas
Comparator.comparingLong(
(Path path) ->
mtimes.computeIfAbsent(
path, RepoContentsCache::getLastModifiedTimeOrZero))
path, LocalRepoContentsCache::getLastModifiedTimeOrZero))
.reversed())
.map(CandidateRepo::fromRecordedInputsFile)
.collect(toImmutableList());
Expand Down Expand Up @@ -254,7 +254,7 @@ private void runGc(Duration maxAge) throws InterruptedException, IOException {
var recordedInputsFiles =
path.getChild(dirent.getName()).getDirectoryEntries().stream()
.filter(file -> file.getBaseName().endsWith(RECORDED_INPUTS_SUFFIX))
.sorted(comparingLong(RepoContentsCache::getLastModifiedTimeOrZero).reversed())
.sorted(comparingLong(LocalRepoContentsCache::getLastModifiedTimeOrZero).reversed())
.collect(toImmutableList());
var seen = new HashSet<HashCode>();
for (Path recordedInputsFile : recordedInputsFiles) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,21 @@

/**
* A cache directory related to repositories, containing both the {@link DownloadCache} and the
* {@link RepoContentsCache}.
* {@link LocalRepoContentsCache}.
*/
public class RepositoryCache {
// Repository cache subdirectories
private static final String CAS_DIR = "content_addressable";
private static final String CONTENTS_DIR = "contents";

private final DownloadCache downloadCache;
private final RepoContentsCache repoContentsCache;
private final LocalRepoContentsCache repoContentsCache;

@Nullable private Path path;

public RepositoryCache() {
downloadCache = new DownloadCache();
repoContentsCache = new RepoContentsCache();
repoContentsCache = new LocalRepoContentsCache();
}

public void setPath(@Nullable Path path) {
Expand All @@ -51,7 +51,7 @@ public DownloadCache getDownloadCache() {
return downloadCache;
}

public RepoContentsCache getRepoContentsCache() {
public LocalRepoContentsCache getRepoContentsCache() {
return repoContentsCache;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.runtime.CommandLinePathFactory;
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory;
import com.google.devtools.build.lib.runtime.RepositoryRemoteHelpersFactory;
import com.google.devtools.build.lib.runtime.ServerBuilder;
import com.google.devtools.build.lib.runtime.WorkspaceBuilder;
import com.google.devtools.build.lib.server.FailureDetails;
Expand Down Expand Up @@ -169,8 +169,8 @@ public ManagedChannel newChannel(
private final BuildEventArtifactUploaderFactoryDelegate
buildEventArtifactUploaderFactoryDelegate = new BuildEventArtifactUploaderFactoryDelegate();

private final RepositoryRemoteExecutorFactoryDelegate repositoryRemoteExecutorFactoryDelegate =
new RepositoryRemoteExecutorFactoryDelegate();
private final RepositoryRemoteHelpersFactoryDelegate repositoryRemoteHelpersFactoryDelegate =
new RepositoryRemoteHelpersFactoryDelegate();

private Downloader remoteDownloader;

Expand All @@ -180,7 +180,7 @@ public ManagedChannel newChannel(
public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builder) {
builder.addBuildEventArtifactUploaderFactory(
buildEventArtifactUploaderFactoryDelegate, "remote");
builder.setRepositoryRemoteExecutorFactory(repositoryRemoteExecutorFactoryDelegate);
builder.setRepositoryHelpersFactory(repositoryRemoteHelpersFactoryDelegate);
}

/** Returns whether remote execution should be enabled. */
Expand Down Expand Up @@ -721,16 +721,6 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
remoteOutputChecker,
outputService,
knownMissingCasDigests);
repositoryRemoteExecutorFactoryDelegate.init(
new RemoteRepositoryRemoteExecutorFactory(
remoteCache,
remoteExecutor,
digestUtil,
buildRequestId,
invocationId,
env.getWorkspaceName(),
remoteOptions.remoteInstanceName,
remoteOptions.remoteAcceptCached));
} else {
if (enableDiskCache) {
try {
Expand Down Expand Up @@ -765,6 +755,15 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
knownMissingCasDigests);
}

repositoryRemoteHelpersFactoryDelegate.init(
new RemoteRepositoryHelpersFactory(
actionContextProvider.getCombinedCache(),
actionContextProvider.getRemoteExecutionClient(),
buildRequestId,
invocationId,
env.getWorkspaceName(),
remoteOptions.remoteInstanceName,
remoteOptions.remoteAcceptCached));
buildEventArtifactUploaderFactoryDelegate.init(
new ByteStreamBuildEventArtifactUploaderFactory(
executorService,
Expand Down Expand Up @@ -963,7 +962,7 @@ public void afterCommand() {
lastBuildId = Preconditions.checkNotNull(env).getCommandId().toString();

buildEventArtifactUploaderFactoryDelegate.reset();
repositoryRemoteExecutorFactoryDelegate.reset();
repositoryRemoteHelpersFactoryDelegate.reset();
remoteDownloader = null;
actionContextProvider = null;
actionInputFetcher = null;
Expand Down Expand Up @@ -1194,12 +1193,12 @@ private static AbruptExitException createExitException(
.build()));
}

private static class RepositoryRemoteExecutorFactoryDelegate
implements RepositoryRemoteExecutorFactory {
private static class RepositoryRemoteHelpersFactoryDelegate
implements RepositoryRemoteHelpersFactory {

private volatile RepositoryRemoteExecutorFactory delegate;
private volatile RepositoryRemoteHelpersFactory delegate;

public void init(RepositoryRemoteExecutorFactory delegate) {
public void init(RepositoryRemoteHelpersFactory delegate) {
Preconditions.checkState(this.delegate == null);
this.delegate = delegate;
}
Expand All @@ -1210,12 +1209,12 @@ public void reset() {

@Nullable
@Override
public RepositoryRemoteExecutor create() {
RepositoryRemoteExecutorFactory delegate = this.delegate;
public RepositoryRemoteExecutor createExecutor() {
RepositoryRemoteHelpersFactory delegate = this.delegate;
if (delegate == null) {
return null;
}
return delegate.create();
return delegate.createExecutor();
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,35 +14,32 @@
package com.google.devtools.build.lib.remote;

import com.google.devtools.build.lib.remote.common.RemoteExecutionClient;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory;
import com.google.devtools.build.lib.runtime.RepositoryRemoteHelpersFactory;
import javax.annotation.Nullable;

/** Factory for {@link RemoteRepositoryRemoteExecutor}. */
class RemoteRepositoryRemoteExecutorFactory implements RepositoryRemoteExecutorFactory {
class RemoteRepositoryHelpersFactory implements RepositoryRemoteHelpersFactory {

private final RemoteExecutionCache remoteExecutionCache;
private final RemoteExecutionClient remoteExecutor;
private final DigestUtil digestUtil;
private final CombinedCache cache;
@Nullable private final RemoteExecutionClient remoteExecutor;
private final String buildRequestId;
private final String commandId;
private final String workspaceName;

private final String remoteInstanceName;
private final boolean acceptCached;

RemoteRepositoryRemoteExecutorFactory(
RemoteExecutionCache remoteExecutionCache,
RemoteExecutionClient remoteExecutor,
DigestUtil digestUtil,
RemoteRepositoryHelpersFactory(
CombinedCache cache,
@Nullable RemoteExecutionClient remoteExecutor,
String buildRequestId,
String commandId,
String workspaceName,
String remoteInstanceName,
boolean acceptCached) {
this.remoteExecutionCache = remoteExecutionCache;
this.cache = cache;
this.remoteExecutor = remoteExecutor;
this.digestUtil = digestUtil;
this.buildRequestId = buildRequestId;
this.commandId = commandId;
this.workspaceName = workspaceName;
Expand All @@ -51,11 +48,14 @@ class RemoteRepositoryRemoteExecutorFactory implements RepositoryRemoteExecutorF
}

@Override
public RepositoryRemoteExecutor create() {
public RepositoryRemoteExecutor createExecutor() {
if (remoteExecutor == null) {
return null;
}
return new RemoteRepositoryRemoteExecutor(
remoteExecutionCache,
(RemoteExecutionCache) cache,
remoteExecutor,
digestUtil,
cache.digestUtil,
buildRequestId,
commandId,
workspaceName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ public final class BlazeRuntime implements BugReport.BlazeRuntimeInterface {
private final String productName;
private final BuildEventArtifactUploaderFactoryMap buildEventArtifactUploaderFactoryMap;
private final ActionKeyContext actionKeyContext;
@Nullable private final RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory;
@Nullable private final RepositoryRemoteHelpersFactory repositoryRemoteHelpersFactory;

// Workspace state (currently exactly one workspace per server)
private BlazeWorkspace workspace;
Expand Down Expand Up @@ -232,7 +232,7 @@ private BlazeRuntime(
BlazeServiceRegistry blazeServiceRegistry,
String productName,
BuildEventArtifactUploaderFactoryMap buildEventArtifactUploaderFactoryMap,
RepositoryRemoteExecutorFactory repositoryRemoteExecutorFactory,
RepositoryRemoteHelpersFactory repositoryRemoteHelpersFactory,
InstrumentationOutputFactory instrumentationOutputFactory,
FileSystemLock installBaseLock) {
// Server state
Expand Down Expand Up @@ -262,7 +262,7 @@ private BlazeRuntime(
new CommandNameCacheImpl(commandMap));
this.productName = productName;
this.buildEventArtifactUploaderFactoryMap = buildEventArtifactUploaderFactoryMap;
this.repositoryRemoteExecutorFactory = repositoryRemoteExecutorFactory;
this.repositoryRemoteHelpersFactory = repositoryRemoteHelpersFactory;
this.instrumentationOutputFactory = instrumentationOutputFactory;
this.installBaseLock = installBaseLock;
this.cgroupsInfo = VirtualCgroup.getInstance().cgroupsInfo();
Expand Down Expand Up @@ -1602,8 +1602,8 @@ public BuildEventArtifactUploaderFactoryMap getBuildEventArtifactUploaderFactory
return buildEventArtifactUploaderFactoryMap;
}

public RepositoryRemoteExecutorFactory getRepositoryRemoteExecutorFactory() {
return repositoryRemoteExecutorFactory;
public RepositoryRemoteHelpersFactory getRepositoryHelpersFactory() {
return repositoryRemoteHelpersFactory;
}

public InstrumentationOutputFactory getInstrumentationOutputFactory() {
Expand Down Expand Up @@ -1748,7 +1748,7 @@ public BlazeRuntime build() throws AbruptExitException {
serverBuilder.getBlazeServiceRegistry(),
productName,
serverBuilder.getBuildEventArtifactUploaderMap(),
serverBuilder.getRepositoryRemoteExecutorFactory(),
serverBuilder.getRepositoryHelpersFactory(),
serverBuilder.createInstrumentationOutputFactory(),
installBaseLock);
AutoProfiler.setClock(runtime.getClock());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
import javax.annotation.Nullable;

/** Factory for {@link RepositoryRemoteExecutor}. */
public interface RepositoryRemoteExecutorFactory {
public interface RepositoryRemoteHelpersFactory {

/** Returns a new {@link RepositoryRemoteExecutor} or {@code null}. */
@Nullable
RepositoryRemoteExecutor create();
RepositoryRemoteExecutor createExecutor();
}
Loading
Loading