Skip to content

Commit

Permalink
Run coverage report generation on host platform
Browse files Browse the repository at this point in the history
  • Loading branch information
fmeum committed Oct 10, 2023
1 parent 9a7b1f7 commit 7b28c63
Show file tree
Hide file tree
Showing 14 changed files with 145 additions and 27 deletions.
2 changes: 2 additions & 0 deletions src/main/java/com/google/devtools/build/lib/analysis/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:package_roots",
"//src/main/java/com/google/devtools/build/lib/actions:resource_manager",
"//src/main/java/com/google/devtools/build/lib/analysis/platform",
"//src/main/java/com/google/devtools/build/lib/bugreport",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
Expand Down Expand Up @@ -2673,6 +2674,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:action_lookup_key",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/analysis/platform",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/skyframe:coverage_report_value",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import com.google.devtools.build.lib.analysis.constraints.PlatformRestrictionsResult;
import com.google.devtools.build.lib.analysis.constraints.RuleContextConstraintSemantics;
import com.google.devtools.build.lib.analysis.constraints.TopLevelConstraintSemantics;
import com.google.devtools.build.lib.analysis.platform.PlatformInfo;
import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory;
import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory.CoverageReportActionsWrapper;
import com.google.devtools.build.lib.analysis.test.InstrumentedFilesInfo;
Expand Down Expand Up @@ -301,9 +302,14 @@ public AnalysisResult update(
bugReporter,
Preconditions.checkNotNull(resourceManager), // non-null for skymeld.
Preconditions.checkNotNull(buildResultListener), // non-null for skymeld.
(configuredTargets, allTargetsToTest) ->
(configuredTargets, allTargetsToTest, hostPlatformInfo) ->
memoizedGetCoverageArtifactsHelper(
configuredTargets, allTargetsToTest, eventHandler, eventBus, loadingResult),
configuredTargets,
allTargetsToTest,
hostPlatformInfo,
eventHandler,
eventBus,
loadingResult),
keepGoing,
skipIncompatibleExplicitTargets,
targetOptions.get(CoreOptions.class).strictConflictChecks,
Expand Down Expand Up @@ -538,7 +544,7 @@ private AnalysisResult createResult(
// Coverage
artifactsToBuild.addAll(
memoizedGetCoverageArtifactsHelper(
configuredTargets, allTargetsToTest, eventHandler, eventBus, loadingResult));
configuredTargets, allTargetsToTest, null, eventHandler, eventBus, loadingResult));

// TODO(cparsons): If extra actions are ever removed, this filtering step can probably be
// removed as well: the only concern would be action conflicts involving coverage artifacts,
Expand Down Expand Up @@ -835,6 +841,7 @@ private static void pollInterruptedStatus() throws InterruptedException {
private ImmutableSet<Artifact> memoizedGetCoverageArtifactsHelper(
Set<ConfiguredTarget> configuredTargets,
Set<ConfiguredTarget> allTargetsToTest,
PlatformInfo hostPlatformInfo,
EventHandler eventHandler,
EventBus eventBus,
TargetPatternPhaseValue loadingResult)
Expand All @@ -854,6 +861,7 @@ private ImmutableSet<Artifact> memoizedGetCoverageArtifactsHelper(
eventBus,
directories,
allTargetsToTest,
hostPlatformInfo,
baselineCoverageArtifacts,
getArtifactFactory(),
skyframeExecutor.getActionKeyContext(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.devtools.build.lib.actions.MutableActionGraph;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.platform.PlatformInfo;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.skyframe.CoverageReportValue;
Expand Down Expand Up @@ -85,6 +86,7 @@ CoverageReportActionsWrapper createCoverageReportActionsWrapper(
EventBus eventBus,
BlazeDirectories directories,
Collection<ConfiguredTarget> targetsToTest,
PlatformInfo hostPlatformInfo,
NestedSet<Artifact> baselineCoverageArtifacts,
ArtifactFactory artifactFactory,
ActionKeyContext actionKeyContext,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/analysis:incompatible_platform_provider",
"//src/main/java/com/google/devtools/build/lib/analysis:test/coverage_report",
"//src/main/java/com/google/devtools/build/lib/analysis:test/coverage_report_action_factory",
"//src/main/java/com/google/devtools/build/lib/analysis/platform",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//src/main/java/com/google/devtools/build/lib/events",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.platform.PlatformInfo;
import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.events.EventHandler;
Expand Down Expand Up @@ -86,6 +87,7 @@ public CoverageReportActionsWrapper createCoverageReportActionsWrapper(
EventBus eventBus,
BlazeDirectories directories,
Collection<ConfiguredTarget> targetsToTest,
PlatformInfo hostPlatformInfo,
NestedSet<Artifact> baselineCoverageArtifacts,
ArtifactFactory artifactFactory,
ActionKeyContext actionKeyContext,
Expand All @@ -101,14 +103,15 @@ public CoverageReportActionsWrapper createCoverageReportActionsWrapper(
eventHandler,
directories,
targetsToTest,
hostPlatformInfo,
baselineCoverageArtifacts,
artifactFactory,
actionKeyContext,
actionLookupKey,
workspaceName,
this::getArgs,
this::getLocationMessage,
/*htmlReport=*/ false);
/* htmlReport= */ false);
}

private ImmutableList<String> getArgs(CoverageArgs args) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.FilesToRunProvider;
import com.google.devtools.build.lib.analysis.platform.PlatformInfo;
import com.google.devtools.build.lib.bazel.coverage.CoverageReportActionBuilder.ArgsFunc;
import com.google.devtools.build.lib.bazel.coverage.CoverageReportActionBuilder.LocationFunc;
import com.google.devtools.build.lib.vfs.PathFragment;
Expand All @@ -37,8 +38,13 @@ public abstract class CoverageArgs {
public abstract ImmutableList<Artifact> coverageArtifacts();
public abstract Artifact lcovArtifact();
public abstract ArtifactFactory factory();

public abstract ArtifactOwner artifactOwner();

public abstract PlatformInfo hostPlatformInfo();

public abstract FilesToRunProvider reportGenerator();

public abstract String workspaceName();
public abstract boolean htmlReport();
@Nullable
Expand All @@ -52,22 +58,39 @@ public static CoverageArgs create(
Artifact lcovArtifact,
ArtifactFactory factory,
ArtifactOwner artifactOwner,
PlatformInfo hostPlatformInfo,
FilesToRunProvider reportGenerator,
String workspaceName,
boolean htmlReport) {
return new AutoValue_CoverageArgs(directories, coverageArtifacts, lcovArtifact, factory,
artifactOwner, reportGenerator, workspaceName, htmlReport,
/*coverageDir=*/ null,
/*lcovOutput=*/ null);
return new AutoValue_CoverageArgs(
directories,
coverageArtifacts,
lcovArtifact,
factory,
artifactOwner,
hostPlatformInfo,
reportGenerator,
workspaceName,
htmlReport,
/* coverageDir= */ null,
/* lcovOutput= */ null);
}

public static CoverageArgs createCopyWithCoverageDirAndLcovOutput(
CoverageArgs args,
PathFragment coverageDir,
Artifact lcovOutput) {
return new AutoValue_CoverageArgs(
args.directories(), args.coverageArtifacts(), args.lcovArtifact(),
args.factory(), args.artifactOwner(), args.reportGenerator(), args.workspaceName(),
args.htmlReport(), coverageDir, lcovOutput);
args.directories(),
args.coverageArtifacts(),
args.lcovArtifact(),
args.factory(),
args.artifactOwner(),
args.hostPlatformInfo(),
args.reportGenerator(),
args.workspaceName(),
args.htmlReport(),
coverageDir,
lcovOutput);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.AbstractAction;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
Expand Down Expand Up @@ -48,6 +49,7 @@
import com.google.devtools.build.lib.analysis.RunfilesSupport;
import com.google.devtools.build.lib.analysis.actions.Compression;
import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
import com.google.devtools.build.lib.analysis.platform.PlatformInfo;
import com.google.devtools.build.lib.analysis.test.CoverageReport;
import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory.CoverageReportActionsWrapper;
import com.google.devtools.build.lib.analysis.test.TestProvider;
Expand Down Expand Up @@ -98,8 +100,6 @@ public final class CoverageReportActionBuilder {
private static final ResourceSet LOCAL_RESOURCES =
ResourceSet.createWithRamCpu(/* memoryMb= */ 750, /* cpuUsage= */ 1);

private static final ActionOwner ACTION_OWNER = ActionOwner.SYSTEM_ACTION_OWNER;

// SpawnActions can't be used because they need the AnalysisEnvironment and this action is
// created specially at the very end of the analysis phase when we don't have it anymore.
@Immutable
Expand Down Expand Up @@ -190,6 +190,7 @@ public CoverageReportActionsWrapper createCoverageActionsWrapper(
EventHandler reporter,
BlazeDirectories directories,
Collection<ConfiguredTarget> targetsToTest,
PlatformInfo hostPlatformInfo,
NestedSet<Artifact> baselineCoverageArtifacts,
ArtifactFactory factory,
ActionKeyContext actionKeyContext,
Expand Down Expand Up @@ -225,15 +226,26 @@ public CoverageReportActionsWrapper createCoverageActionsWrapper(
ImmutableList<Artifact> coverageArtifacts = builder.build();
if (!coverageArtifacts.isEmpty()) {
PathFragment coverageDir = TestRunnerAction.COVERAGE_TMP_ROOT;
Artifact lcovArtifact = factory.getDerivedArtifact(
coverageDir.getRelative("lcov_files.tmp"),
directories.getBuildDataDirectory(workspaceName),
artifactOwner);
Artifact lcovArtifact =
factory.getDerivedArtifact(
coverageDir.getRelative("lcov_files.tmp"),
directories.getBuildDataDirectory(workspaceName),
artifactOwner);
Action lcovFileAction = generateLcovFileWriteAction(lcovArtifact, coverageArtifacts);
Action coverageReportAction = generateCoverageReportAction(
CoverageArgs.create(directories, coverageArtifacts, lcovArtifact, factory, artifactOwner,
reportGenerator, workspaceName, htmlReport),
argsFunction, locationFunc);
Action coverageReportAction =
generateCoverageReportAction(
CoverageArgs.create(
directories,
coverageArtifacts,
lcovArtifact,
factory,
artifactOwner,
hostPlatformInfo,
reportGenerator,
workspaceName,
htmlReport),
argsFunction,
locationFunc);
return new CoverageReportActionsWrapper(
lcovFileAction, coverageReportAction, actionKeyContext);
} else {
Expand All @@ -250,7 +262,7 @@ private static FileWriteAction generateLcovFileWriteAction(
filepaths.add(artifact.getExecPathString());
}
return FileWriteAction.create(
ACTION_OWNER,
ActionOwner.SYSTEM_ACTION_OWNER,
lcovArtifact,
Joiner.on('\n').join(filepaths),
/* makeExecutable= */ false,
Expand Down Expand Up @@ -290,8 +302,17 @@ private static CoverageReportAction generateCoverageReportAction(
if (runfilesSupport != null) {
inputsBuilder.add(runfilesSupport.getRunfilesMiddleman());
}
ActionOwner actionOwner =
ActionOwner.create(
ActionOwner.SYSTEM_ACTION_OWNER.getLabel(),
ActionOwner.SYSTEM_ACTION_OWNER.getLocation(),
ActionOwner.SYSTEM_ACTION_OWNER.getTargetKind(),
ActionOwner.SYSTEM_ACTION_OWNER.getBuildConfigurationInfo(),
args.hostPlatformInfo(),
ActionOwner.SYSTEM_ACTION_OWNER.getAspectDescriptors(),
ActionOwner.SYSTEM_ACTION_OWNER.getExecProperties());
return new CoverageReportAction(
ACTION_OWNER,
actionOwner,
inputsBuilder.build(),
ImmutableSet.of(lcovOutput),
actionArgs,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ public String getSymlinkPrefix(String productName) {
*/
@Option(
name = "experimental_merged_skyframe_analysis_execution",
defaultValue = "false",
defaultValue = "true",
documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
metadataTags = OptionMetadataTag.EXPERIMENTAL,
effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS, OptionEffectTag.EXECUTION},
Expand Down
13 changes: 13 additions & 0 deletions src/main/java/com/google/devtools/build/lib/skyframe/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -918,9 +918,11 @@ java_library(
deps = [
":aspect_key_creator",
":configured_target_key",
":host_platform_info_event",
":top_level_status_events",
"//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
"//src/main/java/com/google/devtools/build/lib/analysis:configured_target",
"//src/main/java/com/google/devtools/build/lib/analysis/platform",
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//third_party:guava",
],
Expand Down Expand Up @@ -2940,3 +2942,14 @@ java_library(
"//third_party:jsr305",
],
)

java_library(
name = "host_platform_info_event",
srcs = ["HostPlatformInfoEvent.java"],
deps = [
"//src/main/java/com/google/devtools/build/lib/analysis/platform",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/skyframe",
"//third_party:auto_value",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@
// limitations under the License.
package com.google.devtools.build.lib.skyframe;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
import com.google.devtools.build.lib.analysis.ConfiguredAspect;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.platform.PlatformInfo;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.skyframe.AspectKeyCreator.AspectKey;
import com.google.devtools.build.lib.skyframe.TopLevelStatusEvents.AspectAnalyzedEvent;
Expand All @@ -28,9 +30,12 @@
import com.google.devtools.build.lib.skyframe.TopLevelStatusEvents.TopLevelTargetAnalyzedEvent;
import com.google.devtools.build.lib.skyframe.TopLevelStatusEvents.TopLevelTargetBuiltEvent;
import com.google.devtools.build.lib.skyframe.TopLevelStatusEvents.TopLevelTargetSkippedEvent;

import javax.annotation.Nullable;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;

/**
* Listens to the various status events of the top level targets/aspects.
Expand All @@ -49,6 +54,7 @@ public class BuildResultListener {
// Also includes test targets.
private final Set<ConfiguredTargetKey> builtTargets = ConcurrentHashMap.newKeySet();
private final Set<AspectKey> builtAspects = ConcurrentHashMap.newKeySet();
private final AtomicReference<PlatformInfo> hostPlatformInfo = new AtomicReference<>();

@Subscribe
@AllowConcurrentEvents
Expand Down Expand Up @@ -86,6 +92,12 @@ public void addBuiltAspect(AspectBuiltEvent event) {
builtAspects.add(event.aspectKey());
}

@Subscribe
@AllowConcurrentEvents
public void setHostPlatformInfo(HostPlatformInfoEvent hostPlatformInfoEvent) {
hostPlatformInfo.compareAndSet(null, hostPlatformInfoEvent.hostPlatformInfo());
}

public ImmutableSet<ConfiguredTarget> getAnalyzedTargets() {
return ImmutableSet.copyOf(analyzedTargets);
}
Expand All @@ -106,6 +118,10 @@ public ImmutableSet<ConfiguredTargetKey> getBuiltTargets() {
return ImmutableSet.copyOf(builtTargets);
}

public PlatformInfo getHostPlatformInfo() {
return Preconditions.checkNotNull(hostPlatformInfo.get());
}

public ImmutableSet<AspectKey> getBuiltAspects() {
return ImmutableSet.copyOf(builtAspects);
}
Expand Down
Loading

0 comments on commit 7b28c63

Please sign in to comment.