From 70a9d0475ec954ffd749c12d8b898a4d51a630d6 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:55:32 +0100 Subject: [PATCH 1/2] Implement fail-fast tests ordering for JUnit 5 (#8055) --- .../config/ConfigurationApiImpl.java | 16 ++- .../config/ExecutionSettings.java | 1 - .../config/ExecutionSettingsFactoryImpl.java | 9 +- .../config/TestIdentifierSerializer.java | 4 +- .../domain/AbstractTestSession.java | 9 +- .../domain/TestFrameworkModule.java | 2 + .../domain/buildsystem/ProxyTestModule.java | 5 + .../domain/headless/HeadlessTestModule.java | 5 + .../events/TestEventsHandlerImpl.java | 10 ++ .../civisibility/test/ExecutionStrategy.java | 5 + .../CiVisibilityInstrumentationTest.groovy | 44 +++++- .../src/test/groovy/CucumberTest.groovy | 2 + .../src/test/groovy/MUnitTest.groovy | 2 + .../src/test/groovy/JUnit4Test.groovy | 2 + .../instrumentation/junit-5.3/build.gradle | 12 +- .../junit-5.3/cucumber-junit-5/build.gradle | 1 + .../junit5/JUnit5CucumberInstrumentation.java | 16 --- .../src/test/groovy/CucumberTest.groovy | 2 + .../junit-5.3/junit-5.8/build.gradle | 12 +- .../junit5/order/FailFastClassOrderer.java | 65 +++++++++ .../junit5/order/FailFastMethodOrderer.java | 46 +++++++ .../junit5/order/JUnit5OrderUtils.java | 27 ++++ .../order/JUnit5TestOrderInstrumentation.java | 127 ++++++++++++++++++ .../src/test/groovy/JUnit58Test.groovy | 57 ++++++++ .../java/org/example/TestParameterized.java | 35 +++++ .../test/java/org/example/TestSucceed.java | 18 +++ .../java/org/example/TestSucceedAnother.java | 13 ++ .../test-before-all-after-all/events.ftl | 2 + .../test-before-each-after-each/events.ftl | 18 +-- .../test-failed-after-all/events.ftl | 2 + .../test-failed-after-each/events.ftl | 10 +- .../test-failed-before-each/events.ftl | 10 +- .../junit-5.3/spock-junit-5/build.gradle | 1 + .../junit5/JUnit5SpockInstrumentation.java | 16 --- .../src/test/groovy/SpockTest.groovy | 9 ++ .../junit5/JUnit5Instrumentation.java | 26 +++- .../junit5/JUnitPlatformUtils.java | 7 + .../src/test/groovy/JUnit5Test.groovy | 2 + .../karate/src/test/groovy/KarateTest.groovy | 2 + .../src/test/groovy/ScalatestTest.groovy | 2 + .../instrumentation/testng/TestNGTest.groovy | 2 + .../trace/api/config/CiVisibilityConfig.java | 1 + .../main/java/datadog/trace/api/Config.java | 6 + .../trace/api/civisibility/CIConstants.java | 2 + .../events/TestEventsHandler.java | 4 + .../telemetry/CiVisibilityCountMetric.java | 19 ++- .../CiVisibilityDistributionMetric.java | 8 +- .../tag/AgentlessLogSubmissionEnabled.java | 12 ++ .../tag/FailFastTestOrderEnabled.java | 12 ++ .../tag/FlakyTestRetriesEnabled.java | 12 ++ 50 files changed, 654 insertions(+), 78 deletions(-) create mode 100644 dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastClassOrderer.java create mode 100644 dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastMethodOrderer.java create mode 100644 dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5OrderUtils.java create mode 100644 dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5TestOrderInstrumentation.java create mode 100644 dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestParameterized.java create mode 100644 dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceed.java create mode 100644 dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceedAnother.java create mode 100644 internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/AgentlessLogSubmissionEnabled.java create mode 100644 internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FailFastTestOrderEnabled.java create mode 100644 internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FlakyTestRetriesEnabled.java diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java index b13979809a4..fb4b59b280c 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java @@ -15,6 +15,7 @@ import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; import datadog.trace.api.civisibility.telemetry.tag.CoverageEnabled; import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionEnabled; +import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled; import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled; import datadog.trace.api.civisibility.telemetry.tag.ItrSkipEnabled; import datadog.trace.api.civisibility.telemetry.tag.RequireGit; @@ -132,6 +133,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr settings.getEarlyFlakeDetectionSettings().isEnabled() ? EarlyFlakeDetectionEnabled.TRUE : null, + settings.isFlakyTestRetriesEnabled() ? FlakyTestRetriesEnabled.TRUE : null, settings.isGitUploadRequired() ? RequireGit.TRUE : null); return settings; @@ -185,6 +187,14 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr @Override public Map> getFlakyTestsByModule( TracerEnvironment tracerEnvironment) throws IOException { + OkHttpUtils.CustomListener telemetryListener = + new TelemetryListener.Builder(metricCollector) + .requestCount(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST) + .requestErrors(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST_ERRORS) + .requestDuration(CiVisibilityDistributionMetric.FLAKY_TESTS_REQUEST_MS) + .responseBytes(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_BYTES) + .build(); + String uuid = uuidGenerator.get(); EnvelopeDto request = new EnvelopeDto<>( @@ -196,11 +206,12 @@ public Map> getFlakyTestsByModule( FLAKY_TESTS_URI, requestBody, is -> testIdentifiersResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data, - null, + telemetryListener, false); LOGGER.debug("Received {} flaky tests in total", response.size()); + int flakyTestsCount = 0; Map> testIdentifiers = new HashMap<>(); for (DataDto dataDto : response) { TestIdentifierJson testIdentifierJson = dataDto.getAttributes(); @@ -209,7 +220,10 @@ public Map> getFlakyTestsByModule( testIdentifiers .computeIfAbsent(moduleName, k -> new HashSet<>()) .add(testIdentifierJson.toTestIdentifier()); + flakyTestsCount++; } + + metricCollector.add(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_TESTS, flakyTestsCount); return testIdentifiers; } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java index 20a653576b9..dce2f5dffc7 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java @@ -114,7 +114,6 @@ public Collection getKnownTests() { @Nullable public Collection getFlakyTests() { - // backend does not store module info for flaky tests yet return flakyTests; } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java index 117fdb90447..d1b345c1808 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java @@ -1,6 +1,7 @@ package datadog.trace.civisibility.config; import datadog.trace.api.Config; +import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestMetadata; @@ -139,11 +140,17 @@ private TracerEnvironment buildTracerEnvironment( Map> flakyTestsByModule = flakyTestRetriesEnabled && config.isCiVisibilityFlakyRetryOnlyKnownFlakes() + || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase( + config.getCiVisibilityTestOrder()) ? getFlakyTestsByModule(tracerEnvironment) : null; Map> knownTestsByModule = - earlyFlakeDetectionEnabled ? getKnownTestsByModule(tracerEnvironment) : null; + earlyFlakeDetectionEnabled + || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase( + config.getCiVisibilityTestOrder()) + ? getKnownTestsByModule(tracerEnvironment) + : null; Set moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS)); moduleNames.addAll(skippableTestIdentifiers.keySet()); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java index 2d45a6c9979..73a6e113ca4 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java @@ -13,8 +13,10 @@ public static void serialize(Serializer serializer, TestIdentifier testIdentifie } public static TestIdentifier deserialize(ByteBuffer buffer) { + String suiteName = Serializer.readString(buffer); return new TestIdentifier( - Serializer.readString(buffer), + // suite name repeats a lot; interning it to save memory + suiteName != null ? suiteName.intern() : null, Serializer.readString(buffer), Serializer.readString(buffer)); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java index 1eb3caaf9c6..ff1de16e33e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java @@ -5,11 +5,14 @@ import datadog.trace.api.Config; import datadog.trace.api.DDTraceId; import datadog.trace.api.IdGenerationStrategy; +import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; import datadog.trace.api.civisibility.telemetry.TagValue; +import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled; import datadog.trace.api.civisibility.telemetry.tag.AutoInjected; import datadog.trace.api.civisibility.telemetry.tag.EventType; +import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled; import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner; import datadog.trace.api.civisibility.telemetry.tag.IsHeadless; import datadog.trace.api.civisibility.telemetry.tag.IsUnsupportedCI; @@ -109,7 +112,11 @@ public AbstractTestSession( CiVisibilityCountMetric.TEST_SESSION, 1, ciProvider, - config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null); + config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null, + config.isAgentlessLogSubmissionEnabled() ? AgentlessLogSubmissionEnabled.TRUE : null, + CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder()) + ? FailFastTestOrderEnabled.TRUE + : null); if (instrumentationType == InstrumentationType.MANUAL_API) { metricCollector.add(CiVisibilityCountMetric.MANUAL_API_EVENTS, 1, EventType.SESSION); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java index 42723fcffd1..6c98136fa7e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java @@ -25,6 +25,8 @@ TestSuiteImpl testSuiteStart( */ boolean isNew(TestIdentifier test); + boolean isFlaky(TestIdentifier test); + /** * Checks if a given test should be skipped with Intelligent Test Runner or not * diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java index 0758a693361..cc0fb403234 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java @@ -87,6 +87,11 @@ public boolean isNew(TestIdentifier test) { return executionStrategy.isNew(test); } + @Override + public boolean isFlaky(TestIdentifier test) { + return executionStrategy.isFlaky(test); + } + @Override public boolean shouldBeSkipped(TestIdentifier test) { return executionStrategy.shouldBeSkipped(test); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java index fc92ba099c2..2a527fa5580 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java @@ -72,6 +72,11 @@ public boolean isNew(TestIdentifier test) { return executionStrategy.isNew(test); } + @Override + public boolean isFlaky(TestIdentifier test) { + return executionStrategy.isFlaky(test); + } + @Override public boolean shouldBeSkipped(TestIdentifier test) { return executionStrategy.shouldBeSkipped(test); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java index a634586f5cd..94958461f41 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java @@ -272,6 +272,16 @@ public TestRetryPolicy retryPolicy(TestIdentifier test) { return testModule.retryPolicy(test); } + @Override + public boolean isNew(TestIdentifier test) { + return testModule.isNew(test); + } + + @Override + public boolean isFlaky(TestIdentifier test) { + return testModule.isFlaky(test); + } + @Override public void close() { testModule.end(null); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java index 12639864f4b..819a54fa2c9 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java @@ -43,6 +43,11 @@ public boolean isNew(TestIdentifier test) { return knownTests != null && !knownTests.contains(test.withoutParameters()); } + public boolean isFlaky(TestIdentifier test) { + Collection flakyTests = executionSettings.getFlakyTests(); + return flakyTests != null && flakyTests.contains(test.withoutParameters()); + } + public boolean shouldBeSkipped(TestIdentifier test) { if (test == null) { return false; diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index f38bf6e4779..d1c0fc936be 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -5,6 +5,7 @@ import datadog.communication.serialization.GrowableBuffer import datadog.communication.serialization.msgpack.MsgPackWriter import datadog.trace.agent.test.AgentTestRunner import datadog.trace.api.Config +import datadog.trace.api.civisibility.CIConstants import datadog.trace.api.civisibility.DDTest import datadog.trace.api.civisibility.DDTestSuite import datadog.trace.api.civisibility.InstrumentationBridge @@ -124,7 +125,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { skippableTestsWithMetadata, [:], flakyTests, - earlyFlakinessDetectionEnabled ? knownTests : null) + earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null) } } @@ -224,12 +225,22 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { def givenFlakyTests(List tests) { flakyTests.addAll(tests) - flakyRetryEnabled = true } def givenKnownTests(List tests) { knownTests.addAll(tests) - earlyFlakinessDetectionEnabled = true + } + + def givenFlakyRetryEnabled(boolean flakyRetryEnabled) { + this.flakyRetryEnabled = flakyRetryEnabled + } + + def givenEarlyFlakinessDetectionEnabled(boolean earlyFlakinessDetectionEnabled) { + this.earlyFlakinessDetectionEnabled = earlyFlakinessDetectionEnabled + } + + def givenTestsOrder(String testsOrder) { + injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder) } @Override @@ -274,6 +285,33 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements) } + def assertTestsOrder(List expectedOrder) { + TEST_WRITER.waitForTraces(expectedOrder.size() + 1) + def traces = TEST_WRITER.toList() + def events = getEventsAsJson(traces) + def identifiers = getTestIdentifiers(events) + if (identifiers != expectedOrder) { + throw new AssertionError("Expected order: $expectedOrder, but got: $identifiers") + } + return true + } + + def getTestIdentifiers(List events) { + events.sort(Comparator.comparing { it['content']['start'] as Long }) + def testIdentifiers = [] + for (Map event : events) { + if (event['content']['meta']['test.name']) { + testIdentifiers.add(test(event['content']['meta']['test.suite'] as String, event['content']['meta']['test.name'] as String)) + } + } + return testIdentifiers + } + + def test(String suite, String name, String parameters = null) { + + return new TestIdentifier(suite, name, parameters) + } + def getEventsAsJson(List> traces) { return getSpansAsJson(new CiTestCycleMapperV1(Config.get().getCiVisibilityWellKnownTags(), false), traces) } diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy index 1eef3e38000..13fb5ca7215 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy @@ -53,6 +53,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runFeatures(features) @@ -71,6 +72,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runFeatures(features) diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy index 68d12b2efd3..43ac216a46b 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy @@ -31,6 +31,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runTests(tests) @@ -45,6 +46,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runTests(tests) diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy index a97d82a7225..38f51a32850 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy @@ -90,6 +90,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runTests(tests) @@ -110,6 +111,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runTests(tests) diff --git a/dd-java-agent/instrumentation/junit-5.3/build.gradle b/dd-java-agent/instrumentation/junit-5.3/build.gradle index 132c4805010..10e805e4010 100644 --- a/dd-java-agent/instrumentation/junit-5.3/build.gradle +++ b/dd-java-agent/instrumentation/junit-5.3/build.gradle @@ -33,9 +33,9 @@ dependencies { // versions used below are not the minimum ones that we support, // but the tests need to use them in order to be compliant with Spock 2.x - testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2' - testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2' - testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2' + testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2' latestDepTestImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '+' latestDepTestImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '+' @@ -44,8 +44,8 @@ dependencies { configurations.matching({ it.name.startsWith('test') }).each({ it.resolutionStrategy { - force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2' - force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2' - force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2' + force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2' + force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2' + force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2' } }) diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/build.gradle b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/build.gradle index 40057b40131..53e87629257 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/build.gradle +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/build.gradle @@ -13,6 +13,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { implementation project(':dd-java-agent:instrumentation:junit-5.3') + compileOnly group: 'io.cucumber', name: 'cucumber-junit-platform-engine', version: '5.4.0' compileOnly group: 'io.cucumber', name: 'cucumber-java', version: '5.4.0' diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java index 35649a927d1..01939287fed 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java @@ -7,17 +7,12 @@ import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.bootstrap.ContextStore; -import datadog.trace.bootstrap.InstrumentationContext; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.cucumber.junit.platform.engine.CucumberTestEngine; -import java.util.Collections; -import java.util.Map; import net.bytebuddy.asm.Advice; import net.bytebuddy.matcher.ElementMatcher; import org.junit.platform.engine.EngineExecutionListener; import org.junit.platform.engine.ExecutionRequest; -import org.junit.platform.engine.TestDescriptor; import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService; @@ -51,11 +46,6 @@ public String[] helperClassNames() { }; } - @Override - public Map contextStore() { - return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object"); - } - @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -87,12 +77,6 @@ public static void addTracingListener( return; } - ContextStore contextStore = - InstrumentationContext.get(TestDescriptor.class, Object.class); - TestEventsHandlerHolder.setContextStores( - (ContextStore) contextStore, (ContextStore) contextStore); - TestEventsHandlerHolder.start(); - CucumberTracingListener tracingListener = new CucumberTracingListener(testEngine); EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener(); EngineExecutionListener compositeListener = diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy index 2fc55fca889..15096e6601b 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy @@ -55,6 +55,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runFeatures(features, false) @@ -76,6 +77,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runFeatures(features, false) diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/build.gradle b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/build.gradle index c4418493d73..9ef5699a265 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/build.gradle +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/build.gradle @@ -36,9 +36,9 @@ dependencies { // versions used below are not the minimum ones that we support, // but the tests need to use them in order to be compliant with Spock 2.x - testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2' - testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2' - testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2' + testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2' latestDepTestImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '+' latestDepTestImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '+' @@ -47,8 +47,8 @@ dependencies { configurations.matching({ it.name.startsWith('test') }).each({ it.resolutionStrategy { - force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2' - force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2' - force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2' + force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2' + force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2' + force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2' } }) diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastClassOrderer.java b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastClassOrderer.java new file mode 100644 index 00000000000..32e89b901c3 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastClassOrderer.java @@ -0,0 +1,65 @@ +package datadog.trace.instrumentation.junit5.order; + +import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.instrumentation.junit5.JUnitPlatformUtils; +import java.util.Comparator; +import java.util.Set; +import javax.annotation.Nullable; +import org.junit.jupiter.api.ClassDescriptor; +import org.junit.jupiter.api.ClassOrderer; +import org.junit.jupiter.api.ClassOrdererContext; +import org.junit.platform.engine.TestDescriptor; + +public class FailFastClassOrderer implements ClassOrderer { + + private final TestEventsHandler testEventsHandler; + private final @Nullable ClassOrderer delegate; + private final Comparator knownTestSuitesComparator; + + public FailFastClassOrderer( + TestEventsHandler testEventsHandler, + @Nullable ClassOrderer delegate) { + this.testEventsHandler = testEventsHandler; + this.delegate = delegate; + this.knownTestSuitesComparator = Comparator.comparing(this::knownAndStableTestsPercentage); + } + + private int knownAndStableTestsPercentage(ClassDescriptor classDescriptor) { + TestDescriptor testDescriptor = JUnit5OrderUtils.getTestDescriptor(classDescriptor); + return 100 - unknownAndFlakyTestsPercentage(testDescriptor); + } + + private int unknownAndFlakyTestsPercentage(TestDescriptor testDescriptor) { + if (testDescriptor == null) { + return 0; + } + if (testDescriptor.isTest() || JUnitPlatformUtils.isParameterizedTest(testDescriptor)) { + TestIdentifier testIdentifier = JUnitPlatformUtils.toTestIdentifier(testDescriptor); + return testEventsHandler.isNew(testIdentifier) || testEventsHandler.isFlaky(testIdentifier) + ? 100 + : 0; + } + Set children = testDescriptor.getChildren(); + if (children.isEmpty()) { + return 0; + } + + int uknownTestsPercentage = 0; + for (TestDescriptor child : children) { + uknownTestsPercentage += unknownAndFlakyTestsPercentage(child); + } + return uknownTestsPercentage / children.size(); + } + + @Override + public void orderClasses(ClassOrdererContext classOrdererContext) { + if (delegate != null) { + // first use delegate if available + delegate.orderClasses(classOrdererContext); + } + // then apply our ordering (since sorting is stable, delegate's ordering will be preserved for + // classes with the same "known/stable" status) + classOrdererContext.getClassDescriptors().sort(knownTestSuitesComparator); + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastMethodOrderer.java b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastMethodOrderer.java new file mode 100644 index 00000000000..2b1d1385c89 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/FailFastMethodOrderer.java @@ -0,0 +1,46 @@ +package datadog.trace.instrumentation.junit5.order; + +import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.instrumentation.junit5.JUnitPlatformUtils; +import java.util.Comparator; +import javax.annotation.Nullable; +import org.junit.jupiter.api.MethodDescriptor; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.MethodOrdererContext; +import org.junit.platform.engine.TestDescriptor; + +public class FailFastMethodOrderer implements MethodOrderer { + + private final TestEventsHandler testEventsHandler; + private final @Nullable MethodOrderer delegate; + private final Comparator knownTestMethodsComparator; + + public FailFastMethodOrderer( + TestEventsHandler testEventsHandler, + @Nullable MethodOrderer delegate) { + this.testEventsHandler = testEventsHandler; + this.delegate = delegate; + this.knownTestMethodsComparator = Comparator.comparing(this::isKnownAndStable); + } + + private boolean isKnownAndStable(MethodDescriptor methodDescriptor) { + TestDescriptor testDescriptor = JUnit5OrderUtils.getTestDescriptor(methodDescriptor); + if (testDescriptor == null) { + return true; + } + TestIdentifier testIdentifier = JUnitPlatformUtils.toTestIdentifier(testDescriptor); + return !testEventsHandler.isNew(testIdentifier) && !testEventsHandler.isFlaky(testIdentifier); + } + + @Override + public void orderMethods(MethodOrdererContext methodOrdererContext) { + if (delegate != null) { + // first use delegate if available + delegate.orderMethods(methodOrdererContext); + } + // then apply our ordering (since sorting is stable, delegate's ordering will be preserved for + // methods with the same "known/stable" status) + methodOrdererContext.getMethodDescriptors().sort(knownTestMethodsComparator); + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5OrderUtils.java b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5OrderUtils.java new file mode 100644 index 00000000000..8fcd20ef126 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5OrderUtils.java @@ -0,0 +1,27 @@ +package datadog.trace.instrumentation.junit5.order; + +import datadog.trace.util.MethodHandles; +import java.lang.invoke.MethodHandle; +import org.junit.jupiter.api.ClassDescriptor; +import org.junit.jupiter.api.MethodDescriptor; +import org.junit.platform.commons.util.ClassLoaderUtils; +import org.junit.platform.engine.TestDescriptor; + +public class JUnit5OrderUtils { + + private static final MethodHandles METHOD_HANDLES = + new MethodHandles(ClassLoaderUtils.getDefaultClassLoader()); + + private static final MethodHandle GET_TEST_DESCRIPTOR = + METHOD_HANDLES.privateFieldGetter( + "org.junit.jupiter.engine.discovery.AbstractAnnotatedDescriptorWrapper", + "testDescriptor"); + + public static TestDescriptor getTestDescriptor(ClassDescriptor classDescriptor) { + return METHOD_HANDLES.invoke(GET_TEST_DESCRIPTOR, classDescriptor); + } + + public static TestDescriptor getTestDescriptor(MethodDescriptor methodDescriptor) { + return METHOD_HANDLES.invoke(GET_TEST_DESCRIPTOR, methodDescriptor); + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5TestOrderInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5TestOrderInstrumentation.java new file mode 100644 index 00000000000..fb58e56327b --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/main/java/datadog/trace/instrumentation/junit5/order/JUnit5TestOrderInstrumentation.java @@ -0,0 +1,127 @@ +package datadog.trace.instrumentation.junit5.order; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.implementsInterface; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Config; +import datadog.trace.api.civisibility.CIConstants; +import datadog.trace.bootstrap.CallDepthThreadLocalMap; +import datadog.trace.instrumentation.junit5.JUnitPlatformUtils; +import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder; +import datadog.trace.util.Strings; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Optional; +import java.util.Set; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import org.junit.jupiter.api.ClassOrderer; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.engine.config.JupiterConfiguration; + +@AutoService(InstrumenterModule.class) +public class JUnit5TestOrderInstrumentation extends InstrumenterModule.CiVisibility + implements Instrumenter.ForTypeHierarchy { + + private final String parentPackageName = + Strings.getPackageName(JUnitPlatformUtils.class.getName()); + + public JUnit5TestOrderInstrumentation() { + super("ci-visibility", "junit-5", "test-order"); + } + + @Override + public boolean isApplicable(Set enabledSystems) { + return super.isApplicable(enabledSystems) && Config.get().getCiVisibilityTestOrder() != null; + } + + @Override + public String hierarchyMarkerType() { + return "org.junit.jupiter.engine.config.JupiterConfiguration"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + return implementsInterface(named(hierarchyMarkerType())); + } + + @Override + public String[] helperClassNames() { + return new String[] { + parentPackageName + ".TestEventsHandlerHolder", + parentPackageName + ".JUnitPlatformUtils", + packageName + ".JUnit5OrderUtils", + packageName + ".FailFastClassOrderer", + packageName + ".FailFastMethodOrderer", + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + named("getDefaultTestClassOrderer"), + JUnit5TestOrderInstrumentation.class.getName() + "$ClassOrdererAdvice"); + transformer.applyAdvice( + named("getDefaultTestMethodOrderer"), + JUnit5TestOrderInstrumentation.class.getName() + "$MethodOrdererAdvice"); + } + + public static class ClassOrdererAdvice { + @Advice.OnMethodEnter + public static void onGetClassOrdererEnter() { + CallDepthThreadLocalMap.incrementCallDepth(JupiterConfiguration.class); + } + + @SuppressFBWarnings( + value = "UC_USELESS_OBJECT", + justification = "classOrderer is the return value of the instrumented method") + @Advice.OnMethodExit + public static void onGetClassOrdererExit( + @Advice.Return(readOnly = false) Optional classOrderer) { + if (CallDepthThreadLocalMap.decrementCallDepth(JupiterConfiguration.class) != 0) { + // nested call + return; + } + String testOrder = Config.get().getCiVisibilityTestOrder(); + if (CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(testOrder)) { + classOrderer = + Optional.of( + new FailFastClassOrderer( + TestEventsHandlerHolder.TEST_EVENTS_HANDLER, classOrderer.orElse(null))); + } else { + throw new IllegalArgumentException("Unknown test order: " + testOrder); + } + } + } + + public static class MethodOrdererAdvice { + @Advice.OnMethodEnter + public static void onGetMethodOrdererEnter() { + CallDepthThreadLocalMap.incrementCallDepth(JupiterConfiguration.class); + } + + @SuppressFBWarnings( + value = "UC_USELESS_OBJECT", + justification = "methodOrderer is the return value of the instrumented method") + @Advice.OnMethodExit + public static void onGetMethodOrdererExit( + @Advice.Return(readOnly = false) Optional methodOrderer) { + if (CallDepthThreadLocalMap.decrementCallDepth(JupiterConfiguration.class) != 0) { + // nested call + return; + } + String testOrder = Config.get().getCiVisibilityTestOrder(); + if (CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(testOrder)) { + methodOrderer = + Optional.of( + new FailFastMethodOrderer( + TestEventsHandlerHolder.TEST_EVENTS_HANDLER, methodOrderer.orElse(null))); + } else { + throw new IllegalArgumentException("Unknown test order: " + testOrder); + } + } + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy index 5c81fbcecab..dbfdaba2b67 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/groovy/JUnit58Test.groovy @@ -1,7 +1,11 @@ import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.CIConstants import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder import org.example.* +import org.junit.jupiter.api.ClassOrderer +import org.junit.jupiter.api.MethodOrderer +import org.junit.jupiter.engine.Constants import org.junit.jupiter.engine.JupiterTestEngine import org.junit.platform.engine.DiscoverySelector import org.junit.platform.launcher.core.LauncherConfig @@ -13,6 +17,12 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class JUnit58Test extends CiVisibilityInstrumentationTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + givenTestsOrder(CIConstants.FAIL_FAST_TEST_ORDER) + } + def "test #testcaseName"() { runTests(tests) @@ -28,6 +38,51 @@ class JUnit58Test extends CiVisibilityInstrumentationTest { "test-failed-after-each" | [TestFailedAfterEach] | 2 } + def "test known tests ordering #testcaseName"() { + givenKnownTests(knownTestsList) + + runTests(tests) + + assertTestsOrder(expectedOrder) + + where: + testcaseName | tests | knownTestsList | expectedOrder + "ordering-methods" | [TestSucceed] | [test("org.example.TestSucceed", "test_succeed_1")] | [ + test("org.example.TestSucceed", "test_succeed_2"), + test("org.example.TestSucceed", "test_succeed_1") + ] + "ordering-classes" | [TestSucceed, TestSucceedAnother] | [test("org.example.TestSucceed", "test_succeed_1")] | [ + test("org.example.TestSucceedAnother", "test_succeed_1"), + test("org.example.TestSucceed", "test_succeed_2"), + test("org.example.TestSucceed", "test_succeed_1") + ] + "ordering-parameterized-methods" | [TestParameterized] | [test("org.example.TestParameterized", "test_another_parameterized")] | [ + test("org.example.TestParameterized", "test_parameterized"), + test("org.example.TestParameterized", "test_another_parameterized") + ] + } + + def "test flaky tests ordering #testcaseName"() { + givenKnownTests(expectedOrder) + givenFlakyTests(flakyTestsList) + + runTests(tests) + + assertTestsOrder(expectedOrder) + + where: + testcaseName | tests | flakyTestsList | expectedOrder + "ordering-methods" | [TestSucceed] | [test("org.example.TestSucceed", "test_succeed_2")] | [ + test("org.example.TestSucceed", "test_succeed_2"), + test("org.example.TestSucceed", "test_succeed_1") + ] + "ordering-classes" | [TestSucceed, TestSucceedAnother] | [test("org.example.TestSucceedAnother", "test_succeed_1")] | [ + test("org.example.TestSucceedAnother", "test_succeed_1"), + test("org.example.TestSucceed", "test_succeed_1"), + test("org.example.TestSucceed", "test_succeed_2") + ] + } + private static void runTests(List> tests) { TestEventsHandlerHolder.startForcefully() @@ -37,6 +92,8 @@ class JUnit58Test extends CiVisibilityInstrumentationTest { } def launcherReq = LauncherDiscoveryRequestBuilder.request() + .configurationParameter(Constants.DEFAULT_TEST_CLASS_ORDER_PROPERTY_NAME, ClassOrderer.ClassName.name) + .configurationParameter(Constants.DEFAULT_TEST_METHOD_ORDER_PROPERTY_NAME, MethodOrderer.MethodName.name) .selectors(selectors) .build() diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestParameterized.java b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestParameterized.java new file mode 100644 index 00000000000..0b28259cfa5 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestParameterized.java @@ -0,0 +1,35 @@ +package org.example; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class TestParameterized { + + static List parameters() { + return Collections.singletonList(() -> new Object[] {0, 0, "0", "some:\"parameter\""}); + } + + @ParameterizedTest + @MethodSource("parameters") + public void test_parameterized( + final int first, final int second, final int expectedSum, final String message) { + final int actualSum = first + second; + assertEquals(expectedSum, actualSum); + assertNotNull(message); + } + + @ParameterizedTest + @MethodSource("parameters") + public void test_another_parameterized( + final int first, final int second, final int expectedSum, final String message) { + final int actualSum = first + second; + assertEquals(expectedSum, actualSum); + assertNotNull(message); + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceed.java b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceed.java new file mode 100644 index 00000000000..22907b66d3e --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceed.java @@ -0,0 +1,18 @@ +package org.example; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class TestSucceed { + + @Test + public void test_succeed_1() { + assertTrue(true); + } + + @Test + public void test_succeed_2() { + assertTrue(true); + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceedAnother.java b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceedAnother.java new file mode 100644 index 00000000000..f336cac5c27 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/java/org/example/TestSucceedAnother.java @@ -0,0 +1,13 @@ +package org.example; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class TestSucceedAnother { + + @Test + public void test_succeed_1() { + assertTrue(true); + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-all-after-all/events.ftl b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-all-after-all/events.ftl index 22736145e85..66970704b3d 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-all-after-all/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-all-after-all/events.ftl @@ -131,6 +131,7 @@ "test.module" : "junit-5.8", "test.status" : "pass", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "another_test_succeed", @@ -178,6 +179,7 @@ "test.module" : "junit-5.8", "test.status" : "pass", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "test_succeed", diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-each-after-each/events.ftl b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-each-after-each/events.ftl index 2090ff27a32..a0f2336c5ab 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-each-after-each/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-before-each-after-each/events.ftl @@ -131,6 +131,7 @@ "test.module" : "junit-5.8", "test.status" : "pass", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "another_test_succeed", @@ -178,6 +179,7 @@ "test.module" : "junit-5.8", "test.status" : "pass", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "test_succeed", @@ -198,9 +200,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id_2}, + "trace_id" : ${content_trace_id}, "span_id" : ${content_span_id_3}, - "parent_id" : ${content_span_id_2}, + "parent_id" : ${content_span_id}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "setUp", "resource" : "setUp", @@ -218,9 +220,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id}, + "trace_id" : ${content_trace_id_2}, "span_id" : ${content_span_id_4}, - "parent_id" : ${content_span_id}, + "parent_id" : ${content_span_id_2}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "setUp", "resource" : "setUp", @@ -238,9 +240,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id_2}, + "trace_id" : ${content_trace_id}, "span_id" : ${content_span_id_5}, - "parent_id" : ${content_span_id_2}, + "parent_id" : ${content_span_id}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "tearDown", "resource" : "tearDown", @@ -258,9 +260,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id}, + "trace_id" : ${content_trace_id_2}, "span_id" : ${content_span_id_6}, - "parent_id" : ${content_span_id}, + "parent_id" : ${content_span_id_2}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "tearDown", "resource" : "tearDown", diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-all/events.ftl b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-all/events.ftl index 2253dea2640..a12043d16d6 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-all/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-all/events.ftl @@ -134,6 +134,7 @@ "test.module" : "junit-5.8", "test.status" : "pass", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "another_test_succeed", @@ -181,6 +182,7 @@ "test.module" : "junit-5.8", "test.status" : "pass", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "test_succeed", diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-each/events.ftl b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-each/events.ftl index 180f3341f04..a758ae37c46 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-each/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-after-each/events.ftl @@ -131,6 +131,7 @@ "test.module" : "junit-5.8", "test.status" : "fail", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "another_test_succeed", @@ -181,6 +182,7 @@ "test.module" : "junit-5.8", "test.status" : "fail", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "test_succeed", @@ -204,9 +206,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id_2}, + "trace_id" : ${content_trace_id}, "span_id" : ${content_span_id_3}, - "parent_id" : ${content_span_id_2}, + "parent_id" : ${content_span_id}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "tearDown", "resource" : "tearDown", @@ -227,9 +229,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id}, + "trace_id" : ${content_trace_id_2}, "span_id" : ${content_span_id_4}, - "parent_id" : ${content_span_id}, + "parent_id" : ${content_span_id_2}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "tearDown", "resource" : "tearDown", diff --git a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-before-each/events.ftl b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-before-each/events.ftl index 6809b426552..d395452b0d9 100644 --- a/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-before-each/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/junit-5.8/src/test/resources/test-failed-before-each/events.ftl @@ -131,6 +131,7 @@ "test.module" : "junit-5.8", "test.status" : "fail", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "another_test_succeed", @@ -181,6 +182,7 @@ "test.module" : "junit-5.8", "test.status" : "fail", "language" : "jvm", + "test.is_new" : "true", "test.codeowners" : "[\"owner1\",\"owner2\"]", "library_version" : ${content_meta_library_version}, "test.name" : "test_succeed", @@ -204,9 +206,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id_2}, + "trace_id" : ${content_trace_id}, "span_id" : ${content_span_id_3}, - "parent_id" : ${content_span_id_2}, + "parent_id" : ${content_span_id}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "setUp", "resource" : "setUp", @@ -227,9 +229,9 @@ "type" : "span", "version" : 1, "content" : { - "trace_id" : ${content_trace_id}, + "trace_id" : ${content_trace_id_2}, "span_id" : ${content_span_id_4}, - "parent_id" : ${content_span_id}, + "parent_id" : ${content_span_id_2}, "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", "name" : "setUp", "resource" : "setUp", diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/build.gradle b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/build.gradle index b6ca4dfd623..3419895640f 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/build.gradle +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/build.gradle @@ -15,6 +15,7 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { implementation project(':dd-java-agent:instrumentation:junit-5.3') + compileOnly group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.7.2' compileOnly group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.2' compileOnly group: 'org.spockframework', name: 'spock-core', version: "2.0-groovy-${spockGroovyVersion}" diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockInstrumentation.java index e7e4d07c01d..b82dae34334 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockInstrumentation.java @@ -7,16 +7,11 @@ import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.bootstrap.ContextStore; -import datadog.trace.bootstrap.InstrumentationContext; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.Collections; -import java.util.Map; import net.bytebuddy.asm.Advice; import net.bytebuddy.matcher.ElementMatcher; import org.junit.platform.engine.EngineExecutionListener; import org.junit.platform.engine.ExecutionRequest; -import org.junit.platform.engine.TestDescriptor; import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService; import org.spockframework.runtime.SpockEngine; @@ -51,11 +46,6 @@ public String[] helperClassNames() { }; } - @Override - public Map contextStore() { - return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object"); - } - @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -87,12 +77,6 @@ public static void addTracingListener( return; } - ContextStore contextStore = - InstrumentationContext.get(TestDescriptor.class, Object.class); - TestEventsHandlerHolder.setContextStores( - (ContextStore) contextStore, (ContextStore) contextStore); - TestEventsHandlerHolder.start(); - SpockTracingListener tracingListener = new SpockTracingListener(testEngine); EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener(); EngineExecutionListener compositeListener = diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index 8e45825c72e..6d316ee4de1 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -1,4 +1,5 @@ import datadog.trace.api.DisableTestTrace +import datadog.trace.api.civisibility.CIConstants import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder @@ -27,6 +28,12 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class SpockTest extends CiVisibilityInstrumentationTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + givenTestsOrder(CIConstants.FAIL_FAST_TEST_ORDER) + } + def "test #testcaseName"() { runTests(tests) @@ -63,6 +70,7 @@ class SpockTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runTests(tests) @@ -83,6 +91,7 @@ class SpockTest extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runTests(tests) diff --git a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5Instrumentation.java b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5Instrumentation.java index 8caf64042f0..33049b4975f 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5Instrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5Instrumentation.java @@ -61,11 +61,31 @@ public Map contextStore() { @Override public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + named("discover") + .and( + takesArgument(0, named("org.junit.platform.engine.EngineDiscoveryRequest")) + .and(takesArgument(1, named("org.junit.platform.engine.UniqueId")))), + JUnit5Instrumentation.class.getName() + "$ContextStoreAdvice"); transformer.applyAdvice( named("execute").and(takesArgument(0, named("org.junit.platform.engine.ExecutionRequest"))), JUnit5Instrumentation.class.getName() + "$JUnit5Advice"); } + public static class ContextStoreAdvice { + @SuppressFBWarnings( + value = "UC_USELESS_OBJECT", + justification = "executionRequest is the argument of the original method") + @Advice.OnMethodEnter + public static void setContextStores() { + ContextStore contextStore = + InstrumentationContext.get(TestDescriptor.class, Object.class); + TestEventsHandlerHolder.setContextStores( + (ContextStore) contextStore, (ContextStore) contextStore); + TestEventsHandlerHolder.start(); + } + } + public static class JUnit5Advice { @SuppressFBWarnings( value = "UC_USELESS_OBJECT", @@ -94,12 +114,6 @@ public static void addTracingListener( return; } - ContextStore contextStore = - InstrumentationContext.get(TestDescriptor.class, Object.class); - TestEventsHandlerHolder.setContextStores( - (ContextStore) contextStore, (ContextStore) contextStore); - TestEventsHandlerHolder.start(); - TracingListener tracingListener = new TracingListener(testEngine); EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener(); EngineExecutionListener compositeListener = diff --git a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java index 2a10d78066c..48b64fb5de7 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java +++ b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java @@ -157,6 +157,13 @@ public static boolean isSuite(TestDescriptor testDescriptor) { || "nested-class".equals(lastSegment.getType()); // nested JUnit test class } + public static boolean isParameterizedTest(TestDescriptor testDescriptor) { + UniqueId uniqueId = testDescriptor.getUniqueId(); + List segments = uniqueId.getSegments(); + UniqueId.Segment lastSegment = segments.get(segments.size() - 1); + return "test-template".equals(lastSegment.getType()); + } + public static boolean isRetry(TestDescriptor testDescriptor) { UniqueId uniqueId = testDescriptor.getUniqueId(); List segments = uniqueId.getSegments(); diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy index dea3e7eaae6..8f58d5f18ec 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy @@ -97,6 +97,7 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runTests(tests) @@ -117,6 +118,7 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runTests(tests) diff --git a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy index 76cc50de5d5..73ec97a0cbd 100644 --- a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy @@ -65,6 +65,7 @@ class KarateTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runTests(tests) @@ -84,6 +85,7 @@ class KarateTest extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runTests(tests) diff --git a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy index e28262c55a4..1646f5173ac 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy +++ b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy @@ -51,6 +51,7 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runTests(tests) @@ -68,6 +69,7 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runTests(tests) diff --git a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index 0713854760d..28a892fd542 100644 --- a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -110,6 +110,7 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { } def "test flaky retries #testcaseName"() { + givenFlakyRetryEnabled(true) givenFlakyTests(retriedTests) runTests(tests, null) @@ -128,6 +129,7 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { def "test early flakiness detection #testcaseName"() { Assumptions.assumeTrue(isEFDSupported()) + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) runTests(tests) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java index bfd00a91fcd..2e8324e4224 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java @@ -71,6 +71,7 @@ public final class CiVisibilityConfig { "civisibility.remote.env.vars.provider.url"; public static final String CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY = "civisibility.remote.env.vars.provider.key"; + public static final String CIVISIBILITY_TEST_ORDER = "civisibility.test.order"; /* COVERAGE SETTINGS */ public static final String CIVISIBILITY_CODE_COVERAGE_ENABLED = diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index d0b3c7e8235..cde58e539ca 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -360,6 +360,7 @@ public static String getHostName() { private final boolean ciVisibilityAutoInjected; private final String ciVisibilityRemoteEnvVarsProviderUrl; private final String ciVisibilityRemoteEnvVarsProviderKey; + private final String ciVisibilityTestOrder; private final boolean remoteConfigEnabled; private final boolean remoteConfigIntegrityCheckEnabled; @@ -1453,6 +1454,7 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getString(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL); ciVisibilityRemoteEnvVarsProviderKey = configProvider.getString(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY); + ciVisibilityTestOrder = configProvider.getString(CIVISIBILITY_TEST_ORDER); remoteConfigEnabled = configProvider.getBoolean( @@ -2837,6 +2839,10 @@ public String getCiVisibilityRemoteEnvVarsProviderKey() { return ciVisibilityRemoteEnvVarsProviderKey; } + public String getCiVisibilityTestOrder() { + return ciVisibilityTestOrder; + } + public String getAppSecRulesFile() { return appSecRulesFile; } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java b/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java index 262732e7fd3..bc5e89eb5e5 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java @@ -10,4 +10,6 @@ public interface CIConstants { String SELENIUM_BROWSER_DRIVER = "selenium"; String CI_VISIBILITY_INSTRUMENTATION_NAME = "civisibility"; + + String FAIL_FAST_TEST_ORDER = "FAILFAST"; } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java index 218017b68e6..767c2663c0c 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java @@ -78,6 +78,10 @@ void onTestIgnore( @Nonnull TestRetryPolicy retryPolicy(TestIdentifier test); + boolean isNew(TestIdentifier test); + + boolean isFlaky(TestIdentifier test); + @Override void close(); diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java index bdbfbc163fe..1a3c87085d3 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java @@ -1,5 +1,6 @@ package datadog.trace.api.civisibility.telemetry; +import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled; import datadog.trace.api.civisibility.telemetry.tag.AutoInjected; import datadog.trace.api.civisibility.telemetry.tag.BrowserDriver; import datadog.trace.api.civisibility.telemetry.tag.Command; @@ -11,6 +12,8 @@ import datadog.trace.api.civisibility.telemetry.tag.ErrorType; import datadog.trace.api.civisibility.telemetry.tag.EventType; import datadog.trace.api.civisibility.telemetry.tag.ExitCode; +import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled; +import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled; import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner; import datadog.trace.api.civisibility.telemetry.tag.IsBenchmark; import datadog.trace.api.civisibility.telemetry.tag.IsHeadless; @@ -33,7 +36,12 @@ public enum CiVisibilityCountMetric { * The number of test sessions started. This metric is separate from event_created to avoid * increasing the cardinality too much */ - TEST_SESSION("test_session", Provider.class, AutoInjected.class), + TEST_SESSION( + "test_session", + Provider.class, + AutoInjected.class, + AgentlessLogSubmissionEnabled.class, + FailFastTestOrderEnabled.class), /** The number of events created */ EVENT_CREATED( "event_created", @@ -97,6 +105,7 @@ public enum CiVisibilityCountMetric { ItrSkipEnabled.class, CoverageEnabled.class, EarlyFlakeDetectionEnabled.class, + FlakyTestRetriesEnabled.class, RequireGit.class), /** The number of requests sent to the itr skippable tests endpoint */ ITR_SKIPPABLE_TESTS_REQUEST("itr_skippable_tests.request", RequestCompressed.class), @@ -116,8 +125,12 @@ public enum CiVisibilityCountMetric { ITR_FORCED_RUN("itr_forced_run", EventType.class), /** The number of requests sent to the known tests endpoint */ EFD_REQUEST("early_flake_detection.request", RequestCompressed.class), - /** The number of known tests requests sent to the endpoint that errored */ - EFD_REQUEST_ERRORS("early_flake_detection.request_errors", ErrorType.class, StatusCode.class); + /** The number of known tests requests sent to the known tests endpoint that errored */ + EFD_REQUEST_ERRORS("early_flake_detection.request_errors", ErrorType.class, StatusCode.class), + /** The number of requests sent to the flaky tests endpoint */ + FLAKY_TESTS_REQUEST("flaky_tests.request", RequestCompressed.class), + /** The number of known tests requests sent to the flaky tests that errored */ + FLAKY_TESTS_REQUEST_ERRORS("flaky_tests.request_errors", ErrorType.class, StatusCode.class); // need a "holder" class, as accessing static fields from enum constructors is illegal static class IndexHolder { diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java index 78bdba67119..7b8ef4a9d15 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityDistributionMetric.java @@ -44,7 +44,13 @@ public enum CiVisibilityDistributionMetric { /** The number of bytes received by the known tests endpoint */ EFD_RESPONSE_BYTES("early_flake_detection.response_bytes", ResponseCompressed.class), /** The number of tests received by the known tests endpoint */ - EFD_RESPONSE_TESTS("early_flake_detection.response_tests"); + EFD_RESPONSE_TESTS("early_flake_detection.response_tests"), + /* The time it takes to get the response of the flaky tests endpoint request in ms */ + FLAKY_TESTS_REQUEST_MS("flaky_tests.request_ms"), + /** The number of bytes received by the flaky tests endpoint */ + FLAKY_TESTS_RESPONSE_BYTES("flaky_tests.response_bytes", ResponseCompressed.class), + /** The number of tests received by the flaky tests endpoint */ + FLAKY_TESTS_RESPONSE_TESTS("flaky_tests.response_tests"); private static final String NAMESPACE = "civisibility"; diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/AgentlessLogSubmissionEnabled.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/AgentlessLogSubmissionEnabled.java new file mode 100644 index 00000000000..c82f10e7ee9 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/AgentlessLogSubmissionEnabled.java @@ -0,0 +1,12 @@ +package datadog.trace.api.civisibility.telemetry.tag; + +import datadog.trace.api.civisibility.telemetry.TagValue; + +public enum AgentlessLogSubmissionEnabled implements TagValue { + TRUE; + + @Override + public String asString() { + return "agentless_log_submission_enabled:true"; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FailFastTestOrderEnabled.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FailFastTestOrderEnabled.java new file mode 100644 index 00000000000..2c66e7a0122 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FailFastTestOrderEnabled.java @@ -0,0 +1,12 @@ +package datadog.trace.api.civisibility.telemetry.tag; + +import datadog.trace.api.civisibility.telemetry.TagValue; + +public enum FailFastTestOrderEnabled implements TagValue { + TRUE; + + @Override + public String asString() { + return "fail_fast_test_order_enabled:true"; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FlakyTestRetriesEnabled.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FlakyTestRetriesEnabled.java new file mode 100644 index 00000000000..b1008fe66a6 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/FlakyTestRetriesEnabled.java @@ -0,0 +1,12 @@ +package datadog.trace.api.civisibility.telemetry.tag; + +import datadog.trace.api.civisibility.telemetry.TagValue; + +public enum FlakyTestRetriesEnabled implements TagValue { + TRUE; + + @Override + public String asString() { + return "flaky_test_retries_enabled:true"; + } +} From 6d420cca514b8cf31c01d6228938f6c693188f47 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Thu, 5 Dec 2024 13:57:07 +0000 Subject: [PATCH 2/2] Custom ScopeManagers are deprecated and will be removed in a future release of dd-trace-ot (#8058) --- dd-trace-ot/src/main/java/datadog/opentracing/DDTracer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dd-trace-ot/src/main/java/datadog/opentracing/DDTracer.java b/dd-trace-ot/src/main/java/datadog/opentracing/DDTracer.java index be2db34e368..eb419c514a7 100644 --- a/dd-trace-ot/src/main/java/datadog/opentracing/DDTracer.java +++ b/dd-trace-ot/src/main/java/datadog/opentracing/DDTracer.java @@ -123,7 +123,10 @@ public DDTracerBuilder extractor(HttpCodec.Extractor extractor) { return this; } + @Deprecated public DDTracerBuilder scopeManager(ScopeManager scopeManager) { + log.warn( + "Custom ScopeManagers are deprecated and will be removed in a future release of dd-trace-ot"); this.scopeManager = scopeManager; return this; }