Skip to content

Commit dcf73eb

Browse files
daniel-mohedanodaniel-mohedano
authored
Implement test disabling (#8377)
1 parent f8c95c8 commit dcf73eb

File tree

131 files changed

+2834
-215
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

131 files changed

+2834
-215
lines changed

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ TestSuiteImpl testSuiteStart(
3333

3434
boolean isQuarantined(TestIdentifier test);
3535

36+
boolean isDisabled(TestIdentifier test);
37+
3638
/**
3739
* Returns the reason for skipping a test, IF it can be skipped.
3840
*

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import datadog.trace.api.civisibility.telemetry.tag.BrowserDriver;
2020
import datadog.trace.api.civisibility.telemetry.tag.EventType;
2121
import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries;
22+
import datadog.trace.api.civisibility.telemetry.tag.IsDisabled;
2223
import datadog.trace.api.civisibility.telemetry.tag.IsModified;
2324
import datadog.trace.api.civisibility.telemetry.tag.IsNew;
2425
import datadog.trace.api.civisibility.telemetry.tag.IsQuarantined;
@@ -277,7 +278,8 @@ public void end(@Nullable Long endTime) {
277278
EventType.TEST,
278279
span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null,
279280
span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null,
280-
span.getTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
281+
span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
282+
span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED) != null ? IsDisabled.TRUE : null,
281283
span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
282284
span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
283285
retryReason instanceof TagValue ? (TagValue) retryReason : null,

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,11 @@ public boolean isQuarantined(TestIdentifier test) {
109109
return executionStrategy.isQuarantined(test);
110110
}
111111

112+
@Override
113+
public boolean isDisabled(TestIdentifier test) {
114+
return executionStrategy.isDisabled(test);
115+
}
116+
112117
@Nullable
113118
@Override
114119
public SkipReason skipReason(TestIdentifier test) {

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,11 @@ public boolean isQuarantined(TestIdentifier test) {
9494
return executionStrategy.isQuarantined(test);
9595
}
9696

97+
@Override
98+
public boolean isDisabled(TestIdentifier test) {
99+
return executionStrategy.isDisabled(test);
100+
}
101+
97102
@Nullable
98103
@Override
99104
public SkipReason skipReason(TestIdentifier test) {

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,11 @@ public void onTestStart(
168168
}
169169

170170
if (testModule.isQuarantined(thisTest)) {
171-
test.setTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED, true);
171+
test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED, true);
172+
}
173+
174+
if (testModule.isDisabled(thisTest)) {
175+
test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED, true);
172176
}
173177

174178
if (testExecutionHistory != null) {

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,15 @@ public SkipReason skipReason(TestIdentifier test) {
9090
if (test == null) {
9191
return null;
9292
}
93+
94+
if (isDisabled(test)) {
95+
return SkipReason.DISABLED;
96+
}
97+
9398
if (!executionSettings.isTestSkippingEnabled()) {
9499
return null;
95100
}
101+
96102
Map<TestIdentifier, TestMetadata> skippableTests = executionSettings.getSkippableTests();
97103
TestMetadata testMetadata = skippableTests.get(test);
98104
if (testMetadata == null) {

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -294,14 +294,17 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
294294

295295
def givenQuarantinedTests(List<TestFQN> tests) {
296296
quarantinedTests.addAll(tests)
297+
testManagementEnabled = true
297298
}
298299

299300
def givenDisabledTests(List<TestFQN> tests) {
300301
disabledTests.addAll(tests)
302+
testManagementEnabled = true
301303
}
302304

303305
def givenAttemptToFixTests(List<TestFQN> tests) {
304306
attemptToFixTests.addAll(tests)
307+
testManagementEnabled = true
305308
}
306309

307310
def givenDiff(Diff diff) {
@@ -320,10 +323,6 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
320323
this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled
321324
}
322325

323-
def givenTestManagementEnabled(boolean testManagementEnabled) {
324-
this.testManagementEnabled = testManagementEnabled
325-
}
326-
327326
def givenTestsOrder(String testsOrder) {
328327
injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
329328
}

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ public JUnit4CucumberSkipInstrumentation() {
3434

3535
@Override
3636
public boolean isApplicable(Set<TargetSystem> enabledSystems) {
37-
return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled();
37+
return super.isApplicable(enabledSystems)
38+
&& (Config.get().isCiVisibilityTestSkippingEnabled()
39+
|| Config.get().isCiVisibilityTestManagementEnabled());
3840
}
3941

4042
@Override

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
9191
}
9292

9393
def "test quarantined #testcaseName"() {
94-
givenTestManagementEnabled(true)
9594
givenQuarantinedTests(quarantined)
9695

9796
runFeatures(features, true)
@@ -106,7 +105,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
106105
}
107106

108107
def "test quarantined auto-retries #testcaseName"() {
109-
givenTestManagementEnabled(true)
110108
givenQuarantinedTests(quarantined)
111109

112110
givenFlakyRetryEnabled(true)
@@ -118,16 +116,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
118116
assertSpansData(testcaseName)
119117

120118
where:
121-
testcaseName | features | quarantined | retried
119+
testcaseName | features | quarantined | retried
122120
"test-quarantined-failed-atr" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
123121
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
124-
] | [
122+
] | [
125123
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
126124
]
127125
}
128126

129127
def "test quarantined early flakiness detection #testcaseName"() {
130-
givenTestManagementEnabled(true)
131128
givenQuarantinedTests(quarantined)
132129

133130
givenEarlyFlakinessDetectionEnabled(true)
@@ -139,15 +136,29 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
139136
assertSpansData(testcaseName)
140137

141138
where:
142-
testcaseName | features | quarantined | known
139+
testcaseName | features | quarantined | known
143140
"test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
144141
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
145-
] | [
142+
] | [
146143
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
147144
]
148145
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
149146
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
150-
] | []
147+
] | []
148+
}
149+
150+
def "test disabled #testcaseName"() {
151+
givenDisabledTests(disabled)
152+
153+
runFeatures(features, true)
154+
155+
assertSpansData(testcaseName)
156+
157+
where:
158+
testcaseName | features | disabled
159+
"test-disabled-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
160+
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
161+
]
151162
}
152163

153164
private String version() {

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

0 commit comments

Comments
 (0)