add check to ensure junit5 tests are not used with junit4 Rule/ClassRule

README.md

@@ -159,6 +159,7 @@ Safe Logging can be found at [github.com/palantir/safe-logging](https://github.c
 - `SwitchStatementDefaultCase`: Switch statements should avoid using default cases. Default cases prevent the [MissingCasesInEnumSwitch](http://errorprone.info/bugpattern/MissingCasesInEnumSwitch.html) check from detecting when an enum value is not explicitly handled. This check is important to help avoid incorrect behavior when new enum values are introduced.
 - `GradleCacheableTaskAction`: Gradle plugins should not call `Task.doFirst` or `Task.doLast` with a lambda, as that is not cacheable. See [gradle/gradle#5510](https://github.com/gradle/gradle/issues/5510) for more details.
 - `PreferBuiltInConcurrentKeySet`: Discourage relying on Guava's `com.google.common.collect.Sets.newConcurrentHashSet()`, when Java's `java.util.concurrent.ConcurrentHashMap.newKeySet()` serves the same purpose.
+- `JUnit5RuleUsage`: Prevent accidental usage of `org.junit.Rule`/`org.junit.ClassRule` within Junit5 tests
 
 ## com.palantir.baseline-checkstyle
 Checkstyle rules can be suppressed on a per-line or per-block basis. (It is good practice to first consider formatting

baseline-error-prone/src/main/java/com/palantir/baseline/errorprone/JUnit5RuleUsage.java

@@ -0,0 +1,70 @@
+/*
+ * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.baseline.errorprone;
+
+import com.google.auto.service.AutoService;
+import com.google.errorprone.BugPattern;
+import com.google.errorprone.VisitorState;
+import com.google.errorprone.bugpatterns.BugChecker;
+import com.google.errorprone.matchers.Description;
+import com.google.errorprone.matchers.Matcher;
+import com.google.errorprone.matchers.Matchers;
+import com.google.errorprone.util.ASTHelpers;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.tree.VariableTree;
+import com.sun.tools.javac.code.Symbol;
+
+@AutoService(BugChecker.class)
+@BugPattern(
+        name = "JUnit5RuleUsage",
+        severity = BugPattern.SeverityLevel.ERROR,
+        summary = "Using Rule/ClassRules in Junit5 tests results in the rules silently not executing")
+public final class JUnit5RuleUsage extends BugChecker implements BugChecker.ClassTreeMatcher {
+    private static final String JUNIT4_RULE = "org.junit.Rule";
+    private static final String JUNIT4_CLASS_RULE = "org.junit.ClassRule";
+    private static final String JUNIT5_TEST_ANNOTATION = "org.junit.jupiter.api.Test";
+
+    private static final Matcher<ClassTree> hasJunit5TestCases =
+            Matchers.hasMethod(Matchers.hasAnnotationOnAnyOverriddenMethod(JUNIT5_TEST_ANNOTATION));
+
+    private static final Matcher<ClassTree> hasJunit4Rules = hasVariable(
+            Matchers.anyOf(hasAnnotationOnVariable(JUNIT4_CLASS_RULE), hasAnnotationOnVariable(JUNIT4_RULE)));
+
+    @Override
+    public Description matchClass(ClassTree tree, VisitorState state) {
+        if (hasJunit5TestCases.matches(tree, state) && hasJunit4Rules.matches(tree, state)) {
+            return buildDescription(tree)
+                    .setMessage("Do not use Rule/ClassRule with junit-jupiter")
+                    .build();
+        }
+
+        return Description.NO_MATCH;
+    }
+
+    static Matcher<ClassTree> hasVariable(Matcher<VariableTree> matcher) {
+        return (classTree, state) -> classTree.getMembers().stream()
+                .filter(tree -> tree instanceof VariableTree)
+                .anyMatch(tree -> matcher.matches((VariableTree) tree, state));
+    }
+
+    static Matcher<VariableTree> hasAnnotationOnVariable(String annotation) {
+        return (variableTree, state) -> {
+            Symbol.VarSymbol sym = ASTHelpers.getSymbol(variableTree);
+            return sym != null && ASTHelpers.hasAnnotation(sym, annotation, state);
+        };
+    }
+}

baseline-error-prone/src/test/java/com/palantir/baseline/errorprone/JUnit5RuleUsageTest.java

@@ -0,0 +1,75 @@
+/*
+ * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.baseline.errorprone;
+
+import com.google.errorprone.CompilationTestHelper;
+import org.junit.Before;
+import org.junit.Test;
+
+public class JUnit5RuleUsageTest {
+
+    private CompilationTestHelper compilationHelper;
+
+    @Before
+    public void before() {
+        compilationHelper = CompilationTestHelper.newInstance(JUnit5RuleUsage.class, getClass());
+    }
+
+    @Test
+    public void test_rule_with_junit5() {
+        compilationHelper.addSourceLines(
+                "TestCase.java",
+                "import org.junit.Rule;",
+                "import org.junit.jupiter.api.Test;",
+                "// BUG: Diagnostic contains: Do not use Rule/ClassRule",
+                "class TestCase {",
+                "@Rule public int foo = 1;",
+                "@Test",
+                "public void test() { }",
+                "}")
+                .doTest();
+    }
+
+    @Test
+    public void test_classrule_with_junit5() {
+        compilationHelper.addSourceLines(
+                "TestCase.java",
+                "import org.junit.ClassRule;",
+                "import org.junit.jupiter.api.Test;",
+                "// BUG: Diagnostic contains: Do not use Rule/ClassRule",
+                "class TestCase {",
+                "@ClassRule public static int foo = 1;",
+                "@Test",
+                "public void test() { }",
+                "}")
+                .doTest();
+    }
+
+    @Test
+    public void test_rule_with_junit4() {
+        compilationHelper.addSourceLines(
+                "TestCase.java",
+                "import org.junit.Rule;",
+                "import org.junit.Test;",
+                "class TestCase {",
+                "@Rule public static int foo = 1;",
+                "@Test",
+                "public void test() { }",
+                "}")
+                .doTest();
+    }
+}

changelog/@unreleased/pr-714.v2.yml

@@ -0,0 +1,6 @@
+type: improvement
+improvement:
+  description: Add errorprone check to ensure junit5 tests are not used with junit4
+    Rule/ClassRule
+  links:
+  - https://github.com/palantir/gradle-baseline/pull/714