fix: JUnit 5 tests should not throw exception if only pending failures

pact-foundation · Jun 18, 2020 · 9fb20fd · 9fb20fd
1 parent 1b34871
commit 9fb20fd
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 10 deletions.
diff --git a/core/matchers/src/main/kotlin/au/com/dius/pact/core/matchers/Matching.kt b/core/matchers/src/main/kotlin/au/com/dius/pact/core/matchers/Matching.kt
@@ -36,10 +36,12 @@ object Matching : KLogging() {
       if (a.containsKey(values.key)) {
         val actual = a[values.key].orEmpty()
         list + values.value.mapIndexed { index, headerValue ->
-          HeaderMatcher.compareHeader(values.key, headerValue, actual.getOrElse(index) { "" }, matchers ?: MatchingRulesImpl())
+          HeaderMatcher.compareHeader(values.key, headerValue, actual.getOrElse(index) { "" },
+            matchers ?: MatchingRulesImpl())
         }.filterNotNull()
       } else {
-        list + HeaderMismatch(values.key, values.value.joinToString(separator = ", "), "", "Expected a header '${values.key}' but was missing")
+        list + HeaderMismatch(values.key, values.value.joinToString(separator = ", "), "",
+          "Expected a header '${values.key}' but was missing")
       }
     }
   }
@@ -113,14 +115,19 @@ object Matching : KLogging() {
   }
 
   @JvmStatic
-  fun compareMessageMetadata(e: Map<String, Any?>, a: Map<String, Any?>, matchers: MatchingRules?): List<MetadataMismatch> {
+  fun compareMessageMetadata(
+    e: Map<String, Any?>,
+    a: Map<String, Any?>,
+    matchers: MatchingRules?
+  ): List<MetadataMismatch> {
     return e.entries.fold(listOf()) { list, value ->
       if (a.containsKey(value.key)) {
         val actual = a[value.key]
         val compare = MetadataMatcher.compare(value.key, value.value, actual, matchers ?: MatchingRulesImpl())
         if (compare != null) list + compare else list
       } else if (value.key.toLowerCase() != "contenttype" && value.key.toLowerCase() != "content-type") {
-        list + MetadataMismatch(value.key, value.value, null, "Expected metadata '${value.key}' but was missing")
+        list + MetadataMismatch(value.key, value.value, null,
+          "Expected metadata '${value.key}' but was missing")
       } else {
         list
       }

diff --git a/...junit5/src/main/kotlin/au/com/dius/pact/provider/junit5/PactJUnit5VerificationProvider.kt b/...junit5/src/main/kotlin/au/com/dius/pact/provider/junit5/PactJUnit5VerificationProvider.kt
@@ -87,11 +87,14 @@ data class PactVerificationContext @JvmOverloads constructor(
     val request = store.get("request")
     val testContext = store.get("interactionContext") as PactVerificationContext
     try {
-      this.testExecutionResult.addAll(validateTestExecution(client, request, testContext.executionContext ?: emptyMap())
-        .filterIsInstance<VerificationResult.Failed>())
+      val result = validateTestExecution(client, request, testContext.executionContext ?: emptyMap())
+        .filterIsInstance<VerificationResult.Failed>()
+      this.testExecutionResult.addAll(result)
       if (testExecutionResult.isNotEmpty()) {
         verifier!!.displayFailures(testExecutionResult)
-        throw AssertionError(verifier!!.generateErrorStringFromVerificationResult(testExecutionResult))
+        if (testExecutionResult.any { !it.pending }) {
+          throw AssertionError(verifier!!.generateErrorStringFromVerificationResult(testExecutionResult))
+        }
       }
     } finally {
       verifier!!.finaliseReports()

diff --git a/...unit5/src/test/groovy/au/com/dius/pact/provider/junit5/PactVerificationContextSpec.groovy b/...unit5/src/test/groovy/au/com/dius/pact/provider/junit5/PactVerificationContextSpec.groovy
@@ -9,6 +9,7 @@ import au.com.dius.pact.provider.ConsumerInfo
 import au.com.dius.pact.provider.IConsumerInfo
 import au.com.dius.pact.provider.IProviderInfo
 import au.com.dius.pact.provider.IProviderVerifier
+import au.com.dius.pact.provider.VerificationFailureType
 import au.com.dius.pact.provider.VerificationResult
 import org.junit.jupiter.api.extension.ExtensionContext
 import spock.lang.Specification
@@ -51,10 +52,54 @@ class PactVerificationContextSpec extends Specification {
     then:
     thrown(AssertionError)
     context.testExecutionResult[0] instanceof VerificationResult.Failed
-    context.testExecutionResult[0].results.size() == 1
-    context.testExecutionResult[0].results[0].message == 'Request to provider failed with an exception'
-    context.testExecutionResult[0].results[0].exception.cause instanceof IOException
+    context.testExecutionResult[0].description == 'Request to provider failed with an exception'
+    context.testExecutionResult[0].failures.size() == 1
+    context.testExecutionResult[0].failures[0] instanceof VerificationFailureType.ExceptionFailure
     context.testExecutionResult[0].interactionId == '12345'
   }
 
+  @SuppressWarnings('UnnecessaryGetter')
+  def 'only throw an exception if there are non-pending failures'() {
+    given:
+    PactVerificationContext context
+    ExtensionContext.Store store = Stub {
+      get(_) >> { args ->
+        if (args[0] == 'interactionContext') {
+          context
+        }
+      }
+    }
+    ExtensionContext extContext = Stub {
+      getStore(_) >> store
+    }
+    TestTarget target = Stub {
+      executeInteraction(_, _) >> { throw new IOException('Boom!') }
+    }
+    IProviderVerifier verifier = Stub()
+    ValueResolver valueResolver = Stub()
+    IProviderInfo provider = Stub {
+      getName() >> 'Stub'
+    }
+    IConsumerInfo consumer = Mock(IConsumerInfo) {
+      getName() >> 'test'
+      getPending() >> true
+    }
+    Interaction interaction = new RequestResponseInteraction('Test Interaction', [], new Request(),
+      new Response(), '12345')
+    List<VerificationResult> testResults = []
+
+    context = new PactVerificationContext(store, extContext, target, verifier, valueResolver,
+      provider, consumer, interaction, testResults)
+
+    when:
+    context.verifyInteraction()
+
+    then:
+    noExceptionThrown()
+    context.testExecutionResult[0] instanceof VerificationResult.Failed
+    context.testExecutionResult[0].description == 'Request to provider failed with an exception'
+    context.testExecutionResult[0].failures.size() == 1
+    context.testExecutionResult[0].failures[0] instanceof VerificationFailureType.ExceptionFailure
+    context.testExecutionResult[0].interactionId == '12345'
+  }
 }
diff --git a/provider/src/main/kotlin/au/com/dius/pact/provider/VerificationResult.kt b/provider/src/main/kotlin/au/com/dius/pact/provider/VerificationResult.kt
@@ -16,6 +16,7 @@ private fun padLines(str: String, indent: Int): String {
 }
 
 sealed class VerificationFailureType {
+  abstract fun description(): String
   abstract fun formatForDisplay(t: TermColors): String
   abstract fun hasException(): Boolean
   abstract fun getException(): Throwable?
@@ -25,6 +26,7 @@ sealed class VerificationFailureType {
     val interaction: Interaction? = null,
     val pact: Pact<Interaction>? = null
   ) : VerificationFailureType() {
+    override fun description() = formatForDisplay(TermColors())
     override fun formatForDisplay(t: TermColors): String {
       return when (mismatch) {
         is BodyMismatch -> {
@@ -56,6 +58,7 @@ sealed class VerificationFailureType {
   }
 
   data class ExceptionFailure(val e: Throwable) : VerificationFailureType() {
+    override fun description() = e.message ?: e.javaClass.name
     override fun formatForDisplay(t: TermColors): String {
       return if (e.message.isNotEmpty()) {
         padLines(e.message!!, 6)
@@ -69,6 +72,7 @@ sealed class VerificationFailureType {
   }
 
   data class StateChangeFailure(val result: StateChangeResult) : VerificationFailureType() {
+    override fun description() = formatForDisplay(TermColors())
     override fun formatForDisplay(t: TermColors): String {
       val e = result.stateChangeResult.getError()
       return "State change callback failed with an exception - " + e?.message.toString()