Fix for #248 skip tests with known failure messages

src/canopy/configuration.fs

@@ -37,4 +37,5 @@ let mutable showInfoDiv = true
 let mutable failureScreenshotsEnabled = true
 let mutable skipAllTestsOnFailure = false
 let mutable skipRemainingTestsInContextOnFailure = false
-let mutable skipNextTest = false
+let mutable skipNextTest = false
+let mutable failureMessagesThatShoulBeTreatedAsSkip : string list = []

src/canopy/runner.fs

@@ -69,25 +69,28 @@ let skip id =
     reporter.skip id
 
 let fail (ex : Exception) (test : Test) (suite : suite) autoFail url =
-    if skipAllTestsOnFailure = true || skipRemainingTestsInContextOnFailure = true then skipNextTest <- true
-    if autoFail then
-        skip test.Id //dont take the time to fail all the tests just skip them
+    if failureMessagesThatShoulBeTreatedAsSkip |> List.exists (fun message -> ex.Message = message) then
+        skip test.Id
     else
-        try
-            if failFast = ref true then failed <- true        
-            failedCount <- failedCount + 1
-            contextFailed <- true
-            let f = configuration.failScreenshotFileName test suite
-            if failureScreenshotsEnabled = true then
-              let ss = screenshot configuration.failScreenshotPath f
-              reporter.fail ex test.Id ss url
-            else reporter.fail ex test.Id Array.empty<byte> url
-        with 
-            | failExc -> 
-                //Fail during error report (likely  OpenQA.Selenium.WebDriverException.WebDriverTimeoutException ). 
-                // Don't fail the runner itself, but report it.
-                reporter.write (sprintf "Error during fail reporting: %s" (failExc.ToString()))
-                reporter.fail ex test.Id Array.empty url
+        if skipAllTestsOnFailure = true || skipRemainingTestsInContextOnFailure = true then skipNextTest <- true
+        if autoFail then
+            skip test.Id //dont take the time to fail all the tests just skip them
+        else
+            try
+                if failFast = ref true then failed <- true        
+                failedCount <- failedCount + 1
+                contextFailed <- true
+                let f = configuration.failScreenshotFileName test suite
+                if failureScreenshotsEnabled = true then
+                  let ss = screenshot configuration.failScreenshotPath f
+                  reporter.fail ex test.Id ss url
+                else reporter.fail ex test.Id Array.empty<byte> url
+            with 
+                | failExc -> 
+                    //Fail during error report (likely  OpenQA.Selenium.WebDriverException.WebDriverTimeoutException ). 
+                    // Don't fail the runner itself, but report it.
+                    reporter.write (sprintf "Error during fail reporting: %s" (failExc.ToString()))
+                    reporter.fail ex test.Id Array.empty url
 
 let safelyGetUrl () = if browser = null then "no browser = no url" else browser.Url
 

tests/basictests/Program.fs

@@ -13,6 +13,7 @@ pageTimeout <- 3.0
 runFailedContextsFirst <- true
 reporter <- new LiveHtmlReporter(Chrome, configuration.chromeDir) :> IReporter 
 reporter.setEnvironment "My Machine"
+configuration.failureMessagesThatShoulBeTreatedAsSkip <- ["Skip me when I fail"]
 
 configuration.failScreenshotFileName <- 
   (fun test suite -> 
@@ -36,6 +37,10 @@ let testpage = "http://lefthandedgoat.github.io/canopy/testpages/"
 
 "intentionally skipped shows blue in LiveHtmlReport" &&! skipped
 
+"Should skip even though it fails" &&& fun _ ->    
+    url testpage
+    failwith "Skip me when I fail"
+
 "Apostrophes don't break anything" &&& fun _ ->    
     url testpage
     count "I've got an apostrophe" 1
@@ -746,4 +751,4 @@ coverage testpage
 coverage()
 coverage "http://scrumy.com/silenter39delayed"
 
-quit()
+//quit()