Merge pull request #22315 from JuliaLang/cv/test-fixes

Fix issues with showing failed `@test` results

base/test.jl

@@ -188,19 +188,22 @@ struct Threw <: ExecutionResult
     backtrace
 end
 
-function eval_comparison(ex::Expr)
+function eval_comparison(evaluated::Expr, quoted::Expr)
     res = true
     i = 1
-    a = ex.args
-    n = length(a)
+    args = evaluated.args
+    quoted_args = quoted.args
+    n = length(args)
     while i < n
-        res = a[i+1](a[i], a[i+2])
-        if !isa(res,Bool) || !res
-            break
+        a, op, b = args[i], args[i+1], args[i+2]
+        if res
+            res = op(a, b) === true  # Keep `res` type stable
         end
+        quoted_args[i] = a
+        quoted_args[i+2] = b
         i += 2
     end
-    Returned(res, ex)
+    Returned(res, quoted)
 end
 
 const comparison_prec = Base.operator_precedence(:(==))
@@ -295,18 +298,19 @@ end
 function get_test_result(ex)
     orig_ex = Expr(:inert, ex)
     # Normalize non-dot comparison operator calls to :comparison expressions
-    if isa(ex, Expr) && ex.head == :call && length(ex.args)==3 &&
+    if isa(ex, Expr) && ex.head == :call && length(ex.args) == 3 &&
         first(string(ex.args[1])) != '.' &&
         (ex.args[1] === :(==) || Base.operator_precedence(ex.args[1]) == comparison_prec)
-        testret = :(eval_comparison(Expr(:comparison,
-                                         $(esc(ex.args[2])), $(esc(ex.args[1])), $(esc(ex.args[3])))))
-    elseif isa(ex, Expr) && ex.head == :comparison
+        ex = Expr(:comparison, ex.args[2], ex.args[1], ex.args[3])
+    end
+    if isa(ex, Expr) && ex.head == :comparison
         # pass all terms of the comparison to `eval_comparison`, as an Expr
-        terms = ex.args
-        for i = 1:length(terms)
-            terms[i] = esc(terms[i])
-        end
-        testret = :(eval_comparison(Expr(:comparison, $(terms...))))
+        escaped_terms = [esc(arg) for arg in ex.args]
+        quoted_terms = [QuoteNode(arg) for arg in ex.args]
+        testret = :(eval_comparison(
+            Expr(:comparison, $(escaped_terms...)),
+            Expr(:comparison, $(quoted_terms...)),
+        ))
     else
         testret = :(Returned($(esc(ex)), nothing))
     end

test/test.jl

@@ -7,6 +7,12 @@
 @test strip("\t  hi   \n") == "hi"
 @test strip("\t  this should fail   \n") != "hi"
 
+# @test should only evaluate the arguments once
+let g = Int[], f = (x) -> (push!(g, x); x)
+    @test f(1) == 1
+    @test g == [1]
+end
+
 # Test @test_broken with fail
 @test_broken false
 @test_broken 1 == 2
@@ -60,16 +66,46 @@ fails = @testset NoThrowTestSet begin
     @test_throws OverflowError 1 + 1
     # Fail - comparison
     @test 1+1 == 2+2
+    # Fail - approximate comparison
+    @test 1/1 ≈ 2/1
+    # Fail - chained comparison
+    @test 1+0 == 2+0 == 3+0
+    # Fail - comparison call
+    @test ==(1 - 2, 2 - 1)
     # Error - unexpected pass
     @test_broken true
 end
-for i in 1:3
+for i in 1:length(fails) - 1
     @test isa(fails[i], Base.Test.Fail)
 end
-@test contains(sprint(show, fails[1]), "Thrown: ErrorException")
-@test contains(sprint(show, fails[2]), "No exception thrown")
-@test contains(sprint(show, fails[3]), "Evaluated: 2 == 4")
-@test contains(sprint(show, fails[4]), "Unexpected Pass")
+
+str = sprint(show, fails[1])
+@test contains(str, "Expression: error()")
+@test contains(str, "Thrown: ErrorException")
+
+str = sprint(show, fails[2])
+@test contains(str, "Expression: 1 + 1")
+@test contains(str, "No exception thrown")
+
+str = sprint(show, fails[3])
+@test contains(str, "Expression: 1 + 1 == 2 + 2")
+@test contains(str, "Evaluated: 2 == 4")
+
+str = sprint(show, fails[4])
+@test contains(str, "Expression: 1 / 1 ≈ 2 / 1")
+@test contains(str, "Evaluated: 1.0 ≈ 2.0")
+
+str = sprint(show, fails[5])
+@test contains(str, "Expression: 1 + 0 == 2 + 0 == 3 + 0")
+@test contains(str, "Evaluated: 1 == 2 == 3")
+
+str = sprint(show, fails[6])
+@test contains(str, "Expression: 1 - 2 == 2 - 1")
+@test contains(str, "Evaluated: -1 == 1")
+
+str = sprint(show, fails[7])
+@test contains(str, "Unexpected Pass")
+@test contains(str, "Expression: true")
 
 # Test printing of a TestSetException
 tse_str = sprint(show, Test.TestSetException(1,2,3,4,Vector{Union{Base.Test.Error, Base.Test.Fail}}()))