Merge branch 'master' into cv/script-runner-const

JuliaTesting · May 22, 2024 · 090e76b · 090e76b
2 parents d80e1ca + 16f8d25
commit 090e76b
Show file tree

Hide file tree

Showing 14 changed files with 347 additions and 44 deletions.
diff --git a/src/runner.jl b/src/runner.jl
@@ -142,6 +142,7 @@ function gen_runner_code(testfilename, logfilename, test_args)
 
         using Test
         using TestReports
+        using TestReports.EzXML: prettyprint
 
         append!(empty!(ARGS), $(repr(test_args.exec)))
 
@@ -150,7 +151,9 @@ function gen_runner_code(testfilename, logfilename, test_args)
         end
 
         # Flatten before calling `report` to avoid a `deepcopy`.
-        write($(repr(logfilename)), report(TestReports.flatten_results!(ts)))
+        open($(repr(logfilename)), "w") do io
+            prettyprint(io, report(TestReports.flatten_results!(ts)))
+        end
         any_problems(ts) && exit(TestReports.TESTS_FAILED)
         """
     return runner_code

diff --git a/test/references/complexexample.txt b/test/references/complexexample.txt
@@ -1,25 +1,83 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="19" failures="6" errors="4"><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="4 % 2 == 0" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
+<testsuites tests="19" failures="6" errors="4">
+  <testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0">
+    <testcase name="4 % 2 == 0" id="1" classname="Math" time="0.0"/>
+    <testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0">
+      <failure message="4.47213595499958 == 5" type="test">Test Failed
   Expression: sqrt(20) == 5
    Evaluated: 4.47213595499958 == 5
-</failure></testcase><testcase name="16 == 16" id="3" classname="Math" time="0.0"/></testsuite><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="1 * 3 == 3" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
+</failure>
+    </testcase>
+    <testcase name="16 == 16" id="3" classname="Math" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1">
+    <testcase name="1 * 3 == 3" id="1" classname="Math/Multiplication" time="0.0"/>
+    <testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0">
+      <failure message="2 == 5" type="test">Test Failed
   Expression: 1 * 2 == 5
    Evaluated: 2 == 5
-</failure></testcase><testcase name="1 * 4 == 4" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="1 + 1 == 2" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
+</failure>
+    </testcase>
+    <testcase name="1 * 4 == 4" id="3" classname="Math/Multiplication" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2">
+    <testcase name="1 + 1 == 2" id="1" classname="Math/addition" time="0.0"/>
+    <testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0">
+      <failure message="3 == 5" type="test">Test Failed
   Expression: 1 + 2 == 5
    Evaluated: 3 == 5
-</failure></testcase><testcase name="1 + 4 == 5" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="1 + -1 == 0" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
+</failure>
+    </testcase>
+    <testcase name="1 + 4 == 5" id="3" classname="Math/addition" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3">
+    <testcase name="1 + -1 == 0" id="1" classname="Math/addition/negative addition" time="0.0"/>
+    <testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0">
+      <failure message="-1 == 1" type="test">Test Failed
   Expression: 1 + -2 == 1
    Evaluated: -1 == 1
-</failure></testcase><testcase name="10 + -5 == 5" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><testcase name="1 / 0" id="2" classname="Math/other" time="0.0"><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"></error></testcase><testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0"><error message="Nooo" type="ErrorException">Nooo
-</error></testcase><testcase name="true" id="4" classname="Math/other" time="0.0"><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"></error></testcase></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0"><error message="Got exception outside of a @test" type="ErrorException">Outside of tests
-</error></testcase></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
+</failure>
+    </testcase>
+    <testcase name="10 + -5 == 5" id="3" classname="Math/addition/negative addition" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="4">
+    <testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0">
+      <skip/>
+    </testcase>
+    <testcase name="1 / 0" id="2" classname="Math/other" time="0.0">
+      <error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"/>
+    </testcase>
+    <testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0">
+      <error message="Nooo" type="ErrorException">Nooo
+</error>
+    </testcase>
+    <testcase name="true" id="4" classname="Math/other" time="0.0">
+      <error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"/>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="5">
+    <testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0">
+      <error message="Got exception outside of a @test" type="ErrorException">Outside of tests
+</error>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6">
+    <testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0">
+      <failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
   Expression: throw(ArgumentError("1"))
     Expected: DimensionMismatch
       Thrown: ArgumentError
-</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
+</failure>
+    </testcase>
+    <testcase name="true" id="2" classname="Math/Different failures" time="0.0">
+      <failure message="No exception thrown" type="test_throws_nothing">Test Failed
   Expression: true
     Expected: DimensionMismatch
   No exception thrown
-</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="nthreads() &gt; 0" id="1" classname="Math/using function from a module" time="0.0"/></testsuite></testsuites>
-
+</failure>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7">
+    <testcase name="nthreads() &gt; 0" id="1" classname="Math/using function from a module" time="0.0"/>
+  </testsuite>
+</testsuites>
diff --git a/test/references/complexexample_pre_1_7.txt b/test/references/complexexample_pre_1_7.txt
@@ -1,19 +1,77 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="19" failures="6" errors="4"><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
+<testsuites tests="19" failures="6" errors="4">
+  <testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0">
+    <testcase name="pass (info lost) (Test 1)" id="1" classname="Math" time="0.0"/>
+    <testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0">
+      <failure message="4.47213595499958 == 5" type="test">Test Failed
   Expression: sqrt(20) == 5
-   Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math" time="0.0"/></testsuite><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
+   Evaluated: 4.47213595499958 == 5</failure>
+    </testcase>
+    <testcase name="pass (info lost) (Test 3)" id="3" classname="Math" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1">
+    <testcase name="pass (info lost) (Test 1)" id="1" classname="Math/Multiplication" time="0.0"/>
+    <testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0">
+      <failure message="2 == 5" type="test">Test Failed
   Expression: 1 * 2 == 5
-   Evaluated: 2 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
+   Evaluated: 2 == 5</failure>
+    </testcase>
+    <testcase name="pass (info lost) (Test 3)" id="3" classname="Math/Multiplication" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2">
+    <testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition" time="0.0"/>
+    <testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0">
+      <failure message="3 == 5" type="test">Test Failed
   Expression: 1 + 2 == 5
-   Evaluated: 3 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
+   Evaluated: 3 == 5</failure>
+    </testcase>
+    <testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3">
+    <testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition/negative addition" time="0.0"/>
+    <testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0">
+      <failure message="-1 == 1" type="test">Test Failed
   Expression: 1 + -2 == 1
-   Evaluated: -1 == 1</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><testcase name="1 / 0" id="2" classname="Math/other" time="0.0"><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"></error></testcase><testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0"><error message="Nooo" type="ErrorException">Nooo
-</error></testcase><testcase name="true" id="4" classname="Math/other" time="0.0"><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"></error></testcase></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0"><error message="Got exception outside of a @test" type="ErrorException">Outside of tests
-</error></testcase></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
+   Evaluated: -1 == 1</failure>
+    </testcase>
+    <testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition/negative addition" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="4">
+    <testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0">
+      <skip/>
+    </testcase>
+    <testcase name="1 / 0" id="2" classname="Math/other" time="0.0">
+      <error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"/>
+    </testcase>
+    <testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0">
+      <error message="Nooo" type="ErrorException">Nooo
+</error>
+    </testcase>
+    <testcase name="true" id="4" classname="Math/other" time="0.0">
+      <error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"/>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="5">
+    <testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0">
+      <error message="Got exception outside of a @test" type="ErrorException">Outside of tests
+</error>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6">
+    <testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0">
+      <failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
   Expression: throw(ArgumentError("1"))
     Expected: DimensionMismatch
-      Thrown: ArgumentError</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
+      Thrown: ArgumentError</failure>
+    </testcase>
+    <testcase name="true" id="2" classname="Math/Different failures" time="0.0">
+      <failure message="No exception thrown" type="test_throws_nothing">Test Failed
   Expression: true
     Expected: DimensionMismatch
-  No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/using function from a module" time="0.0"/></testsuite></testsuites>
-
+  No exception thrown</failure>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7">
+    <testcase name="pass (info lost) (Test 1)" id="1" classname="Math/using function from a module" time="0.0"/>
+  </testsuite>
+</testsuites>