JuliaTesting · omus · May 22, 2024 · May 21, 2024 · May 21, 2024 · May 21, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,13 +30,9 @@ jobs:
           arch: ${{ matrix.arch }}
       - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
-        continue-on-error: ${{ matrix.version == 'nightly' }}
       - uses: julia-actions/julia-runtest@v1
-        continue-on-error: ${{ matrix.version == 'nightly' }}
       - uses: julia-actions/julia-processcoverage@v1
-        continue-on-error: ${{ matrix.version == 'nightly' }}
       - uses: codecov/codecov-action@v4
-        continue-on-error: ${{ matrix.version == 'nightly' }}
         with:
           files: lcov.info
   docs:

diff --git a/src/runner.jl b/src/runner.jl
@@ -393,27 +393,26 @@ function test!(pkg::AbstractString,
         cmd = gen_command(runner_code, julia_args, coverage)
         test_folder_has_project_file = has_test_project_file(testfilepath)
 
-        if VERSION >= v"1.4.0" || (VERSION >= v"1.2.0" && test_folder_has_project_file)
+        if VERSION >= v"1.4" || (VERSION >= v"1.2" && test_folder_has_project_file)
             # Operations.sandbox() has different arguments between versions
-            sandbox_args = (ctx,
-                            pkgspec,
-                            pkgspec.path,
-                            joinpath(pkgspec.path, "test"))
-            if VERSION >= v"1.8.0"
-                test_project_override = test_folder_has_project_file ?
-                    nothing :
+            test_project_override = if VERSION >= v"1.4" && !test_folder_has_project_file
+                if VERSION >= v"1.8"
                     gen_target_project(ctx, pkgspec, pkgspec.path::String, "test")
-                sandbox_args = (sandbox_args..., test_project_override)
-            elseif VERSION >= v"1.7.0"
-                test_project_override = test_folder_has_project_file ?
-                    nothing :
+                elseif VERSION >= v"1.7"
                     gen_target_project(ctx.env, ctx.registries, pkgspec, pkgspec.path, "test")
-                sandbox_args = (sandbox_args..., test_project_override)
-            elseif VERSION >= v"1.4.0"
-                test_project_override = test_folder_has_project_file ?
-                    nothing :
+                else
                     gen_target_project(ctx, pkgspec, pkgspec.path, "test")
-                sandbox_args = (sandbox_args..., test_project_override)
+                end
+            else
+                nothing
+            end
+
+            sandbox_args = if VERSION >= v"1.11-"
+                (ctx, pkgspec, joinpath(pkgspec.path, "test"), test_project_override)
+            elseif VERSION >= v"1.4"
+                (ctx, pkgspec, pkgspec.path, joinpath(pkgspec.path, "test"), test_project_override)
+            else
+                (ctx, pkgspec, pkgspec.path, joinpath(pkgspec.path, "test"))
             end
 
             sandbox(sandbox_args...) do

diff --git a/test/references/complexexample.txt b/test/references/complexexample.txt
@@ -67,7 +67,8 @@
   Expression: throw(ArgumentError("1"))
     Expected: DimensionMismatch
       Thrown: ArgumentError
-</failure>
+      ArgumentError: 1
+      </failure>
     </testcase>
     <testcase name="true" id="2" classname="Math/Different failures" time="0.0">
       <failure message="No exception thrown" type="test_throws_nothing">Test Failed

diff --git a/test/references/complexexample_pre_1_11.txt b/test/references/complexexample_pre_1_11.txt
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="19" failures="6" errors="4">
+  <testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0">
+    <testcase name="4 % 2 == 0" id="1" classname="Math" time="0.0"/>
+    <testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0">
+      <failure message="4.47213595499958 == 5" type="test">Test Failed
+  Expression: sqrt(20) == 5
+   Evaluated: 4.47213595499958 == 5
+</failure>
+    </testcase>
+    <testcase name="16 == 16" id="3" classname="Math" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1">
+    <testcase name="1 * 3 == 3" id="1" classname="Math/Multiplication" time="0.0"/>
+    <testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0">
+      <failure message="2 == 5" type="test">Test Failed
+  Expression: 1 * 2 == 5
+   Evaluated: 2 == 5
+</failure>
+    </testcase>
+    <testcase name="1 * 4 == 4" id="3" classname="Math/Multiplication" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2">
+    <testcase name="1 + 1 == 2" id="1" classname="Math/addition" time="0.0"/>
+    <testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0">
+      <failure message="3 == 5" type="test">Test Failed
+  Expression: 1 + 2 == 5
+   Evaluated: 3 == 5
+</failure>
+    </testcase>
+    <testcase name="1 + 4 == 5" id="3" classname="Math/addition" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3">
+    <testcase name="1 + -1 == 0" id="1" classname="Math/addition/negative addition" time="0.0"/>
+    <testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0">
+      <failure message="-1 == 1" type="test">Test Failed
+  Expression: 1 + -2 == 1
+   Evaluated: -1 == 1
+</failure>
+    </testcase>
+    <testcase name="10 + -5 == 5" id="3" classname="Math/addition/negative addition" time="0.0"/>
+  </testsuite>
+  <testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="4">
+    <testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0">
+      <skip/>
+    </testcase>
+    <testcase name="1 / 0" id="2" classname="Math/other" time="0.0">
+      <error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"/>
+    </testcase>
+    <testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0">
+      <error message="Nooo" type="ErrorException">Nooo
+</error>
+    </testcase>
+    <testcase name="true" id="4" classname="Math/other" time="0.0">
+      <error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"/>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="5">
+    <testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0">
+      <error message="Got exception outside of a @test" type="ErrorException">Outside of tests
+</error>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6">
+    <testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0">
+      <failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
+  Expression: throw(ArgumentError("1"))
+    Expected: DimensionMismatch
+      Thrown: ArgumentError
+</failure>
+    </testcase>
+    <testcase name="true" id="2" classname="Math/Different failures" time="0.0">
+      <failure message="No exception thrown" type="test_throws_nothing">Test Failed
+  Expression: true
+    Expected: DimensionMismatch
+  No exception thrown
+</failure>
+    </testcase>
+  </testsuite>
+  <testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7">
+    <testcase name="nthreads() &gt; 0" id="1" classname="Math/using function from a module" time="0.0"/>
+  </testsuite>
+</testsuites>
diff --git a/test/references/reporttests_fail_pre_1_9.xml b/test/references/reporttests_fail_pre_1_9.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="1" errors="0">
+  <testsuite name="reporttest" tests="1" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0">
+    <testcase name="ARGS == []" id="1" classname="reporttest" time="0.0">
+      <failure message="[&quot;foo&quot;, &quot;-e&quot;, &quot;bar&quot;] == Any[]" type="test">Test Failed
+  Expression: ARGS == []
+   Evaluated: ["foo", "-e", "bar"] == Any[]</failure>
+    </testcase>
+  </testsuite>
+</testsuites>
diff --git a/test/reportgeneration.jl b/test/reportgeneration.jl
@@ -12,8 +12,10 @@ const TEST_PKG = (name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-2369062
 end
 
 @testset "Complex Example" begin
-    test_file = if VERSION >= v"1.9.0-beta4.29"  # https://github.com/JuliaLang/julia/pull/48526
+    test_file = if VERSION >= v"1.11-"
         "references/complexexample.txt"
+    elseif VERSION >= v"1.9.0-beta4.29"  # https://github.com/JuliaLang/julia/pull/48526
+        "references/complexexample_pre_1_11.txt"
     elseif VERSION >= v"1.7.0"
         "references/complexexample_pre_1_9.txt"
     else

diff --git a/test/reporttests_script.jl b/test/reporttests_script.jl
@@ -15,7 +15,20 @@ else
     `$(TestReports.RUNNER_SCRIPT)`
 end
 test_script = "reporttests_testsets.jl"
-reference_suffix = VERSION >= v"1.7" ? "" : "_pre_1_7"
+
+reference_file_pass = if VERSION >= v"1.7"
+    "references/reporttests_pass.xml"
+else
+    "references/reporttests_pass_pre_1_7.xml"
+end
+
+reference_file_fail = if VERSION >= v"1.9"
+    "references/reporttests_fail.xml"
+elseif VERSION >= v"1.7"
+    "references/reporttests_fail_pre_1_9.xml"
+else
+    "references/reporttests_fail_pre_1_7.xml"
+end
 
 @testset "parse_args" begin
     include(TestReports.RUNNER_SCRIPT)
@@ -61,39 +74,36 @@ end
 end
 
 @testset "default output file" begin
-    reference_file = "references/reporttests_pass$reference_suffix.xml"
     output_file = "testlog.xml"
     p = run(ignorestatus(`$runner_cmd $test_script`))
     try
         @test success(p)
         @test isfile(output_file)
-        @test_reference reference_file read(output_file, String) |> clean_output
+        @test_reference reference_file_pass read(output_file, String) |> clean_output
     finally
         isfile(output_file) && rm(output_file)
     end
 end
 
 @testset "specify output file" begin
-    reference_file = "references/reporttests_pass$reference_suffix.xml"
     output_file = "junit-report.xml"
     p = run(ignorestatus(`$runner_cmd $test_script --output=$output_file`))
     try
         @test success(p)
         @test isfile(output_file)
-        @test_reference reference_file read(output_file, String) |> clean_output
+        @test_reference reference_file_pass read(output_file, String) |> clean_output
     finally
         isfile(output_file) && rm(output_file)
     end
 end
 
 @testset "test args" begin
-    reference_file = "references/reporttests_fail$reference_suffix.xml"
     output_file = "testlog.xml"
     p = run(ignorestatus(`$runner_cmd $test_script -- foo -e bar`))
     try
         @test !success(p)
         @test isfile(output_file)
-        @test_reference reference_file read(output_file, String) |> clean_output
+        @test_reference reference_file_fail read(output_file, String) |> clean_output
     finally
         isfile(output_file) && rm(output_file)
     end