From a33a4643a81dc3bc57c353b7297ea79d08481753 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 21 May 2024 14:42:46 -0500 Subject: [PATCH 01/13] Update sandbox for Julia 1.11 --- src/runner.jl | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/runner.jl b/src/runner.jl index b8c4ced..267024b 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -380,27 +380,26 @@ function test!(pkg::AbstractString, cmd = gen_command(runner_code, julia_args, coverage) test_folder_has_project_file = has_test_project_file(testfilepath) - if VERSION >= v"1.4.0" || (VERSION >= v"1.2.0" && test_folder_has_project_file) + if VERSION >= v"1.4" || (VERSION >= v"1.2" && test_folder_has_project_file) # Operations.sandbox() has different arguments between versions - sandbox_args = (ctx, - pkgspec, - pkgspec.path, - joinpath(pkgspec.path, "test")) - if VERSION >= v"1.8.0" - test_project_override = test_folder_has_project_file ? - nothing : + test_project_override = if VERSION >= v"1.4" && !test_folder_has_project_file + if VERSION >= v"1.8" gen_target_project(ctx, pkgspec, pkgspec.path::String, "test") - sandbox_args = (sandbox_args..., test_project_override) - elseif VERSION >= v"1.7.0" - test_project_override = test_folder_has_project_file ? - nothing : + elseif VERSION >= v"1.7" gen_target_project(ctx.env, ctx.registries, pkgspec, pkgspec.path, "test") - sandbox_args = (sandbox_args..., test_project_override) - elseif VERSION >= v"1.4.0" - test_project_override = test_folder_has_project_file ? - nothing : + else gen_target_project(ctx, pkgspec, pkgspec.path, "test") - sandbox_args = (sandbox_args..., test_project_override) + end + else + nothing + end + + sandbox_args = if VERSION >= v"1.11.0-" + (ctx, pkgspec, joinpath(pkgspec.path, "test"), test_project_override) + elseif VERSION >= v"1.4.0" + (ctx, pkgspec, pkgspec.path, joinpath(pkgspec.path, "test"), test_project_override) + else + (ctx, pkgspec, pkgspec.path, joinpath(pkgspec.path, "test")) end sandbox(sandbox_args...) do From a996d9a358ed0e52f787bb7ae7b2e9f493ac7fb3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 21 May 2024 15:17:20 -0500 Subject: [PATCH 02/13] Update test for Julia 1.11 --- .../{complexexample.txt => complexexample_pre_1_11.txt} | 0 test/reportgeneration.jl | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) rename test/references/{complexexample.txt => complexexample_pre_1_11.txt} (100%) diff --git a/test/references/complexexample.txt b/test/references/complexexample_pre_1_11.txt similarity index 100% rename from test/references/complexexample.txt rename to test/references/complexexample_pre_1_11.txt diff --git a/test/reportgeneration.jl b/test/reportgeneration.jl index 9187ee1..e6b9bdd 100644 --- a/test/reportgeneration.jl +++ b/test/reportgeneration.jl @@ -12,8 +12,10 @@ const TEST_PKG = (name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-2369062 end @testset "Complex Example" begin - test_file = if VERSION >= v"1.9.0-beta4.29" # https://github.com/JuliaLang/julia/pull/48526 + test_file = if VERSION >= v"1.11-" "references/complexexample.txt" + elseif VERSION >= v"1.9.0-beta4.29" # https://github.com/JuliaLang/julia/pull/48526 + "references/complexexample_pre_1_11.txt" elseif VERSION >= v"1.7.0" "references/complexexample_pre_1_9.txt" else From c3964c13ef63452d96be279b4e86b048a31b2da9 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 21 May 2024 15:24:00 -0500 Subject: [PATCH 03/13] fixup! Update test for Julia 1.11 --- test/references/complexexample.txt | 84 ++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 test/references/complexexample.txt diff --git a/test/references/complexexample.txt b/test/references/complexexample.txt new file mode 100644 index 0000000..138bd4d --- /dev/null +++ b/test/references/complexexample.txt @@ -0,0 +1,84 @@ + + + + + + Test Failed + Expression: sqrt(20) == 5 + Evaluated: 4.47213595499958 == 5 + + + + + + + + Test Failed + Expression: 1 * 2 == 5 + Evaluated: 2 == 5 + + + + + + + + Test Failed + Expression: 1 + 2 == 5 + Evaluated: 3 == 5 + + + + + + + + Test Failed + Expression: 1 + -2 == 1 + Evaluated: -1 == 1 + + + + + + + + + + + + + Nooo + + + + + + + + + Outside of tests + + + + + + Test Failed + Expression: throw(ArgumentError("1")) + Expected: DimensionMismatch + Thrown: ArgumentError + ArgumentError: 1 + + + + Test Failed + Expression: true + Expected: DimensionMismatch + No exception thrown + + + + + + + From faff47fdc54c87243a39ab205ab8b9b3e9313be9 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 08:50:03 -0500 Subject: [PATCH 04/13] Nightly failures are no longer silent --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8b3e94..5ba604c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,13 +30,9 @@ jobs: arch: ${{ matrix.arch }} - uses: julia-actions/cache@v1 - uses: julia-actions/julia-buildpkg@v1 - continue-on-error: ${{ matrix.version == 'nightly' }} - uses: julia-actions/julia-runtest@v1 - continue-on-error: ${{ matrix.version == 'nightly' }} - uses: julia-actions/julia-processcoverage@v1 - continue-on-error: ${{ matrix.version == 'nightly' }} - uses: codecov/codecov-action@v4 - continue-on-error: ${{ matrix.version == 'nightly' }} with: files: lcov.info docs: From 794d812fa85c8faaeb338e7f01521c68fc5820e7 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 08:51:25 -0500 Subject: [PATCH 05/13] Use GHA supporting Node 20 --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ba604c..a882d2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,11 +24,11 @@ jobs: - x64 steps: - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@v1 + - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: julia-actions/cache@v1 + - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 @@ -40,10 +40,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@v1 + - uses: julia-actions/setup-julia@v2 with: version: '1' - - uses: julia-actions/cache@v1 + - uses: julia-actions/cache@v2 - name: Install dependencies shell: julia --project=docs {0} run: | From 97c2c060ddf7b6658772bf68c751dc12730dc226 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 08:54:12 -0500 Subject: [PATCH 06/13] fixup! Update sandbox for Julia 1.11 --- src/runner.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runner.jl b/src/runner.jl index 267024b..2b827bb 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -394,9 +394,9 @@ function test!(pkg::AbstractString, nothing end - sandbox_args = if VERSION >= v"1.11.0-" + sandbox_args = if VERSION >= v"1.11-" (ctx, pkgspec, joinpath(pkgspec.path, "test"), test_project_override) - elseif VERSION >= v"1.4.0" + elseif VERSION >= v"1.4" (ctx, pkgspec, pkgspec.path, joinpath(pkgspec.path, "test"), test_project_override) else (ctx, pkgspec, pkgspec.path, joinpath(pkgspec.path, "test")) From cb7a15d515ac529a59a0ecc6609c5af2942b4846 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 08:56:29 -0500 Subject: [PATCH 07/13] drop! Ensure we didn't break anything --- .github/workflows/ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a882d2d..ecace67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,12 +14,19 @@ jobs: matrix: version: - '1.0' + - '1.2' + - '1.3' + - '1.4' + - '1.5' - '1.6' - - '1' + - '1.7' + - '1.8' + - '1.9' + - '1.10' - 'nightly' os: - ubuntu-latest - - windows-latest + # - windows-latest arch: - x64 steps: From c67cac2eca0b1b703987d0138d48ad36c86da338 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 09:52:32 -0500 Subject: [PATCH 08/13] Only was introduced in Julia 1.4 --- bin/reporttests.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/reporttests.jl b/bin/reporttests.jl index c39f05c..590057f 100755 --- a/bin/reporttests.jl +++ b/bin/reporttests.jl @@ -82,7 +82,9 @@ if abspath(PROGRAM_FILE) == @__FILE__() try run(cmd) catch e - e isa ProcessFailedException && exit(only(e.procs).exitcode) + if e isa ProcessFailedException && length(e.procs) == 1 + exit(e.procs[1].exitcode) + end rethrow() end end From bfe6dcff99da897a30f7f3a247b5c04e0c46dcf5 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 10:56:18 -0500 Subject: [PATCH 09/13] Fix tests on Julia 1.7/1.8 --- test/references/reporttests_fail_pre_1_9.txt | 10 ++++++++ test/reporttests_script.jl | 24 ++++++++++++++------ 2 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 test/references/reporttests_fail_pre_1_9.txt diff --git a/test/references/reporttests_fail_pre_1_9.txt b/test/references/reporttests_fail_pre_1_9.txt new file mode 100644 index 0000000..9657865 --- /dev/null +++ b/test/references/reporttests_fail_pre_1_9.txt @@ -0,0 +1,10 @@ + + + + + Test Failed + Expression: ARGS == [] + Evaluated: ["foo", "-e", "bar"] == Any[] + + + diff --git a/test/reporttests_script.jl b/test/reporttests_script.jl index 485ceef..cec7b47 100644 --- a/test/reporttests_script.jl +++ b/test/reporttests_script.jl @@ -5,7 +5,20 @@ else end script_runner_cmd = Sys.iswindows() ? `julia $script_runner --` : `$script_runner` test_script = "reporttests_testsets.jl" -reference_suffix = VERSION >= v"1.7" ? "" : "_pre_1_7" + +reference_file_pass = if VERSION >= v"1.7" + "references/reporttests_pass.xml" +else + "references/reporttests_pass_pre_1_7.xml" +end + +reference_file_fail = if VERSION >= v"1.9" + "references/reporttests_fail.xml" +elseif VERSION >= v"1.7" + "references/reporttests_fail_pre_1_9.xml" +else + "references/reporttests_fail_pre_1_7.xml" +end @testset "parse_args" begin include(script_runner) @@ -61,39 +74,36 @@ end end @testset "default output file" begin - reference_file = "references/reporttests_pass$reference_suffix.xml" output_file = "testlog.xml" p = run(ignorestatus(`$script_runner_cmd $test_script`)) try @test success(p) @test isfile(output_file) - @test_reference reference_file read(output_file, String) |> clean_output + @test_reference reference_file_pass read(output_file, String) |> clean_output finally isfile(output_file) && rm(output_file) end end @testset "specify output file" begin - reference_file = "references/reporttests_pass$reference_suffix.xml" output_file = "junit-report.xml" p = run(ignorestatus(`$script_runner_cmd $test_script --output=$output_file`)) try @test success(p) @test isfile(output_file) - @test_reference reference_file read(output_file, String) |> clean_output + @test_reference reference_file_pass read(output_file, String) |> clean_output finally isfile(output_file) && rm(output_file) end end @testset "test args" begin - reference_file = "references/reporttests_fail$reference_suffix.xml" output_file = "testlog.xml" p = run(ignorestatus(`$script_runner_cmd $test_script -- foo -e bar`)) try @test !success(p) @test isfile(output_file) - @test_reference reference_file read(output_file, String) |> clean_output + @test_reference reference_file_fail read(output_file, String) |> clean_output finally isfile(output_file) && rm(output_file) end From e2d1d57f2049b7f9df4cdc19fe0f014627b5ef1a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 10:58:11 -0500 Subject: [PATCH 10/13] fixup! Only was introduced in Julia 1.4 --- bin/reporttests.jl | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bin/reporttests.jl b/bin/reporttests.jl index 590057f..649e7c6 100755 --- a/bin/reporttests.jl +++ b/bin/reporttests.jl @@ -79,12 +79,6 @@ if abspath(PROGRAM_FILE) == @__FILE__() ) cmd = TestReports.gen_command(runner_code, ``, coverage) - try - run(cmd) - catch e - if e isa ProcessFailedException && length(e.procs) == 1 - exit(e.procs[1].exitcode) - end - rethrow() - end + p = run(ignorestatus(cmd)) + success(p) || exit(p.exitcode) end From 01a52e9e25b6e7b06d1e824971bc0987744ac07b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 11:01:39 -0500 Subject: [PATCH 11/13] fixup! Fix tests on Julia 1.7/1.8 --- ...{reporttests_fail_pre_1_9.txt => reporttests_fail_pre_1_9.xml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/references/{reporttests_fail_pre_1_9.txt => reporttests_fail_pre_1_9.xml} (100%) diff --git a/test/references/reporttests_fail_pre_1_9.txt b/test/references/reporttests_fail_pre_1_9.xml similarity index 100% rename from test/references/reporttests_fail_pre_1_9.txt rename to test/references/reporttests_fail_pre_1_9.xml From 1c4f6dc18bf247174e302d72339a1f06f90a9ac4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 11:11:01 -0500 Subject: [PATCH 12/13] Revert "drop! Ensure we didn't break anything" This reverts commit cb7a15d515ac529a59a0ecc6609c5af2942b4846. --- .github/workflows/ci.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecace67..a882d2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,19 +14,12 @@ jobs: matrix: version: - '1.0' - - '1.2' - - '1.3' - - '1.4' - - '1.5' - '1.6' - - '1.7' - - '1.8' - - '1.9' - - '1.10' + - '1' - 'nightly' os: - ubuntu-latest - # - windows-latest + - windows-latest arch: - x64 steps: From c82d14150f69178ff7b2e98103de566a15b8719f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 May 2024 12:11:30 -0500 Subject: [PATCH 13/13] Revert reporttests.jl changes --- bin/reporttests.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/reporttests.jl b/bin/reporttests.jl index 649e7c6..c39f05c 100755 --- a/bin/reporttests.jl +++ b/bin/reporttests.jl @@ -79,6 +79,10 @@ if abspath(PROGRAM_FILE) == @__FILE__() ) cmd = TestReports.gen_command(runner_code, ``, coverage) - p = run(ignorestatus(cmd)) - success(p) || exit(p.exitcode) + try + run(cmd) + catch e + e isa ProcessFailedException && exit(only(e.procs).exitcode) + rethrow() + end end