diff --git a/src/runner.jl b/src/runner.jl index 1edfb77..b8c4ced 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -129,6 +129,7 @@ function gen_runner_code(testfilename, logfilename, test_args) using Test using TestReports + using TestReports.EzXML: prettyprint append!(empty!(ARGS), $(repr(test_args.exec))) @@ -137,7 +138,9 @@ function gen_runner_code(testfilename, logfilename, test_args) end # Flatten before calling `report` to avoid a `deepcopy`. - write($(repr(logfilename)), report(TestReports.flatten_results!(ts))) + open($(repr(logfilename)), "w") do io + prettyprint(io, report(TestReports.flatten_results!(ts))) + end any_problems(ts) && exit(TestReports.TESTS_FAILED) """ return runner_code diff --git a/test/references/complexexample.txt b/test/references/complexexample.txt index 7221463..6aa78cd 100644 --- a/test/references/complexexample.txt +++ b/test/references/complexexample.txt @@ -1,25 +1,83 @@ -Test Failed + + + + + Test Failed Expression: sqrt(20) == 5 Evaluated: 4.47213595499958 == 5 -Test Failed + + + + + + + + Test Failed Expression: 1 * 2 == 5 Evaluated: 2 == 5 -Test Failed + + + + + + + + Test Failed Expression: 1 + 2 == 5 Evaluated: 3 == 5 -Test Failed + + + + + + + + Test Failed Expression: 1 + -2 == 1 Evaluated: -1 == 1 -Nooo -Outside of tests -Test Failed + + + + + + + + + + + + + Nooo + + + + + + + + + Outside of tests + + + + + + Test Failed Expression: throw(ArgumentError("1")) Expected: DimensionMismatch Thrown: ArgumentError -Test Failed + + + + Test Failed Expression: true Expected: DimensionMismatch No exception thrown - - + + + + + + + diff --git a/test/references/complexexample_pre_1_7.txt b/test/references/complexexample_pre_1_7.txt index c2dfd50..07f3521 100644 --- a/test/references/complexexample_pre_1_7.txt +++ b/test/references/complexexample_pre_1_7.txt @@ -1,19 +1,77 @@ -Test Failed + + + + + Test Failed Expression: sqrt(20) == 5 - Evaluated: 4.47213595499958 == 5Test Failed + Evaluated: 4.47213595499958 == 5 + + + + + + + Test Failed Expression: 1 * 2 == 5 - Evaluated: 2 == 5Test Failed + Evaluated: 2 == 5 + + + + + + + Test Failed Expression: 1 + 2 == 5 - Evaluated: 3 == 5Test Failed + Evaluated: 3 == 5 + + + + + + + Test Failed Expression: 1 + -2 == 1 - Evaluated: -1 == 1Nooo -Outside of tests -Test Failed + Evaluated: -1 == 1 + + + + + + + + + + + + Nooo + + + + + + + + + Outside of tests + + + + + + Test Failed Expression: throw(ArgumentError("1")) Expected: DimensionMismatch - Thrown: ArgumentErrorTest Failed + Thrown: ArgumentError + + + Test Failed Expression: true Expected: DimensionMismatch - No exception thrown - + No exception thrown + + + + + + diff --git a/test/references/complexexample_pre_1_9.txt b/test/references/complexexample_pre_1_9.txt index 2efba2e..fea4fd3 100644 --- a/test/references/complexexample_pre_1_9.txt +++ b/test/references/complexexample_pre_1_9.txt @@ -1,19 +1,77 @@ -Test Failed + + + + + Test Failed Expression: sqrt(20) == 5 - Evaluated: 4.47213595499958 == 5Test Failed + Evaluated: 4.47213595499958 == 5 + + + + + + + Test Failed Expression: 1 * 2 == 5 - Evaluated: 2 == 5Test Failed + Evaluated: 2 == 5 + + + + + + + Test Failed Expression: 1 + 2 == 5 - Evaluated: 3 == 5Test Failed + Evaluated: 3 == 5 + + + + + + + Test Failed Expression: 1 + -2 == 1 - Evaluated: -1 == 1Nooo -Outside of tests -Test Failed + Evaluated: -1 == 1 + + + + + + + + + + + + Nooo + + + + + + + + + Outside of tests + + + + + + Test Failed Expression: throw(ArgumentError("1")) Expected: DimensionMismatch - Thrown: ArgumentErrorTest Failed + Thrown: ArgumentError + + + Test Failed Expression: true Expected: DimensionMismatch - No exception thrown - + No exception thrown + + + + + + diff --git a/test/references/reporttests_fail.xml b/test/references/reporttests_fail.xml index c6c5849..e91261f 100644 --- a/test/references/reporttests_fail.xml +++ b/test/references/reporttests_fail.xml @@ -1,5 +1,11 @@ -Test Failed + + + + Test Failed Expression: ARGS == [] Evaluated: ["foo", "-e", "bar"] == Any[] - + + + + diff --git a/test/references/reporttests_fail_pre_1_7.xml b/test/references/reporttests_fail_pre_1_7.xml index 31aa7ab..9657865 100644 --- a/test/references/reporttests_fail_pre_1_7.xml +++ b/test/references/reporttests_fail_pre_1_7.xml @@ -1,4 +1,10 @@ -Test Failed + + + + Test Failed Expression: ARGS == [] - Evaluated: ["foo", "-e", "bar"] == Any[] + Evaluated: ["foo", "-e", "bar"] == Any[] + + + diff --git a/test/references/reporttests_pass.xml b/test/references/reporttests_pass.xml index e4adca7..e6182bc 100644 --- a/test/references/reporttests_pass.xml +++ b/test/references/reporttests_pass.xml @@ -1,2 +1,6 @@ - + + + + + diff --git a/test/references/reporttests_pass_pre_1_7.xml b/test/references/reporttests_pass_pre_1_7.xml index 0dd9f9e..255096f 100644 --- a/test/references/reporttests_pass_pre_1_7.xml +++ b/test/references/reporttests_pass_pre_1_7.xml @@ -1,2 +1,6 @@ - + + + + + diff --git a/test/references/singlenest.txt b/test/references/singlenest.txt index 3ecaf3b..4e9213c 100644 --- a/test/references/singlenest.txt +++ b/test/references/singlenest.txt @@ -1,2 +1,6 @@ - + + + + + diff --git a/test/references/singlenest_pre_1_7.txt b/test/references/singlenest_pre_1_7.txt index 5cb98a2..5856dcb 100644 --- a/test/references/singlenest_pre_1_7.txt +++ b/test/references/singlenest_pre_1_7.txt @@ -1,2 +1,6 @@ - + + + + + diff --git a/test/references/test_with_properties.txt b/test/references/test_with_properties.txt index 81ea393..1d0d25a 100644 --- a/test/references/test_with_properties.txt +++ b/test/references/test_with_properties.txt @@ -1,2 +1,50 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/references/test_with_properties_pre_1_7.txt b/test/references/test_with_properties_pre_1_7.txt index 77281e5..37177f8 100644 --- a/test/references/test_with_properties_pre_1_7.txt +++ b/test/references/test_with_properties_pre_1_7.txt @@ -1,2 +1,50 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/reportgeneration.jl b/test/reportgeneration.jl index d0c80ed..9187ee1 100644 --- a/test/reportgeneration.jl +++ b/test/reportgeneration.jl @@ -8,7 +8,7 @@ const TEST_PKG = (name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-2369062 @testset "SingleNest" begin test_file = VERSION >= v"1.7.0" ? "references/singlenest.txt" : "references/singlenest_pre_1_7.txt" - @test_reference test_file read(`$(Base.julia_cmd()) -e "using Test; using TestReports; (@testset ReportingTestSet \"\" begin @testset \"a\" begin @test 1 == 1 end end) |> report |> print"`, String) |> clean_output + @test_reference test_file read(`$(Base.julia_cmd()) -e "using Test; using TestReports; (@testset ReportingTestSet \"\" begin @testset \"a\" begin @test 1 == 1 end end) |> report |> print"`, String) |> clean_output |> pretty_format_xml end @testset "Complex Example" begin @@ -19,7 +19,7 @@ end else "references/complexexample_pre_1_7.txt" end - @test_reference test_file read(`$(Base.julia_cmd()) $(@__DIR__)/example.jl`, String) |> clean_output + @test_reference test_file read(`$(Base.julia_cmd()) $(@__DIR__)/example.jl`, String) |> clean_output |> pretty_format_xml end @testset "TestsWithProperties" begin @@ -30,7 +30,7 @@ end end logfile = joinpath(@__DIR__, "testlog.xml") test_file = VERSION >= v"1.7.0" ? "references/test_with_properties.txt" : "references/test_with_properties_pre_1_7.txt" - @test_reference test_file read(logfile, String) |> clean_output + @test_reference test_file read(logfile, String) |> clean_output |> pretty_format_xml end @testset "Runner tests" begin diff --git a/test/utils.jl b/test/utils.jl index c0507e6..14ea7a9 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -23,6 +23,8 @@ remove_timestamp_info(str) = replace(str, r"\stimestamp=\\\"[0-9-T:.]*\\\"" => " # Default hostname output - we want "hostname" there to check its being recorded default_hostname_info(str) = replace(str, r"\shostname=\\\"[\S]*\\\"" => " hostname=\"localhost\"") +pretty_format_xml(str) = sprint(prettyprint, parsexml(str)) + const clean_output = strip_filepaths ∘ remove_stacktraces ∘ remove_test_output ∘ remove_timing_info ∘ remove_timestamp_info ∘ default_hostname_info test_package_path(pkg) = joinpath(@__DIR__, "test_packages", pkg)