Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Flattened testsets ordered by appearance #103

Merged
merged 3 commits into from
Apr 30, 2024
Merged

Flattened testsets ordered by appearance #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9268e7c
Flattened testsets ordered by appearance
omus Apr 29, 2024
bdde70d
Merge branch 'master' into cv/flattened-testset-order
omus Apr 30, 2024
f5bbbec
Update reference tests
omus Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/testsets.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ function _flatten_results!(ts::AbstractTestSet)::Vector{<:AbstractTestSet}
if !isempty(results) || has_new_properties
# Use same ts to preserve description
ts.results = results
push!(flattened_results, ts)
pushfirst!(flattened_results, ts)
end
return flattened_results
end
Expand Down
19 changes: 13 additions & 6 deletions test/recordproperty.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,22 @@
properties_nodes = lastnode.(elements(root(rep)))

# Child properties are first in report
@test elements(properties_nodes[1])[1]["name"] == "Prop"
@test elements(properties_nodes[1])[1]["value"] == "Inner 1"
@test elements(properties_nodes[1])[2]["name"] == "ID"
@test elements(properties_nodes[1])[2]["value"] == "TopLevel"
@test length(elements(properties_nodes[1])) == 1
@test elements(properties_nodes[1])[1]["name"] == "ID"
@test elements(properties_nodes[1])[1]["value"] == "TopLevel"

@test length(elements(properties_nodes[2])) == 2
@test elements(properties_nodes[2])[1]["name"] == "Prop"
@test elements(properties_nodes[2])[1]["value"] == "Inner 2"
@test elements(properties_nodes[2])[1]["value"] == "Inner 1"
@test elements(properties_nodes[2])[2]["name"] == "ID"
@test elements(properties_nodes[2])[2]["value"] == "TopLevel"

@test length(elements(properties_nodes[3])) == 2
@test elements(properties_nodes[3])[1]["name"] == "Prop"
@test elements(properties_nodes[3])[1]["value"] == "Inner 2"
@test elements(properties_nodes[3])[2]["name"] == "ID"
@test elements(properties_nodes[3])[2]["value"] == "TopLevel"

# Test full packaage
pkg = "TestsWithProperties"
temp_pkg_dir() do tmp
Expand Down Expand Up @@ -88,7 +95,7 @@
# Force flattening as ts doesn't finish fully as it is not the top level testset
overwrite_text = "Property ID in testest Outer overwritten by child testset Inner"
@test_logs (:warn, overwrite_text) TestReports.flatten_results!(ts)
@test ts.results[1].properties["ID"] == "0"
@test ts.results[2].properties["ID"] == "0"

# Test for parent testset properties not being applied to child due to different type
ts = @testset ReportingTestSet "" begin
Expand Down
24 changes: 12 additions & 12 deletions test/references/complexexample.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="19" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="1 * 3 == 3" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
<testsuites tests="19" failures="6" errors="4"><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="4 % 2 == 0" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
Expression: sqrt(20) == 5
Evaluated: 4.47213595499958 == 5
</failure></testcase><testcase name="16 == 16" id="3" classname="Math" time="0.0"/></testsuite><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="1 * 3 == 3" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
Expression: 1 * 2 == 5
Evaluated: 2 == 5
</failure></testcase><testcase name="1 * 4 == 4" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="1 + -1 == 0" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
Expression: 1 + -2 == 1
Evaluated: -1 == 1
</failure></testcase><testcase name="10 + -5 == 5" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="1 + 1 == 2" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
</failure></testcase><testcase name="1 * 4 == 4" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="1 + 1 == 2" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
Expression: 1 + 2 == 5
Evaluated: 3 == 5
</failure></testcase><testcase name="1 + 4 == 5" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><testcase name="1 / 0" id="2" classname="Math/other" time="0.0"><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"></error></testcase><testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0"><error message="Nooo" type="ErrorException">Nooo
</error></testcase><testcase name="true" id="4" classname="Math/other" time="0.0"><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"></error></testcase></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0"><error message="Got exception outside of a @test" type="ErrorException">Outside of tests
</error></testcase></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
</failure></testcase><testcase name="1 + 4 == 5" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="1 + -1 == 0" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
Expression: 1 + -2 == 1
Evaluated: -1 == 1
</failure></testcase><testcase name="10 + -5 == 5" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><testcase name="1 / 0" id="2" classname="Math/other" time="0.0"><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"></error></testcase><testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0"><error message="Nooo" type="ErrorException">Nooo
</error></testcase><testcase name="true" id="4" classname="Math/other" time="0.0"><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"></error></testcase></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0"><error message="Got exception outside of a @test" type="ErrorException">Outside of tests
</error></testcase></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
Expression: throw(ArgumentError("1"))
Expected: DimensionMismatch
Thrown: ArgumentError
</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
Expression: true
Expected: DimensionMismatch
No exception thrown
</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="nthreads() &gt; 0" id="1" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="4 % 2 == 0" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
Expression: sqrt(20) == 5
Evaluated: 4.47213595499958 == 5
</failure></testcase><testcase name="16 == 16" id="3" classname="Math" time="0.0"/></testsuite></testsuites>
</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="nthreads() &gt; 0" id="1" classname="Math/using function from a module" time="0.0"/></testsuite></testsuites>

20 changes: 10 additions & 10 deletions test/references/complexexample_pre_1_7.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="19" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
<testsuites tests="19" failures="6" errors="4"><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
Expression: sqrt(20) == 5
Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math" time="0.0"/></testsuite><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
Expression: 1 * 2 == 5
Evaluated: 2 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
Expression: 1 + -2 == 1
Evaluated: -1 == 1</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
Evaluated: 2 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
Expression: 1 + 2 == 5
Evaluated: 3 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><testcase name="1 / 0" id="2" classname="Math/other" time="0.0"><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"></error></testcase><testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0"><error message="Nooo" type="ErrorException">Nooo
</error></testcase><testcase name="true" id="4" classname="Math/other" time="0.0"><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"></error></testcase></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0"><error message="Got exception outside of a @test" type="ErrorException">Outside of tests
</error></testcase></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
Evaluated: 3 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
Expression: 1 + -2 == 1
Evaluated: -1 == 1</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><testcase name="1 / 0" id="2" classname="Math/other" time="0.0"><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean"></error></testcase><testcase name="1 == error(&quot;Nooo&quot;)" id="3" classname="Math/other" time="0.0"><error message="Nooo" type="ErrorException">Nooo
</error></testcase><testcase name="true" id="4" classname="Math/other" time="0.0"><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass"></error></testcase></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="()" id="_testcase_id_" classname="Math/Error outside of tests" time="0.0"><error message="Got exception outside of a @test" type="ErrorException">Outside of tests
</error></testcase></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
Expression: throw(ArgumentError("1"))
Expected: DimensionMismatch
Thrown: ArgumentError</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
Expression: true
Expected: DimensionMismatch
No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
Expression: sqrt(20) == 5
Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math" time="0.0"/></testsuite></testsuites>
No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/using function from a module" time="0.0"/></testsuite></testsuites>

Loading
Loading