From 825005047cdf749637adf1acea263a43bdc086f6 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 21 May 2024 13:21:11 -0500 Subject: [PATCH] Support testcase properties (#106) * Support testset and test properties * Add JUnit specific details to *_properties docstrings * Set project version to 1.0.0 * Improve examples --- Project.toml | 2 +- README.md | 7 +- docs/src/index.md | 7 +- docs/src/library.md | 3 +- docs/src/manual.md | 34 +- src/TestReports.jl | 4 +- src/properties.jl | 87 +++++ src/recordproperty.jl | 46 --- src/testsets.jl | 100 ++++-- src/to_xml.jl | 24 +- test/properties.jl | 303 ++++++++++++++++++ test/recordproperty.jl | 144 --------- test/references/test_with_properties.txt | 2 +- .../test_with_properties_pre_1_7.txt | 2 +- test/runtests.jl | 2 +- .../TestsWithProperties/test/runtests.jl | 28 +- test/testsets.jl | 6 +- 17 files changed, 518 insertions(+), 283 deletions(-) create mode 100644 src/properties.jl delete mode 100644 src/recordproperty.jl create mode 100644 test/properties.jl delete mode 100644 test/recordproperty.jl diff --git a/Project.toml b/Project.toml index 12eabde..5ca98a6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "TestReports" uuid = "dcd651b4-b50a-5b6b-8f22-87e9f253a252" -version = "0.7.3" +version = "1.0.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/README.md b/README.md index c89a158..13181cd 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,15 @@ To add to CI: $ julia -e 'using Pkg; Pkg.add("TestReports"); using TestReports; TestReports.test("MyPackage")' ``` -Additionally, properties can be added to your `TestSet`s. To do this, use the `recordproperty` -function like so: +Additionally, properties can be associated with your testsets or tests. To do this, use the `record_testset_property` or `record_test_property` functions within a testset: ```julia using Test using TestReports @testset "MyTests" begin - recordproperty("ID", 1) - @test 1==1 + record_testset_property("ID", 1) + @test 1 == 1 end ``` diff --git a/docs/src/index.md b/docs/src/index.md index 9377ad9..5541ad6 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -25,16 +25,15 @@ To add to CI: $ julia -e 'using Pkg; Pkg.add("TestReports"); using TestReports; TestReports.test("MyPackage")' ``` -Additionally, properties can be added to your `TestSet`s. To do this, use the `recordproperty` -function like so: +Additionally, properties can be associated with your testsets or tests. To do this, use the `record_testset_property` or `record_test_property` functions within a testset: ```julia using Test using TestReports @testset "MyTests" begin - recordproperty("ID", 1) - @test 1==1 + record_testset_property("ID", 1) + @test 1 == 1 end ``` diff --git a/docs/src/library.md b/docs/src/library.md index 9b3ade1..be514e0 100644 --- a/docs/src/library.md +++ b/docs/src/library.md @@ -12,7 +12,8 @@ Documentation for `TestReports.jl`'s public interface. ```@docs TestReports.test -recordproperty +record_testset_property +record_test_property ReportingTestSet any_problems report diff --git a/docs/src/manual.md b/docs/src/manual.md index b4691b7..8d28607 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -77,30 +77,26 @@ julia> TestReports.test(["MyPackage1", "MyPackage2"], logfilename=["1.xml", "2.x # File paths are joinpath("path", "1.xml") and joinpath("path", "2.xml") ``` -## Adding Properties +## Associating Properties -Properties can be added to a `TestSet` using the exported function `recordproperty` -within `runtests.jl` and other `include`d scripts: +Properties can be associated to a testsets and/or tests by using the respective +`record_testset_property` and `record_test_property` functions within a testset: ```julia using Test using TestReports @testset "MyTests" begin - recordproperty("ID", 1) - @test 1==1 + record_testset_property("ID", 1) + @test 1 == 1 end ``` -`recordproperty` will have no affect during normal unit testing. - -The added properties will be added to the corresponding testsuite in the generated report. -Multiple properties can be added, and a property added to a parent `TestSet` will be applied -to all child `TestSet`s. +The `record_*_property` functions will have no affect during normal unit testing. -Properties with the same name are allowed to be set multiple times within the same -`TestSet`. If both a parent and a child set the same named property both properties will -appear in the child when generating the report. +The associated properties will be added to the corresponding testsuite or testcase in the +generated report. Multiple properties can be added, and a properties added to a parent +testset will be applied to all child testsets. Duplicate properties are allowed to be set. The property name must always be a `String`, but the value can be anything that is serializable by `EzXML.jl`. In practice this means that `String`s, `Number`s, `Expr`s and `Symbols` can be used, @@ -113,19 +109,21 @@ using Test using TestReports @testset "TopLevelTests" begin - recordproperty("TestFile" @__FILE__) # This will be added to all child testsets in report + # This will be added to all child testsets in report. May not be appropriate when using + # `include`. + record_testset_property("TestFile", @__FILE__) @testset "MiddleLevelTests" begin - recordproperty("Testsuite", 100) - recordproperty("TestSubject", "Example") + record_testset_property("Testsuite", 100) + record_test_property("TestSubject", "Example") @testset "Inner" begin - recordproperty("Testsuite", 101) # Both testsuite 100 and 101 apply to this testset + record_testset_property("Testsuite", 101) # Associate with both testsuite 100 and 101 @test 1 == 1 end @testset "Types" begin - recordproperty("Prop1", :Val1) + record_test_property("Prop1", :Val1) @test 1 == 1 end end diff --git a/src/TestReports.jl b/src/TestReports.jl index 1c727fe..ee69277 100644 --- a/src/TestReports.jl +++ b/src/TestReports.jl @@ -32,7 +32,7 @@ else using Pkg.Types: SHA1 end -export ReportingTestSet, any_problems, report, recordproperty +export ReportingTestSet, any_problems, report, record_test_property, record_testset_property const TESTREPORTS_VERSION = let # Copied from Documenter.jl project = joinpath(dirname(dirname(pathof(TestReports))), "Project.toml") @@ -50,6 +50,6 @@ include("./testsets.jl") include("to_xml.jl") include("compat_check.jl") include("runner.jl") -include("recordproperty.jl") +include("properties.jl") end # module diff --git a/src/properties.jl b/src/properties.jl new file mode 100644 index 0000000..f7c0a97 --- /dev/null +++ b/src/properties.jl @@ -0,0 +1,87 @@ +""" + record_testset_property(name::AbstractString, value) + +Associates a property with the current testset. The `name` and `value` will be turned into a +`` element within the corresponding `` element within the JUnit XML +report. + +Multiple properties can be added to one testset and child testsets inherit the properties +defined by their parents. If a child testset records a property which is already set both +will be present in the resulting report. + +The suggested use of this function is to place it inside a testset with unspecified type +(see Examples). This will ensure that `Pkg.test` is unnaffected, but that the properties +are added to the report when `TestReports.test` is used. This is because properties are +only added when the `Testset` type has a `TestReports.testset_properties` method defined, as +does the `ReportingTestSet` used by `TestReports`. `TestReports.testset_properties` can be +extended for custom `TestSet`s. + +If a child testset has this method defined but its parent doesn't, the property should +be in the report when `TestReport.test` is used, assuming that the parent testset +type doesn't do anything to affect the reporting behaviour. However this is not tested +functionality. + +The `value` argument must be serializable by EzXML, which gives quite a lot of freedom. + +## Examples + +Using the default testset for compatiblity with `Pkg.test` and `TestReports.test`: + +```julia +using TestReports + +# Default testset used, record property calls are ignored by `Pkg.test` but will be used +# when generating JUnit XML. +@testset "MyTestSet" begin + record_testset_property("ID", 42) + record_test_property("Bool", true) + @test 1 == 1 + @test 2 == 2 +end +``` + +Rendering JUnit reports on the REPL: + +```julia +using Test, TestReports, EzXML + +ts = @testset ReportingTestSet "Root" begin # `` has property "foo" + record_testset_property("foo", 1) + record_test_property("bar", 2) + + @testset "Inner" begin # `` has property "foo" + @test 1 == 1 # `` has the property "bar" + end + + @test 2 != 1 # `` has the property "bar" +end; + +prettyprint(report(ts)) +``` + +See also: [`record_test_property`](@ref) and [`testset_properties`](@ref). +""" +function record_testset_property(name::AbstractString, value) + record_testset_property!(get_testset(), name, value) + return value +end + +""" + record_test_property(name::AbstractString, value) + +Associates a property with the tests contained with the testset. The `name` and `value` will +be turned into a `` element with the corresponding `` element within the +JUnit XML report. + +Multiple test properties can be assigned within a testset and child testsets will inherit +the test properties defined by their parents. If a child testset records a test property +with an already used name both properties will be present in the resulting report. + +For more details and examples see the documentation for [`record_testset_property`](@ref). + +See also: [`record_testset_property`](@ref) and [`test_properties`](@ref). +""" +function record_test_property(name::AbstractString, value) + record_test_property!(get_testset(), name, value) + return value +end diff --git a/src/recordproperty.jl b/src/recordproperty.jl deleted file mode 100644 index 47edb5c..0000000 --- a/src/recordproperty.jl +++ /dev/null @@ -1,46 +0,0 @@ -""" - recordproperty(name::AbstractString, value) - -Associates a property with a testset. The `name` and `value` will be turned into a -`` element within the corresponding `` element within the JUnit XML -report. - -Multiple properties can be added to one testset and child testsets inherit the properties -defined by their parents. If a child testset records a property which is already set both -will be present in the resulting report. - -The suggested use of this function is to place it inside a testset with unspecified type -(see Examples). This will ensure that `Pkg.test` is unnaffected, but that the properties -are added to the report when `TestReports.test` is used. This is because properties are -only added when the `Testset` type has a `TestReports.properties` method defined, as does -the `ReportingTestSet` used by `TestReports`. `TestReports.properties` can be extended for -custom `TestSet`s. - -If a child testset has this method defined but its parent doesn't, the property should -be in the report when `TestReport.test` is used, assuming that the parent testset -type doesn't do anything to affect the reporting behavior. However this is not tested -functionality. - -The `value` argument must be serializable by EzXML, which gives quite a lot of freedom. - -# Examples -```julia -using TestReports - -# Default testset used, so function will not affect Pkg.test but will be used when -# generating JUnit XML. -@testset "MyTestSet" begin - recordproperty("ID", 42) - recordproperty("File", @__FILE__) - recordproperty("Bool", true) - @test 1 == 1 - @test 2 == 2 -end -``` - -See also: [`properties`](@ref) and [`recordproperty!](@ref). -""" -function recordproperty(name::AbstractString, value) - recordproperty!(get_testset(), name, value) - return value -end diff --git a/src/testsets.jl b/src/testsets.jl index 53920d5..6675c38 100644 --- a/src/testsets.jl +++ b/src/testsets.jl @@ -56,18 +56,21 @@ and this is not recommended or supported. A `ReportingTestSet` has the `description` and `results` fields as per a `DefaultTestSet`, and has the following additional fields: -- `properties`: used to record testsuite properties to be inserted into the report. +- `testset_properties`: used to record testset properties to be inserted into the report. +- `test_properties`: used to record test properties to be inserted into the report. - `start_time::DateTime`: the start date and time of the testing (local system time). - `time_taken::Millisecond`: the time taken in milliseconds to run the `TestSet`. - `last_record_time::DateTime`: the time when `record` was last called. - `hostname::String`: the name of host on which the testset was executed. -See also: [`flatten_results!`](@ref), [`recordproperty`](@ref), [`report`](@ref) +See also: [`flatten_results!`](@ref), [`record_testset_property`](@ref), +[`record_testset_property`](@ref), and [`report`](@ref). """ mutable struct ReportingTestSet <: AbstractTestSet description::String results::Vector - properties::Vector{Pair{String, Any}} + testset_properties::Vector{Pair{String, Any}} + test_properties::Vector{Pair{String, Any}} start_time::DateTime time_taken::Millisecond last_record_time::DateTime @@ -76,7 +79,7 @@ mutable struct ReportingTestSet <: AbstractTestSet showtiming::Bool end -ReportingTestSet(desc; verbose=false, showtiming=true) = ReportingTestSet(desc, [], [], now(), Millisecond(0), now(), gethostname(), verbose, showtiming) +ReportingTestSet(desc; verbose=false, showtiming=true) = ReportingTestSet(desc, [], [], [], now(), Millisecond(0), now(), gethostname(), verbose, showtiming) function record(ts::ReportingTestSet, t::Result) push!(ts.results, ReportingResult(t, now()-ts.last_record_time)) @@ -113,27 +116,54 @@ end ################################# """ - properties(ts::AbstractTestSet) -> Union{Vector{Pair{String, Any}}, Nothing} + testset_properties(ts::AbstractTestSet) -> Union{Vector{Pair{String, Any}}, Nothing} -Get the properties of a `ReportingTestSet`. Can be extended for custom testsets. -Defaults to `nothing` for an `AbstractTestSet`. +Get the properties associated with a testset. Can be extended for custom testsets. Defaults +to `nothing` for testsets which do not support testset properties. -See also: [`recordproperty`](@ref) and [`recordproperty!`](@ref). +When generating a JUnit XML report these will be the properties associated with a +`testsuite` element. + +See also: [`test_properties`](@ref). """ -properties(::AbstractTestSet) = nothing -properties(ts::ReportingTestSet) = ts.properties +testset_properties(::AbstractTestSet) = nothing +testset_properties(ts::ReportingTestSet) = ts.testset_properties """ - recordproperty!(ts::AbstractTestSet, name::AbstractString, value) + record_testset_property!(ts::AbstractTestSet, name, value) Associate a property with the testset. Can be extended for custom testsets. +""" +record_testset_property!(ts::AbstractTestSet, name::AbstractString, value) = ts + +function record_testset_property!(ts::ReportingTestSet, name::AbstractString, value) + push!(ts.testset_properties, name => value) + return ts +end + +""" + test_properties(ts::AbstractTestSet) -> Union{Vector{Pair{String, Any}}, Nothing} + +Get the properties associated with tests within a testset. Can be extended for custom +testsets. Will return `nothing` for testsets which do not support test properties. + +When generating a JUnit XML report these will be the properties associated with all +`testcase` elements contained within a `testsuite` element. + +See also: [`testset_properties`](@ref). +""" +test_properties(::AbstractTestSet) = nothing +test_properties(ts::ReportingTestSet) = ts.test_properties -See also: [`properties`](@ref). """ -recordproperty!(ts::AbstractTestSet, name::AbstractString, value) = ts + record_test_property!(ts::AbstractTestSet, name, value) -function recordproperty!(ts::ReportingTestSet, name::AbstractString, value) - push!(ts.properties, name => value) +Associate a property with the tests within the testset. Can be extended for custom testsets. +""" +record_test_property!(ts::AbstractTestSet, name::AbstractString, value) = ts + +function record_test_property!(ts::ReportingTestSet, name::AbstractString, value) + push!(ts.test_properties, name => value) return ts end @@ -242,7 +272,7 @@ Recursively flatten `ts` to a vector of `TestSet`s. """ function _flatten_results!(ts::AbstractTestSet, depth::Int)::Vector{AbstractTestSet} original_results = ts.results - has_new_properties = !isempty(something(properties(ts), tuple())) + has_new_properties = !isempty(something(testset_properties(ts), tuple())) flattened_results = AbstractTestSet[] # Track results that are a Result so that if there are any, they can be added # in their own testset to flattened_results @@ -255,7 +285,7 @@ function _flatten_results!(ts::AbstractTestSet, depth::Int)::Vector{AbstractTest end function inner!(childts::AbstractTestSet) # Make it a sibling - update_testset_properties!(childts, ts) + update_properties!(childts, ts) if depth > 0 || has_description(ts) childts.description = ts.description * "/" * childts.description end @@ -288,7 +318,7 @@ Return vector containing `rs` so that when iterated through, _flatten_results!(rs::Result, depth::Int) = [rs] """ - update_testset_properties!(childts::AbstractTestSet, ts::AbstractTestSet) + update_properties!(childts::AbstractTestSet, ts::AbstractTestSet) Adds properties of `ts` to `childts`. @@ -300,22 +330,24 @@ handled as follows: - If method not defined for `typeof(childts)` and `ts` has properties, then a warning is shown. -See also: [`properties`](@ref). -""" -function update_testset_properties!(childts::AbstractTestSet, ts::AbstractTestSet) - child_props = properties(childts) - parent_props = properties(ts) - - # Children inherit the properties of their parents - if !isnothing(parent_props) && !isempty(parent_props) - if !isnothing(child_props) - prepend!(child_props, parent_props) - else - @warn( - "Properties of testset \"$(ts.description)\" can not be added to " * - "child testset \"$(childts.description)\" as it does not have a " * - "`TestReports.properties` method defined." - ) +See also: [`testset_properties`](@ref) and [`test_properties`](@ref). +""" +function update_properties!(childts::AbstractTestSet, ts::AbstractTestSet) + for properties in (testset_properties, test_properties) + child_props = properties(childts) + parent_props = properties(ts) + + # Children inherit the properties of their parents + if !isnothing(parent_props) && !isempty(parent_props) + if !isnothing(child_props) + prepend!(child_props, parent_props) + else + @warn( + "Properties of testset \"$(ts.description)\" can not be added to " * + "child testset \"$(childts.description)\" as it does not have a " * + "`TestReports.$(nameof(properties))` method defined." + ) + end end end return childts diff --git a/src/to_xml.jl b/src/to_xml.jl index 3510872..f163942 100644 --- a/src/to_xml.jl +++ b/src/to_xml.jl @@ -202,11 +202,12 @@ function to_xml(ts::AbstractTestSet) # Set attributes which require variables in this scope ntests > 0 && set_attribute!(x_testcase, "id", total_ntests) # Ignore both testsuites and errors outside of tests ispass(result) && VERSION < v"1.7.0" && set_attribute!(x_testcase, "name", x_testcase["name"] * " (Test $total_ntests)") + add_properties!(x_testcase, test_properties(ts)) x_testcase end x_testsuite = testsuite_xml(ts.description, total_ntests, total_nfails, total_nerrors, x_testcases, time_taken(ts)::Millisecond, start_time(ts)::DateTime, hostname(ts)) - add_testsuite_properties!(x_testsuite, ts) + add_properties!(x_testsuite, testset_properties(ts)) x_testsuite, total_ntests, total_nfails, total_nerrors end @@ -322,25 +323,22 @@ function get_failure_message(v::Fail) end """ - add_testsuite_properties!(x_testsuite, ts::AbstractTestSet) + add_properties!(x_element, properties) -Add all key value pairs in the `properties` field of a `AbstractTestSet` to the -corresponding testsuite xml element. This function assumes that the type of `ts` -has a `TestReports.properties` method defined. - -See also: [`properties`](@ref) +Add all key value pairs defined within the `properties` to the referenced XML element. """ -function add_testsuite_properties!(x_testsuite, ts::AbstractTestSet) - properties_dict = properties(ts) - if !isnothing(properties_dict) && !isempty(keys(properties_dict)) +function add_properties!(x_element, properties) + if !isempty(properties) x_properties = ElementNode("properties") - for (name, value) in properties_dict + for (name, value) in properties x_property = ElementNode("property") set_attribute!(x_property, "name", name) set_attribute!(x_property, "value", value) link!(x_properties, x_property) end - link!(x_testsuite, x_properties) + link!(x_element, x_properties) end - return x_testsuite + return x_element end + +add_properties!(x_element, properties::Nothing) = x_element diff --git a/test/properties.jl b/test/properties.jl new file mode 100644 index 0000000..113f901 --- /dev/null +++ b/test/properties.jl @@ -0,0 +1,303 @@ +@testset "record_testset_property / testset_properties" begin + using TestReports: testset_properties + + @testset "empty" begin + ts = @testset ReportingTestSet begin end + @test testset_properties(ts) isa AbstractVector + @test length(testset_properties(ts)) == 0 + + ts = @testset "_" begin end + @test testset_properties(ts) === nothing + end + + @testset "record property" begin + ts = @testset ReportingTestSet begin + record_testset_property("tested-item-id", "SAMD-45") + end + @test testset_properties(ts) == ["tested-item-id" => "SAMD-45"] + + ts = @testset ReportingTestSet begin + record_testset_property("count", 3) + end + @test testset_properties(ts) == ["count" => 3] + end + + @testset "multiple properties" begin + ts = @testset ReportingTestSet begin + record_testset_property("tests", "ABC-789") + record_testset_property("tests", "ABC-1011") + end + @test testset_properties(ts) == ["tests" => "ABC-789", "tests" => "ABC-1011"] + + ts = @testset ReportingTestSet begin + record_testset_property("tests", "ABC-789") + record_testset_property("tests", "ABC-789") + end + @test testset_properties(ts) == ["tests" => "ABC-789", "tests" => "ABC-789"] + end + + @testset "nested properties" begin + ts = @testset ReportingTestSet begin + record_testset_property("tests", "ABC-789") + @testset begin + record_testset_property("tests", "ABC-1011") + end + end + @test testset_properties(ts) == ["tests" => "ABC-789"] + @test testset_properties(ts.results[1]) == ["tests" => "ABC-1011"] + + flattened_testsets = TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 2 + @test testset_properties(flattened_testsets[1]) == ["tests" => "ABC-789"] + @test testset_properties(flattened_testsets[2]) == ["tests" => "ABC-789", "tests" => "ABC-1011"] + + ts = @testset ReportingTestSet begin + record_testset_property("tests", "ABC-789") + @testset begin + record_testset_property("tests", "ABC-789") + end + @testset begin + end + end + @test testset_properties(ts) == ["tests" => "ABC-789"] + @test testset_properties(ts.results[1]) == ["tests" => "ABC-789"] + @test testset_properties(ts.results[2]) == [] + + flattened_testsets = TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 2 + @test testset_properties(flattened_testsets[1]) == ["tests" => "ABC-789"] + @test testset_properties(flattened_testsets[2]) == ["tests" => "ABC-789", "tests" => "ABC-789"] + end + + @testset "custom testset support" begin + # Test for parent testset properties not being applied to child due to different type + ts = @testset ReportingTestSet "" begin + @testset ReportingTestSet "Outer" begin + record_testset_property("ID", "42") + @testset TestReportingTestSet "Inner" begin + @test 1 == 1 + end + end + end + fail_text = "Properties of testset \"Outer\" can not be added to child testset \"Inner\" as it does not have a `TestReports.testset_properties` method defined." + flattened_testsets = @test_logs (:warn, fail_text) TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 2 + @test testset_properties(flattened_testsets[1]) == ["ID" => "42"] + @test testset_properties(flattened_testsets[2]) === nothing + + # Test for ReportingTestSet setting a property inside of a parent custom testset + ts = @testset ReportingTestSet "TestReports Wrapper" begin + @testset TestReportingTestSet "Custom" begin + ts = @testset ReportingTestSet "Inner" begin + record_testset_property("ID", "42") + @test 1 == 1 + end + end + end + flattened_testsets = TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 1 + @test testset_properties(flattened_testsets[1]) == ["ID" => "42"] + end + + @testset "ignore properties on unsupported test sets" begin + ts = @testset begin + record_testset_property("id", 1) + end + @test testset_properties(ts) === nothing + end + + @testset "junit report" begin + # Check properties in XML doc variable + ts = @testset ReportingTestSet "TopLevel" begin + record_testset_property("ID", "TopLevel") + @testset begin + record_testset_property("Prop", "Inner 1") + end + @testset begin + record_testset_property("Prop", "Inner 2") + end + end + rep = report(ts) + testsuite_elements = findall("//testsuite", root(rep)) + @test length(testsuite_elements) == 3 + + # Parent properties are first in report + testsuite_property_elements = findall("properties/property", testsuite_elements[1]) + @test length(testsuite_property_elements) == 1 + @test testsuite_property_elements[1]["name"] == "ID" + @test testsuite_property_elements[1]["value"] == "TopLevel" + + testsuite_property_elements = findall("properties/property", testsuite_elements[2]) + @test length(testsuite_property_elements) == 2 + @test testsuite_property_elements[1]["name"] == "ID" + @test testsuite_property_elements[1]["value"] == "TopLevel" + @test testsuite_property_elements[2]["name"] == "Prop" + @test testsuite_property_elements[2]["value"] == "Inner 1" + + testsuite_property_elements = findall("properties/property", testsuite_elements[3]) + @test length(testsuite_property_elements) == 2 + @test testsuite_property_elements[1]["name"] == "ID" + @test testsuite_property_elements[1]["value"] == "TopLevel" + @test testsuite_property_elements[2]["name"] == "Prop" + @test testsuite_property_elements[2]["value"] == "Inner 2" + end +end + +@testset "record_test_property / test_properties" begin + using TestReports: test_properties + + @testset "empty" begin + ts = @testset ReportingTestSet begin end + @test test_properties(ts) isa AbstractVector + @test length(test_properties(ts)) == 0 + + ts = @testset "_" begin end + @test test_properties(ts) === nothing + end + + @testset "record property" begin + ts = @testset ReportingTestSet begin + record_test_property("tested-item-id", "SAMD-45") + end + @test test_properties(ts) == ["tested-item-id" => "SAMD-45"] + + ts = @testset ReportingTestSet begin + record_test_property("count", 3) + end + @test test_properties(ts) == ["count" => 3] + end + + @testset "flattening eliminates testsets without tests" begin + ts = @testset ReportingTestSet begin + record_test_property("tested-item-id", "SAMD-45") + end + flattened_testsets = TestReports.flatten_results!(ts) + @test isempty(flattened_testsets) + + ts = @testset ReportingTestSet begin + record_test_property("tested-item-id", "SAMD-45") + @test true + end + flattened_testsets = TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 1 + @test test_properties(flattened_testsets[1]) == ["tested-item-id" => "SAMD-45"] + end + + @testset "multiple properties" begin + ts = @testset ReportingTestSet begin + record_test_property("tests", "ABC-789") + record_test_property("tests", "ABC-1011") + end + @test test_properties(ts) == ["tests" => "ABC-789", "tests" => "ABC-1011"] + + ts = @testset ReportingTestSet begin + record_test_property("tests", "ABC-789") + record_test_property("tests", "ABC-789") + end + @test test_properties(ts) == ["tests" => "ABC-789", "tests" => "ABC-789"] + end + + @testset "nested properties" begin + ts = @testset ReportingTestSet begin + record_test_property("tests", "ABC-789") + @testset begin + record_test_property("tests", "ABC-1011") + @test true + end + end + @test test_properties(ts) == ["tests" => "ABC-789"] + @test test_properties(ts.results[1]) == ["tests" => "ABC-1011"] + + flattened_testsets = TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 1 + @test test_properties(flattened_testsets[1]) == ["tests" => "ABC-789", "tests" => "ABC-1011"] + + ts = @testset ReportingTestSet begin + record_test_property("tests", "ABC-789") + @testset begin + record_test_property("tests", "ABC-789") + @test true + end + @testset begin + @test true + end + end + @test test_properties(ts) == ["tests" => "ABC-789"] + @test test_properties(ts.results[1]) == ["tests" => "ABC-789"] + @test test_properties(ts.results[2]) == [] + + flattened_testsets = TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 2 + @test test_properties(flattened_testsets[1]) == ["tests" => "ABC-789", "tests" => "ABC-789"] + @test test_properties(flattened_testsets[2]) == ["tests" => "ABC-789"] + end + + @testset "custom testset support" begin + # Test for parent testset properties not being applied to child due to different type + ts = @testset ReportingTestSet "" begin + @testset ReportingTestSet "Outer" begin + record_test_property("ID", "42") + @testset TestReportingTestSet "Inner" begin + @test 1 == 1 + end + end + end + fail_text = "Properties of testset \"Outer\" can not be added to child testset \"Inner\" as it does not have a `TestReports.test_properties` method defined." + flattened_testsets = @test_logs (:warn, fail_text) TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 1 + @test testset_properties(flattened_testsets[1]) === nothing + + # Test for ReportingTestSet setting a property inside of a parent custom testset + ts = @testset ReportingTestSet "TestReports Wrapper" begin + @testset TestReportingTestSet "Custom" begin + ts = @testset ReportingTestSet "Inner" begin + record_test_property("ID", "42") + @test 1 == 1 + end + end + end + flattened_testsets = TestReports.flatten_results!(ts) + @test length(flattened_testsets) == 1 + @test test_properties(flattened_testsets[1]) == ["ID" => "42"] + end + + @testset "ignore properties on unsupported test sets" begin + ts = @testset begin + record_test_property("id", 1) + end + @test test_properties(ts) === nothing + end + + @testset "junit report" begin + # Check properties in XML doc variable + ts = @testset ReportingTestSet "TopLevel" begin + record_test_property("ID", "TopLevel") + @testset begin + record_test_property("Prop", "Inner 1") + @test true + end + @testset begin + record_test_property("Prop", "Inner 2") + @test true + end + end + rep = report(ts) + testsuite_elements = findall("//testsuite", root(rep)) + @test length(testsuite_elements) == 2 + + # Parent properties are first in report + testcase_property_nodes = findall("testcase/properties/property", testsuite_elements[1]) + @test length(testcase_property_nodes) == 2 + @test testcase_property_nodes[1]["name"] == "ID" + @test testcase_property_nodes[1]["value"] == "TopLevel" + @test testcase_property_nodes[2]["name"] == "Prop" + @test testcase_property_nodes[2]["value"] == "Inner 1" + + testcase_property_nodes = findall("testcase/properties/property", testsuite_elements[2]) + @test length(testcase_property_nodes) == 2 + @test testcase_property_nodes[1]["name"] == "ID" + @test testcase_property_nodes[1]["value"] == "TopLevel" + @test testcase_property_nodes[2]["name"] == "Prop" + @test testcase_property_nodes[2]["value"] == "Inner 2" + end +end diff --git a/test/recordproperty.jl b/test/recordproperty.jl deleted file mode 100644 index 49e30ce..0000000 --- a/test/recordproperty.jl +++ /dev/null @@ -1,144 +0,0 @@ -@testset "recordproperty / properties" begin - using TestReports: properties - - @testset "empty" begin - ts = @testset ReportingTestSet begin end - @test properties(ts) isa AbstractVector - @test length(properties(ts)) == 0 - - ts = @testset "_" begin end - @test properties(ts) === nothing - end - - @testset "record property" begin - ts = @testset ReportingTestSet begin - recordproperty("tested-item-id", "SAMD-45") - end - @test properties(ts) == ["tested-item-id" => "SAMD-45"] - - ts = @testset ReportingTestSet begin - recordproperty("count", 3) - end - @test properties(ts) == ["count" => 3] - end - - @testset "multiple properties" begin - ts = @testset ReportingTestSet begin - recordproperty("tests", "ABC-789") - recordproperty("tests", "ABC-1011") - end - @test properties(ts) == ["tests" => "ABC-789", "tests" => "ABC-1011"] - - ts = @testset ReportingTestSet begin - recordproperty("tests", "ABC-789") - recordproperty("tests", "ABC-789") - end - @test properties(ts) == ["tests" => "ABC-789", "tests" => "ABC-789"] - end - - @testset "nested properties" begin - ts = @testset ReportingTestSet begin - recordproperty("tests", "ABC-789") - @testset begin - recordproperty("tests", "ABC-1011") - end - end - @test properties(ts) == ["tests" => "ABC-789"] - @test properties(ts.results[1]) == ["tests" => "ABC-1011"] - - flattened_testsets = TestReports.flatten_results!(ts) - @test length(flattened_testsets) == 2 - @test properties(flattened_testsets[1]) == ["tests" => "ABC-789"] - @test properties(flattened_testsets[2]) == ["tests" => "ABC-789", "tests" => "ABC-1011"] - - ts = @testset ReportingTestSet begin - recordproperty("tests", "ABC-789") - @testset begin - recordproperty("tests", "ABC-789") - end - @testset begin - end - end - @test properties(ts) == ["tests" => "ABC-789"] - @test properties(ts.results[1]) == ["tests" => "ABC-789"] - @test properties(ts.results[2]) == [] - - flattened_testsets = TestReports.flatten_results!(ts) - @test length(flattened_testsets) == 2 - @test properties(flattened_testsets[1]) == ["tests" => "ABC-789"] - @test properties(flattened_testsets[2]) == ["tests" => "ABC-789", "tests" => "ABC-789"] - end - - @testset "custom testset support" begin - # Test for parent testset properties not being applied to child due to different type - ts = @testset ReportingTestSet "" begin - @testset ReportingTestSet "Outer" begin - recordproperty("ID", "42") - @testset TestReportingTestSet "Inner" begin - @test 1 == 1 - end - end - end - fail_text = "Properties of testset \"Outer\" can not be added to child testset \"Inner\" as it does not have a `TestReports.properties` method defined." - flattened_testsets = @test_logs (:warn, fail_text) TestReports.flatten_results!(ts) - @test length(flattened_testsets) == 2 - @test properties(flattened_testsets[1]) == ["ID" => "42"] - @test properties(flattened_testsets[2]) === nothing - - # Test for ReportingTestSet setting a property inside of a parent custom testset - ts = @testset ReportingTestSet "TestReports Wrapper" begin - @testset TestReportingTestSet "Custom" begin - ts = @testset ReportingTestSet "Inner" begin - recordproperty("ID", "42") - @test 1 == 1 - end - end - end - flattened_testsets = TestReports.flatten_results!(ts) - @test length(flattened_testsets) == 1 - @test properties(flattened_testsets[1]) == ["ID" => "42"] - end - - @testset "ignore properties on unsupported test sets" begin - ts = @testset begin - recordproperty("id", 1) - end - @test properties(ts) === nothing - end - - @testset "junit report" begin - # Check properties in XML doc variable - ts = @testset ReportingTestSet "TopLevel" begin - recordproperty("ID", "TopLevel") - @testset begin - recordproperty("Prop", "Inner 1") - end - @testset begin - recordproperty("Prop", "Inner 2") - end - end - rep = report(ts) - testsuite_elements = findall("//testsuite", root(rep)) - @test length(testsuite_elements) == 3 - - # Child properties are first in report - testsuite_property_elements = findall("properties/property", testsuite_elements[1]) - @test length(testsuite_property_elements) == 1 - @test testsuite_property_elements[1]["name"] == "ID" - @test testsuite_property_elements[1]["value"] == "TopLevel" - - testsuite_property_elements = findall("properties/property", testsuite_elements[2]) - @test length(testsuite_property_elements) == 2 - @test testsuite_property_elements[1]["name"] == "ID" - @test testsuite_property_elements[1]["value"] == "TopLevel" - @test testsuite_property_elements[2]["name"] == "Prop" - @test testsuite_property_elements[2]["value"] == "Inner 1" - - testsuite_property_elements = findall("properties/property", testsuite_elements[3]) - @test length(testsuite_property_elements) == 2 - @test testsuite_property_elements[1]["name"] == "ID" - @test testsuite_property_elements[1]["value"] == "TopLevel" - @test testsuite_property_elements[2]["name"] == "Prop" - @test testsuite_property_elements[2]["value"] == "Inner 2" - end -end diff --git a/test/references/test_with_properties.txt b/test/references/test_with_properties.txt index 115705f..81ea393 100644 --- a/test/references/test_with_properties.txt +++ b/test/references/test_with_properties.txt @@ -1,2 +1,2 @@ - + diff --git a/test/references/test_with_properties_pre_1_7.txt b/test/references/test_with_properties_pre_1_7.txt index 0cb14a3..77281e5 100644 --- a/test/references/test_with_properties_pre_1_7.txt +++ b/test/references/test_with_properties_pre_1_7.txt @@ -1,2 +1,2 @@ - + diff --git a/test/runtests.jl b/test/runtests.jl index 4dd3c31..b0bc1cd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -12,7 +12,7 @@ include("utils.jl") # Include other test scripts @testset "TestReports" begin @testset "testsets" begin include("testsets.jl") end - @testset "record property" begin include("recordproperty.jl") end + @testset "properties" begin include("properties.jl") end @testset "report generation" begin include("reportgeneration.jl") end @testset "runner internals" begin include("runnerinternals.jl") end @testset "to_xml" begin include("to_xml.jl") end diff --git a/test/test_packages/TestsWithProperties/test/runtests.jl b/test/test_packages/TestsWithProperties/test/runtests.jl index 0eedf78..c622297 100644 --- a/test/test_packages/TestsWithProperties/test/runtests.jl +++ b/test/test_packages/TestsWithProperties/test/runtests.jl @@ -3,31 +3,39 @@ using TestReports using Base.Threads @testset "Outer" begin - recordproperty("File", "runtests.jl") + record_testset_property("File", "runtests.jl") @testset "Middle 1" begin - recordproperty("ID", 1) + record_testset_property("ID", 1) @test true end @test true @testset "Middle 2" begin - recordproperty("ID", 2) + record_testset_property("ID", 2) @test true @testset "Inner" begin - recordproperty("ID", 3) - recordproperty("AdditionalNest", true) + record_testset_property("AdditionalNest", true) + end + end + + @testset "Middle 3" begin + record_test_property("ID", 3) + @test true + + @testset "Inner" begin + record_test_property("AdditionalNest", true) end end end @testset "Types" begin - recordproperty("String", "TextTests") - recordproperty("Int", 1) - recordproperty("Float", 0.5) - recordproperty("List", ["1"]) - recordproperty("Symbol", :asymbol) + record_testset_property("String", "TextTests") + record_testset_property("Int", 1) + record_testset_property("Float", 0.5) + record_testset_property("List", ["1"]) + record_testset_property("Symbol", :asymbol) @test true end diff --git a/test/testsets.jl b/test/testsets.jl index 60c4cf4..b013d3d 100644 --- a/test/testsets.jl +++ b/test/testsets.jl @@ -1,7 +1,7 @@ -@testset "recordproperty!" begin +@testset "record_test_property!" begin ts = @testset ReportingTestSet begin end - @test TestReports.recordproperty!(ts, "id", 1) === ts - @test TestReports.properties(ts) == ["id" => 1] + @test TestReports.record_test_property!(ts, "id", 1) === ts + @test TestReports.test_properties(ts) == ["id" => 1] end @testset "flatten_results!" begin