Support test arguments in reporttests.jl (#109)

* Support test arguments in `reporttests.jl`

* Show basic help

bin/reporttests.jl

@@ -1,21 +1,88 @@
-#!julia
+#!/usr/bin/env julia
 
-testfilename = popfirst!(ARGS)
+# Usage:
+#   reporttests.jl TEST_FILENAME [--output=LOG_FILENAME] -- [test_args...]
+#   julia reporttests.jl -- TEST_FILENAME [--output=LOG_FILENAME] -- [test_args...]
 
-#testfile_str = String(read(testfilename))
-
-script = """
-using Test
 using TestReports
-ts = @testset ReportingTestSet "" begin
-    include($(repr(testfilename)))
+
+# Basic argument parsing without requring an extra dependency like ArgParse.jl
+function parse_args(args)
+    help_flag = false
+    test_filename = nothing
+    output_filename = nothing
+    test_args = String[]
+
+    option_key = nothing
+    state = :options
+    for (i, arg) in enumerate(args)
+        if state === :options && option_key !== nothing
+            if option_key == "--output"
+                output_filename = arg
+            else
+                error("Unhandled option: `$option_key`")
+            end
+            option_key = nothing
+        elseif state === :options && option_match(("--output", "-o"), arg)
+            option = split(arg, '='; limit=2)
+            if length(option) == 2
+                output_filename = option[2]
+            else
+                option_key = "--output"
+            end
+        elseif state === :options && option_match(("--help", "-h"), arg)
+            help_flag = true
+            break
+        elseif state === :options && arg == "--"
+            # Ignore the first double-dash as Julia versions before 1.9.0-DEV.604 would
+            # automatically exclude it.
+            # https://github.com/JuliaLang/julia/pull/45335
+            i > 1 && (state = :positional_args)
+        elseif test_filename === nothing
+            test_filename = arg
+        else
+            push!(test_args, arg)
+        end
+    end
+
+    if help_flag || isempty(args)
+        println(stderr, """
+            Generate a JUnit test report from running Julia tests.
+
+            Usage:
+              reporttests.jl TEST_FILENAME [--output=LOG_FILENAME] -- [test_args...]
+              julia reporttests.jl -- TEST_FILENAME [--output=LOG_FILENAME] -- [test_args...]
+            """)
+        return nothing
+    end
+
+    if test_filename === nothing
+        throw(ArgumentError("Required positional argument `test_filename` not set"))
+    end
+
+    # Use Julia 1.0 compatible named tuple syntax without Compat.jl
+    return (; test_filename=test_filename, output_filename=output_filename, test_args=test_args)
 end
 
-open("testlog.xml","w") do fh
-    # Flatten before calling `report` to avoid a `deepcopy`.
-    print(fh, report(TestReports.flatten_results!(ts)))
+function option_match(option_keys, arg)
+    return arg in option_keys || any(o -> startswith(arg, "$o="), option_keys)
 end
-exit(any_problems(ts))
-"""
 
-run(`$(Base.julia_cmd()) -e $script`)
+if abspath(PROGRAM_FILE) == @__FILE__()
+    parsed = parse_args(ARGS)
+    parsed === nothing && exit(1)
+    coverage = Base.JLOptions().code_coverage != 0
+    runner_code = TestReports.gen_runner_code(
+        parsed.test_filename,
+        something(parsed.output_filename, "testlog.xml"),
+        Cmd(parsed.test_args),
+    )
+    cmd = TestReports.gen_command(runner_code, ``, coverage)
+
+    try
+        run(cmd)
+    catch e
+        e isa ProcessFailedException && exit(only(e.procs).exitcode)
+        rethrow()
+    end
+end

test/references/reporttests_fail.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="1" errors="0"><testsuite name="reporttest" tests="1" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="ARGS == []" id="1" classname="reporttest" time="0.0"><failure message="[&quot;foo&quot;, &quot;-e&quot;, &quot;bar&quot;] == Any[]" type="test">Test Failed
+  Expression: ARGS == []
+   Evaluated: ["foo", "-e", "bar"] == Any[]
+</failure></testcase></testsuite></testsuites>

test/references/reporttests_fail_pre_1_7.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="1" errors="0"><testsuite name="reporttest" tests="1" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="ARGS == []" id="1" classname="reporttest" time="0.0"><failure message="[&quot;foo&quot;, &quot;-e&quot;, &quot;bar&quot;] == Any[]" type="test">Test Failed
+  Expression: ARGS == []
+   Evaluated: ["foo", "-e", "bar"] == Any[]</failure></testcase></testsuite></testsuites>

test/references/reporttests_pass.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="0" errors="0"><testsuite name="reporttest" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="ARGS == []" id="1" classname="reporttest" time="0.0"/></testsuite></testsuites>

test/references/reporttests_pass_pre_1_7.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="0" errors="0"><testsuite name="reporttest" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="reporttest" time="0.0"/></testsuite></testsuites>

test/reporttests_script.jl

@@ -0,0 +1,100 @@
+script_runner = if VERSION >= v"1.7"
+    joinpath(pkgdir(TestReports), "bin", "reporttests.jl")
+else
+    joinpath(@__DIR__(), "..", "bin", "reporttests.jl")
+end
+script_runner_cmd = Sys.iswindows() ? `julia $script_runner --` : `$script_runner`
+test_script = "reporttests_testsets.jl"
+reference_suffix = VERSION >= v"1.7" ? "" : "_pre_1_7"
+
+@testset "parse_args" begin
+    include(script_runner)
+    @test parse_args([]) === nothing  # Shows help
+    @test_throws ArgumentError parse_args(["--"])
+
+    parsed = parse_args(split("script.jl"))
+    @test parsed.test_filename == "script.jl"
+    @test parsed.output_filename === nothing
+    @test parsed.test_args == String[]
+
+    output_filename = "junit-report.xml"
+    @testset "$output" for output in (
+        "--output $output_filename",
+        "-o $output_filename",
+        "--output=$output_filename",
+        "-o=$output_filename")
+
+        parsed = parse_args(split("script.jl $output"))
+        @test parsed.test_filename == "script.jl"
+        @test parsed.output_filename == output_filename
+        @test parsed.test_args == String[]
+    end
+
+    parsed = parse_args(split("script.jl foo bar -- -o baz"))
+    @test parsed.test_args == String["foo", "bar", "-o", "baz"]
+
+    parsed = parse_args(split("script.jl -- foo bar -- -o baz"))
+    @test parsed.test_args == String["foo", "bar", "--", "-o", "baz"]
+
+    parsed = parse_args(split("-- script.jl foo bar -- -o baz"))
+    @test parsed.test_args == String["foo", "bar", "-o", "baz"]
+
+    @testset "$help" for help in ("--help", "-h")
+        parsed = parse_args([help])
+        @test parsed === nothing
+    end
+end
+
+@testset "executable" begin
+    @test isfile(script_runner)
+
+    if Sys.islinux()
+        @test uperm(script_runner) & 0x01 != 0
+        @test gperm(script_runner) & 0x01 != 0
+        @test operm(script_runner) & 0x01 != 0
+    end
+end
+
+@testset "no specified test script" begin
+    p = run(ignorestatus(`$script_runner_cmd`))
+    @test !success(p)
+end
+
+@testset "default output file" begin
+    reference_file = "references/reporttests_pass$reference_suffix.xml"
+    output_file = "testlog.xml"
+    p = run(ignorestatus(`$script_runner_cmd $test_script`))
+    try
+        @test success(p)
+        @test isfile(output_file)
+        @test_reference reference_file read(output_file, String) |> clean_output
+    finally
+        isfile(output_file) && rm(output_file)
+    end
+end
+
+@testset "specify output file" begin
+    reference_file = "references/reporttests_pass$reference_suffix.xml"
+    output_file = "junit-report.xml"
+    p = run(ignorestatus(`$script_runner_cmd $test_script --output=$output_file`))
+    try
+        @test success(p)
+        @test isfile(output_file)
+        @test_reference reference_file read(output_file, String) |> clean_output
+    finally
+        isfile(output_file) && rm(output_file)
+    end
+end
+
+@testset "test args" begin
+    reference_file = "references/reporttests_fail$reference_suffix.xml"
+    output_file = "testlog.xml"
+    p = run(ignorestatus(`$script_runner_cmd $test_script -- foo -e bar`))
+    try
+        @test !success(p)
+        @test isfile(output_file)
+        @test_reference reference_file read(output_file, String) |> clean_output
+    finally
+        isfile(output_file) && rm(output_file)
+    end
+end

test/reporttests_testsets.jl

@@ -0,0 +1,5 @@
+using Test
+
+@testset "reporttest" begin
+    @test ARGS == []
+end

test/runtests.jl

@@ -16,4 +16,5 @@ include("utils.jl")
     @testset "report generation" begin include("reportgeneration.jl") end
     @testset "runner internals" begin include("runnerinternals.jl") end
     @testset "to_xml" begin include("to_xml.jl") end
+    @testset "reporttests script" begin include("reporttests_script.jl") end
 end