[WIP] test_reference_broken

src/ReferenceTests.jl

@@ -13,10 +13,12 @@ export
     @withcolor,
     @io2str,
     @test_reference,
+    @test_reference_broken,
     psnr_equality
 
 include("utils.jl")
 include("test_reference.jl")
+include("test_reference_broken.jl")
 include("fileio.jl")
 include("equality_metrics.jl")
 include("render.jl")

src/test_reference_broken.jl

@@ -0,0 +1,61 @@
+macro test_reference_broken(reference, actual, kws...)
+    dir = Base.source_dir()
+    expr = :(test_reference_broken(abspath(joinpath($dir, $(esc(reference)))), $(esc(actual))))
+    for kw in kws
+        (kw isa Expr && kw.head == :(=)) || error("invalid signature for @test_reference_broken")
+        k, v = kw.args
+        push!(expr.args, Expr(:kw, k, esc(v)))
+    end
+    expr
+end
+
+function test_reference_broken(
+    filename::AbstractString, raw_actual;
+    by = nothing, render = nothing, kw...)
+
+    test_reference_broken(query_extended(filename), raw_actual, by, render; kw...)
+end
+
+function test_reference_broken(
+    file::File{F},
+    raw_actual::T,
+    equiv=nothing,
+    rendermode=nothing;
+    kw...) where {F <: DataFormat, T}
+
+    path = file.filename
+    dir, filename = splitdir(path)
+
+    # infer the default rendermode here
+    # since `nothing` is always passed to this method from
+    # test_reference_broken(filename::AbstractString, raw_actual; kw...)
+    if rendermode === nothing
+        rendermode = default_rendermode(F, raw_actual)
+    end
+
+    # mark as broken if file doesn't exist -- unlike test_reference
+    if !isfile(path)
+        @test_broken false
+        return nothing
+    end
+
+    # file exists
+    actual = _convert(F, raw_actual; kw...)
+    reference = loadfile(T, file)
+
+    if equiv === nothing
+        # generally, `reference` and `actual` are of the same type after preprocessing
+        equiv = default_equality(reference, actual)
+    end
+
+    if equiv(reference, actual)
+        println("Got correct result for \"$filename\", please change to @test_reference if no longer broken.")
+        render(rendermode, reference, actual)
+
+        @test_broken true
+    else
+        @test_broken false
+    end
+
+    return nothing # TODO: @test_broken returns a Test.Broken or Test.Error
+end

test/runtests.jl

@@ -64,9 +64,9 @@ end
         multiline string that does indeed end with a new line.
     """
 
-    @test_throws ErrorException @test_reference "references/string1.txt" "intentionally wrong to check that this message prints"
-    @test_throws ErrorException @test_reference "references/wrong.txt" "intentional error to check that this message prints"
-    @test_throws ErrorException @test_reference "references/string5.txt" """
+    @test_reference_broken "references/string1.txt" "intentionally wrong to check that this message prints"
+    @test_reference_broken "references/wrong.txt" "intentional error to check that this message prints"
+    @test_reference_broken "references/string5.txt" """
         This is an incorrect
         multiline string that does not end with a new line."""
 end
@@ -87,7 +87,7 @@ end
         @io2str(printstyled(IOContext(::IO, :color=>true), "this should be blue", color=:blue)),
         render = ReferenceTests.BeforeAfterFull()
     )
-    @test_throws ErrorException @test_reference(
+    @test_reference_broken(
         "references/ansii.txt",
         @io2str(printstyled(IOContext(::IO, :color=>true), "this should be red", color=:red)),
         render = ReferenceTests.BeforeAfterFull()
@@ -110,16 +110,16 @@ end
 @testset "images as PNG" begin
     @test_reference "references/camera.png" imresize(camera, (64,64))
     @test_reference "references/camera.png" imresize(camera, (64,64)) by=psnr_equality(25)
-    @test_throws Exception @test_reference "references/wrongfilename.png" imresize(camera, (64,64))
-    @test_throws ErrorException @test_reference "references/camera.png" imresize(lena, (64,64))
-    @test_throws Exception @test_reference "references/camera.png" camera # unequal size
+    @test_reference_broken "references/wrongfilename.png" imresize(camera, (64,64))
+    @test_reference_broken "references/camera.png" imresize(lena, (64,64))
+    @test_reference_broken "references/camera.png" camera # unequal size
 end
 
 using DataFrames, CSVFiles
 @testset "DataFrame as CSV" begin
     @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"])
-    @test_throws ErrorException @test_reference "references/wrongfilename.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"])
-    @test_throws ErrorException @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"])
+    @test_reference_broken "references/wrongfilename.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"])
+    @test_reference_broken "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"])
 end
 
 end