1.0 release and do not automatically create if missing

.github/workflows/UnitTest.yml

@@ -42,7 +42,9 @@ jobs:
         run: |
           julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)"
           rm -rf test/references
+          export JULIA_REFERENCETESTS_UPDATE=true
           julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)"
+          export JULIA_REFERENCETESTS_UPDATE=false
           julia --color=yes --check-bounds=yes --project -e "using Pkg; Pkg.test(coverage=true)"
       - name: Upload Mismatched Files as a Build Artifact
         uses: actions/upload-artifact@v3

Project.toml

@@ -1,7 +1,7 @@
 name = "ReferenceTests"
 uuid = "324d217c-45ce-50fc-942e-d289b448e8cf"
 authors = ["Christof Stocker <[email protected]>", "Frames White <[email protected]>", "Johnny Chen <[email protected]>"]
-version = "0.10.4"
+version = "1.0.0"
 
 [deps]
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"

README.md

@@ -37,10 +37,9 @@ using ReferenceTests
 
 If you put the above code into your `test/runtests.jl` and
 execute the file in an interactive julia session (i.e. with
-`include`), then it will trigger an interactive dialog if the
-results don't match. This dialog allows the user to update the
-reference files. If you do not want to be prompted, just
-delete the reference data before running the tests.
+`include`), then it will trigger an interactive dialog if the refrence is missing or the results don't match.
+This dialog allows the user to create/update the reference files.
+If you do not want to be prompted, set `ENV["JULIA_REFERENCETESTS_UPDATE"]=true` before running.
 
 ![readme1](https://user-images.githubusercontent.com/10854026/30002940-3ba480b0-90b6-11e7-93f6-148ac38bd695.png)
 
@@ -91,13 +90,11 @@ test suite. These tests are easy to run via `pkg> test` but
 the child process used within `pkg> test` is non-interactive, so the
 update prompt will not show if there are mismatches.
 
-To update references within a package test suite, there are three options:
+To update/create references within a package test suite, there are two options:
 
 - Set the environment variable `JULIA_REFERENCETESTS_UPDATE` to `"true"`
-  and run `pkg> test`, which will force update any non-matches. You can then
-  check changes to any git-tracked reference images before commit.
-- Delete any reference images you wish to update and run `pkg> test`, given
-  that missing references are created automatically.
+  and run `pkg> test`, which will force update any non-matches and create any missing files.
+  You can then check changes to any git-tracked reference images before commit.
 - Run the `test/runtests.jl` interactively. This may be easier using
   the [`TestEnv.jl`](https://github.com/JuliaTesting/TestEnv.jl) package,
   given that the test environment used by `pkg> test` is a merge of the

src/test_reference.jl

@@ -117,9 +117,7 @@ function test_reference(
     rendermode=nothing;
     kw...) where {F <: DataFormat, T}
 
-    reference_path = reference_file.filename
-    reference_dir, reference_filename = splitdir(reference_path)
-
+    reference_filename = basename(reference_file.filename)
     actual = _convert(F, raw_actual; kw...)
 
     # infer the default rendermode here
@@ -129,14 +127,14 @@ function test_reference(
         rendermode = default_rendermode(F, actual)
     end
 
-    if !isfile(reference_path)  # when reference file doesn't exists
-        mkpath(reference_dir)
-        savefile(reference_file, actual)
-        @info """Reference file for \"$reference_filename\" did not exist. It has been created:
+    if !isfile(reference_file.filename)  # when reference file doesn't exists
+        actual_path = save_mismatch_file(reference_file, actual)
+
+        @info """Reference file for \"$reference_filename\" did not exist:
         $(render(rendermode, actual))
-        """ new_reference = reference_path
+        """ actual=actual_path
 
-        @info "Please run the tests again for any changes to take effect"
+        handle_mismatch(reference_file, actual_path)
         return nothing # skip current test case
     end
 
@@ -152,33 +150,44 @@ function test_reference(
         @test true # to increase test counter if reached
     else  # When test fails
         # Saving actual file so user can look at it
-        actual_path = joinpath(mismatch_staging_dir(), reference_filename)
-        actual_file = typeof(reference_file)(actual_path)
-        savefile(actual_file, actual)
+        actual_path = save_mismatch_file(reference_file, actual)
 
         # Report to user.
         @info """Reference Test for \"$reference_filename\" failed:
         $(render(rendermode, reference, actual))
-        """ reference = reference_path actual = actual_path
-
-        if !isinteractive() && !force_update()
-            error("""
-            To update the reference images either run the tests interactively with 'include(\"test/runtests.jl\")',
-            or to force-update all failing reference images set the environment variable `JULIA_REFERENCETESTS_UPDATE`
-            to "true" and re-run the tests via Pkg.
-            """)
-        end
-
-        if force_update() || input_bool("Replace reference with actual result?")
-            mv(actual_path, reference_path; force=true)  # overwrite old file it
-            @info "Please run the tests again for any changes to take effect"
-        else
-            @test false
-        end
+        """ reference=reference_file.filename actual=actual_path
+        handle_mismatch(reference_file, actual_path)
+    end
+end
+
+function save_mismatch_file(reference_file::File{F}, actual) where F
+    # Saving actual file so user can look at it
+    actual_path = joinpath(mismatch_staging_dir(), basename(reference_file.filename))
+    actual_file = File{F}(actual_path)
+
+    savefile(actual_file, actual)
+    return actual_path
+end
+
+function handle_mismatch(reference_file, actual_path)
+    if !isinteractive() && !force_update()
+        error("""
+        To update the reference images either run the tests interactively with 'include(\"test/runtests.jl\")',
+        or to force-update/create all failing reference images set the environment variable `JULIA_REFERENCETESTS_UPDATE`
+        to "true" and re-run all tests.
+        """)
+    end
+
+    if force_update() || input_bool("Replace reference with actual result?")
+        mkpath(dirname(reference_file.filename))
+        mv(actual_path, reference_file.filename; force=true)  # overwrite old file
+        @info "Please run the tests again for any changes to take effect"
+    else
+        @test false
     end
 end
 
-force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) === true
+force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false"))
 
 """
     mismatch_staging_dir()

test/runtests.jl

@@ -162,21 +162,25 @@ end
     mktempdir() do path
         newfilename = joinpath(path, "newfilename.$ext")
         @assert !isfile(newfilename)
-        with_logger(test_logger) do
-            @test_reference newfilename val  # this should create it
+        withenv("JULIA_REFERENCETESTS_UPDATE"=>true) do
+            with_logger(test_logger) do
+                @test_reference newfilename val  # this should create it
+            end
         end
         @test isfile(newfilename)  # Was created
         @test_reference newfilename val  # Matches expected content
     end
-end
+ end
 
 @testset "Create new image as txt" begin
     # This is a separate testset as need to use the `size` argument to ``@test_reference`
     mktempdir() do path
         newfilename = joinpath(path, "new_camera.txt")
         @assert !isfile(newfilename)
         with_logger(test_logger) do
-            @test_reference newfilename camera size=(5,10)  # this should create it
+            withenv("JULIA_REFERENCETESTS_UPDATE"=>true) do
+                @test_reference newfilename camera size=(5,10)  # this should create it
+            end
         end
         @test isfile(newfilename)  # Was created
         @test_reference newfilename camera size=(5,10) # Matches expected content