diff --git a/src/core.jl b/src/core.jl
index 97977b7..3200d8d 100644
--- a/src/core.jl
+++ b/src/core.jl
@@ -94,7 +94,8 @@ function _test_reference(equiv, rendermode, file::File, actual::T) where T
                     @test false
                 end
             else
-                error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
+                warn("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images")
+                @test false
             end
         end
     catch ex
@@ -113,7 +114,8 @@ function _test_reference(equiv, rendermode, file::File, actual::T) where T
                     @test false
                 end
             else
-                error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images")
+                warn("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images")
+                @test false
             end
         else
             rethrow(ex)
diff --git a/test/runtests.jl b/test/runtests.jl
index 86467c8..6f1b521 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -15,6 +15,29 @@ copy!(view(cameras,:,:,2), camera)
 square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9]
 rgb_rect = rand(RGB{N0f8}, 2, 3)
 
+struct NonPropagatingTestSet <: AbstractTestset
+	description::String
+	results::Vector
+end
+NonPropagatingTestSet(desc) = NonPropagatingTestSet(desc, [])
+record(ts::NonPropagatingTestSet, t) = (push!(ts.results, t); t)
+function finish(ts::ReportingTestSet)
+    # Teststs are supposed to throw errors if they have failures and no parent testset
+    # or if they do have a parent, then push themselves into the parents results
+    # but we are going to do none of that.
+end
+
+
+macro ensure_fails_tests(expr)
+    quote
+        ts = @testset NonPropagatingTestSet "Ensure Fails" begin
+            $expr
+	end
+	@assert length(ts.results)==1
+	@test ts.results[1] isa Test.Fail
+    end
+end
+
 @testset "io2str" begin
     @test_throws ArgumentError eval(@macroexpand @io2str(::IO))
     @test_throws ArgumentError @io2str(2)
@@ -50,11 +73,11 @@ end
         multiline string that does indeed end with a new line.
     """
 
-    @test_throws ErrorException @test_reference "references/string1.txt" "intentionally wrong to check that this message prints"
-    @test_throws ErrorException @test_reference "references/wrong.txt" "intentional error to check that this message prints"
-    @test_throws ErrorException @test_reference "references/string5.txt" """
+    @ensure_fails_tests(@test_reference "references/string1.txt" "intentionally wrong to check that this message prints")
+    @ensure_fails_tests(@test_reference "references/wrong.txt" "intentional error to check that this message prints")
+    @ensure_fails_tests(@test_reference "references/string5.txt" """
         This is an incorrect
-        multiline string that does not end with a new line."""
+        multiline string that does not end with a new line.""")
 end
 
 @testset "images as txt using ImageInTerminal" begin
@@ -78,13 +101,13 @@ end
 
 @testset "images as PNG" begin
     @test_reference "references/camera.png" imresize(camera, (64,64))
-    @test_throws ErrorException @test_reference "references/wrongfilename.png" imresize(camera, (64,64))
-    @test_throws ErrorException @test_reference "references/camera.png" imresize(lena, (64,64))
+    @ensure_fails_tests(@test_reference "references/wrongfilename.png" imresize(camera, (64,64)))
+    @ensure_fails_tests(@test_reference "references/camera.png" imresize(lena, (64,64)))
 end
 
 using DataFrames, CSVFiles
 @testset "DataFrame as CSV" begin
     @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"])
-    @test_throws ErrorException @test_reference "references/wrongfilename.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"])
-    @test_throws ErrorException @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"])
+    @ensure_fails_tests(@test_reference "references/wrongfilename.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"]))
+    @ensure_fails_tests(@test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"]))
 end