Revert "Feature 461 make compare images configurable (#467)"

This reverts commit 5c189be.

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+relative_files = True

pyproject.toml

@@ -33,11 +33,6 @@ testpaths = ["test"]
 
 [tool.coverage.run]
 source = ["metplotpy/plots"]
-omit = [
-    "config.py",
-    "config-3.py",
-    ]
-relative_files = true
 
 [tool.coverage.report]
 exclude_also = [

test/bar/test_bar.py

@@ -3,7 +3,7 @@
 import pytest
 
 from metplotpy.plots.bar import bar
-from metcalcpy.compare_images import CompareImages
+# from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['bar.png', 'bar.points1']
@@ -71,6 +71,7 @@ def test_no_nans_in_points_file(setup, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
+@pytest.mark.skip("fails on linux host machines")
 def test_images_match(setup, remove_files):
     """
         Compare an expected plot with the
@@ -82,6 +83,7 @@ def test_images_match(setup, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
+@pytest.mark.skip("fails on linux host machines")
 def test_none_data_images_match(setup_nones):
     """
         Compare an expected plot with the
@@ -139,6 +141,7 @@ def test_point_and_plot_files_exist_default(setup_env, remove_files):
     remove_files(cwd, check_files)
 
 
+@pytest.mark.skip("fails on linux host machines")
 def test_threshold_plotting(setup_env, remove_files):
     """
       Verify that the bar plot using data with thresholds is correct.

test/box/test_box.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.box import box
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['box.png', 'box.points1']
@@ -28,6 +28,7 @@ def test_files_exist(setup, test_input, expected, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
+@pytest.mark.skip("fails on linux hosts")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/conftest.py

@@ -1,6 +1,5 @@
 import pytest
 import os
-from unittest.mock import patch
 import shutil
 import json
 import xarray as xr
@@ -11,38 +10,13 @@
 # realative file locations can be used for each test
 # file.
 # NOTE: autouse=True means this applies to ALL tests.
-# Code that updates the cwd inside test is now redundant
-# and can be deleted.
+# Code that updates the cwd inside a test  file is now
+# redundant and can be deleted.
 @pytest.fixture(autouse=True)
 def change_test_dir(request, monkeypatch):
     monkeypatch.chdir(request.fspath.dirname)
 
 
-@pytest.fixture(autouse=True)
-def patch_CompareImages(request):
-    """This fixture controls the use of CompareImages in the
-    test suite. By default, all calls to CompareImages will
-    result in the test skipping. To change this behaviour set
-    an env var METPLOTPY_COMPAREIMAGES
-    """
-    if bool(os.getenv("METPLOTPY_COMPAREIMAGES")):
-        yield
-    else:
-        class mock_CompareImages:
-            def __init__(self, img1, img2):
-                # TODO: rather than skip we could inject an alternative
-                # comparison that is more relaxed. To do this, extend
-                # this this class to generate a self.mssim value.
-                pytest.skip("CompareImages not enabled in pytest. "
-                            "To enable `export METPLOTPY_COMPAREIMAGES=$true`")
-        try:
-            with patch.object(request.module, 'CompareImages', mock_CompareImages) as mock_ci:
-                yield mock_ci
-        except AttributeError:
-            # test module doesn't import CompareImages. Do nothing.
-            yield
-
-
 def ordered(obj):
     """Recursive function to sort JSON, even lists of dicts with the same keys"""
     if isinstance(obj, dict):
@@ -52,6 +26,7 @@ def ordered(obj):
     else:
         return obj
 
+
 @pytest.fixture
 def assert_json_equal():
     def compare_json(fig, expected_json_file):
@@ -61,6 +36,7 @@ def compare_json(fig, expected_json_file):
         actual = json.loads(fig.to_json(), parse_float=str, parse_int=str)
         with open(expected_json_file) as f:
             expected = json.load(f,parse_float=str, parse_int=str)
+
         # Fail with a nice message
         if ordered(actual) == ordered(expected):
             return True
@@ -103,7 +79,6 @@ def remove_the_files(test_dir, file_list):
     return remove_the_files
 
 
-# data for netCDF file
 TEST_NC_DATA = xr.Dataset(
     {
         "precip": xr.DataArray(

test/contour/test_contour.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.contour import contour
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -42,7 +42,7 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
-
+@pytest.mark.skip("fails on linux hosts")
 def test_images_match(setup):
     """
         Compare an expected plots with the

test/eclv/test_eclv.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.eclv import eclv
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -49,6 +49,7 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
+@pytest.mark.skip("fails on linux hosts")
 def test_images_match(setup):
     """
         Compare an expected plots with the

test/ens_ss/test_ens_ss.py

@@ -1,6 +1,6 @@
 import os
 import pytest
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 from metplotpy.plots.ens_ss import ens_ss
 
 cwd = os.path.dirname(__file__)
@@ -43,7 +43,7 @@ def test_files_exist( setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
-
+@pytest.mark.skip("fails on linux hosts")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/equivalence_testing_bounds/test_equivalence_testing_bounds.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.equivalence_testing_bounds import equivalence_testing_bounds as etb
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['equivalence_testing_bounds.png', 'equivalence_testing_bounds.points1']
@@ -29,6 +29,7 @@ def test_files_exist(setup, test_input, expected, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
+@pytest.mark.skip("skimage differences causing failure")
 def test_images_match(setup, remove_files):
     '''
         Compare an expected plot with the

test/histogram/test_prob_hist.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.histogram import prob_hist
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -35,7 +35,7 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(f"{cwd}/{test_input}") == expected
     cleanup()
 
-
+@pytest.mark.skip("Image comparisons fail during Github Actions checks.")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/histogram/test_rank_hist.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.histogram import rank_hist
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -36,6 +36,7 @@ def test_files_exist(setup, test_input, expected):
     cleanup()
 
 
+@pytest.mark.skip("Image comparisons fail during Github Actions checks.")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/histogram/test_rel_hist.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.histogram import rel_hist
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -39,6 +39,7 @@ def test_files_exist(setup, test_input, expected):
     cleanup()
 
 
+@pytest.mark.skip("Image comparisons fail in Github Actions checks.")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/histogram_2d/test_histogram_2d.py

@@ -3,6 +3,8 @@
 
 from metplotpy.plots.histogram_2d import histogram_2d as h2d
 
+# from metcalcpy.compare_images import CompareImages
+
 cwd = os.path.dirname(__file__)
 
 

test/hovmoeller/test_hovmoeller.py

@@ -2,7 +2,7 @@
 import pytest
 import metplotpy.plots.hovmoeller.hovmoeller as hov
 from metplotpy.plots import util
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 def dict_to_yaml(data_dict,
                 output_yaml = "test_hovmoeller.yaml"):
@@ -22,7 +22,8 @@ def cleanup(file_to_remove):
         # don't exist.  Ignore.
         pass
 
-@pytest.mark.skip("Requires specific netCDF input")
+
+@pytest.mark.skip()
 def test_default_plot_images_match():
     '''
         Compare an expected plot with the

test/line/test_line_groups_plot.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.line import line as l
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -47,7 +47,7 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
-
+@pytest.mark.skip("fails on linux hosts")
 def test_images_match(setup):
     '''
         Compare an expected plot with the

test/line/test_line_plot.py

@@ -5,7 +5,7 @@
 
 cwd = os.path.dirname(__file__)
 
-from metcalcpy.compare_images import CompareImages
+# from metcalcpy.compare_images import CompareImages
 
 
 @pytest.fixture
@@ -147,6 +147,7 @@ def test_no_nans_in_points_files():
         pass
 
 
+@pytest.mark.skip()
 def test_images_match(setup):
     '''
         Compare an expected plot with the
@@ -163,6 +164,7 @@ def test_images_match(setup):
     cleanup()
 
 
+@pytest.mark.skip()
 def test_new_images_match():
     '''
         Compare an expected plot with the start_at_zero option, with the
@@ -321,20 +323,21 @@ def test_fixed_var_val():
         pass
 
 
-@pytest.mark.skip("This test currently raises an error on line 81 of line.py. This can"
-                  " probably be fixed be revisiting fbias_fixed_vars_vals.yaml")
+@pytest.mark.skip("Image comparison for development only due to differences in hosts")
 def test_fixed_var_val_image_compare():
     """
         Verify that the fixed_vars_vals_input setting reproduces the
         expected plot.
 
     """
+    from metcalcpy.compare_images import CompareImages
 
     # Set up the METPLOTPY_BASE so that met_plot.py will correctly find
     # the config directory containing all the default config files.
     os.environ['METPLOTPY_BASE'] = f"{cwd}/../../"
     os.environ['TEST_DIR'] = cwd
     custom_config_filename = f"{cwd}/fbias_fixed_vars_vals.yaml"
+
     # Invoke the command to generate a line plot based on
     # the custom config file.
     l.main(custom_config_filename)
@@ -346,7 +349,10 @@ def test_fixed_var_val_image_compare():
         created_file = os.path.join(cwd, plot_file)
 
         # first verify that the output plot was created
-        assert os.path.exists(created_file)
+        if os.path.exists(created_file):
+            assert True
+        else:
+            assert False
 
         comparison = CompareImages(expected_plot, created_file)
 

test/mpr_plot/test_mpr_plot.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.mpr_plot import mpr_plot
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['mpr_plots.png']
@@ -29,6 +29,7 @@ def test_files_exist(setup, test_input, expected, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
+@pytest.mark.skip("unreliable-sometimes fails due to differences between machines.")
 def test_images_match(setup, remove_files):
     """
         Compare an expected plot with the

test/performance_diagram/test_performance_diagram.py

@@ -2,7 +2,7 @@
 import pytest
 
 from metplotpy.plots.performance_diagram import performance_diagram as pd
-from metcalcpy.compare_images import CompareImages
+#from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -89,6 +89,7 @@ def test_files_exist(setup_env, test_input, expected_bool, remove_files):
         pass
 
 
+@pytest.mark.skip()
 def test_images_match(setup, remove_files):
     '''
         Compare an expected plot with the