Pinned versions utilities and abort suppression (#429)

* suppress abort message and add check hash utility

* code changes

setup.py

@@ -45,6 +45,7 @@
                'configuration/*/*/*',
                'configuration/*/*/*/*',
                'configuration/*/*/*/*/*',
+               'utilities/pinned_versions/*.yaml'
              ],
     },
     include_package_data=True,

src/swell/test/code_tests/code_tests.py

@@ -11,11 +11,12 @@
 import os
 import unittest
 
-from swell.test.code_tests.question_dictionary_comparison_test import QuestionDictionaryTest
-from swell.test.code_tests.unused_variables_test import UnusedVariablesTest
+from swell.utilities.logger import Logger
 from swell.test.code_tests.slurm_test import SLURMConfigTest
+from swell.test.code_tests.test_pinned_versions import PinnedVersionsTest
+from swell.test.code_tests.unused_variables_test import UnusedVariablesTest
+from swell.test.code_tests.question_dictionary_comparison_test import QuestionDictionaryTest
 from swell.test.code_tests.test_generate_observing_system import GenerateObservingSystemTest
-from swell.utilities.logger import Logger
 
 
 # --------------------------------------------------------------------------------------------------
@@ -45,6 +46,9 @@ def code_tests() -> None:
     # Load Observing System Generation tests
     test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(GenerateObservingSystemTest))
 
+    # Load Pinned Versions Test
+    test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(PinnedVersionsTest))
+
     # Create a test runner
     test_runner = unittest.TextTestRunner()
 

src/swell/test/code_tests/test_generate_observing_system.py

@@ -4,6 +4,7 @@
 from datetime import datetime as dt
 from swell.utilities.logger import Logger
 from swell.utilities.get_channels import get_channels
+from swell.test.code_tests.testing_utilities import suppress_stdout
 from swell.utilities.observing_system_records import ObservingSystemRecords
 
 
@@ -43,7 +44,7 @@ def test_geos_mksi_main(self):
         abort_message = "\nHERE IS THE TRACEBACK: \n----------------------\n\n" + \
                         "Missing active channels for cris-fsr_npp, " + \
                         "Confirm that you are using the right version of GEOSmksi"
-        with self.assertRaises(SystemExit) as abort:
+        with self.assertRaises(SystemExit) as abort, suppress_stdout():
             get_channels(self.observing_system_records_path, observations[0],
                          self.dt_cycle_time, self.logger)
             self.assertEqual(abort.exception, abort_message)

src/swell/test/code_tests/test_pinned_versions.py

@@ -0,0 +1,29 @@
+import os
+import unittest
+import subprocess
+from swell.utilities.logger import Logger
+from swell.test.code_tests.testing_utilities import suppress_stdout
+from swell.utilities.pinned_versions.check_hashes import check_hashes
+
+
+class PinnedVersionsTest(unittest.TestCase):
+
+    def test_wrong_hash(self) -> None:
+        logger = Logger("PinnedVersionsTest")
+        jedi_bundle_dir = "jedi_bundle/"
+        if not os.path.exists(jedi_bundle_dir):
+            os.makedirs(jedi_bundle_dir)
+
+        # Clone oops repository in jedi_bundle (develop hash)
+        if not os.path.exists(jedi_bundle_dir + "oops"):
+            cmd = ["git", "clone", "https://github.com/JCSDA/oops.git"]
+        else:
+            cmd = ["git", "checkout", "develop"]
+
+        subprocess.run(cmd, cwd=jedi_bundle_dir, stderr=subprocess.DEVNULL,
+                       stdout=subprocess.DEVNULL)
+        abort_message = "Wrong commit hashes found for these repositories in jedi_bundle: [oops]"
+        # Run check hash (expect abort)
+        with self.assertRaises(SystemExit) as abort, suppress_stdout():
+            check_hashes(jedi_bundle_dir, logger)
+            self.assertEqual(abort.exception, abort_message)

src/swell/test/code_tests/testing_utilities.py

@@ -0,0 +1,11 @@
+import io
+import sys
+import contextlib
+
+
+@contextlib.contextmanager
+def suppress_stdout():
+    tmp_stdout = sys.stdout
+    sys.stdout = io.StringIO()
+    yield
+    sys.stdout = tmp_stdout

src/swell/utilities/pinned_versions/__init__.py

@@ -0,0 +1,9 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+import os
+
+repo_directory = os.path.dirname(__file__)

src/swell/utilities/pinned_versions/check_hashes.py

@@ -0,0 +1,43 @@
+import os
+import yaml
+import subprocess
+from pathlib import Path
+import importlib.resources
+from swell.utilities.logger import Logger
+
+
+def get_pinned_vers_path() -> Path:
+    return importlib.resources.files("swell.utilities.pinned_versions") / "pinned_versions.yaml"
+
+
+def check_hashes(jedi_bundle_loc: str, logger: Logger) -> None:
+
+    # Get list of directories in jedi_bundle_loc
+    dirs = os.listdir(jedi_bundle_loc)
+
+    pinned_vers_path = get_pinned_vers_path()
+    # Loaded pinned_versions into dict
+    with open(pinned_vers_path) as stream:
+        pinned_vers = yaml.safe_load(stream)
+
+    incorrect_hash = []
+    for repo_dict in pinned_vers:
+        repo_name = next(iter(repo_dict))
+        expected_hash = repo_dict[repo_name]["branch"]
+
+        # Get hash or branch of repo in jedi bundle
+        if repo_name in dirs:
+            cmd = ["git", "rev-parse", "HEAD"]
+            with subprocess.Popen(cmd, cwd=jedi_bundle_loc+repo_name,
+                                  stdout=subprocess.PIPE) as proc:
+                curr_hash = proc.stdout.read()
+                proc.kill()
+
+            curr_hash = curr_hash.decode("utf-8").rstrip()
+            if expected_hash != curr_hash:
+                incorrect_hash.append(repo_name)
+
+    # If there are incorrect hashes, logger abort
+    if incorrect_hash:
+        logger.abort("Wrong commit hashes found for these repositories "
+                     f"in jedi_bundle: {incorrect_hash}")

src/swell/utilities/pinned_versions/pinned_versions.yaml

@@ -0,0 +1,52 @@
+# Pinned versions for 2024-07-30
+- jedicmake:
+    branch: 40d521f9a2d796fcbc6234d77abceeffefb8eb7f
+    commit: true
+- oops:
+    branch: e6485c0a659103f0daa2b7e2cece39a15bfb0d60
+    commit: true
+- saber:
+    branch: e93b14ff97acc70cdbb6bdd57620da2cc81c5eb0
+    commit: true
+- ioda:
+    branch: c7b8760fc187a9eafb72a466d16fdee284912377
+    commit: true
+- ufo:
+    branch: cd66505007b1559d79cb158bd6dc018a3943c1e7
+    commit: true
+- vader:
+    branch: 36dac46ac170ef8f80c2f147d0a8db7bfd9aba3c
+    commit: true
+- fv3:
+    branch: ab25dc09d955271f34ca6a3fa83af1093c85d9f7
+    commit: true
+- fv3-jedi-lm:
+    branch: a6e97d76ed7c0b2a27cf97512893a93d7e2b44bc
+    commit: true
+- femps:
+    branch: 4f12677d345e683bf910b5f76f0df120ad27482d
+    commit: true
+- fv3-jedi:
+    branch: d3c800b82eef8d1eaadcbc07c3cf7a96d8425233
+    commit: true
+- ioda-data:
+    branch: a8c2bb3265ee2ab222706d606d22282630810750
+    commit: true
+- fv3-jedi-data:
+    branch: dd0524e339250c0b80858812f2d85cceedbf07ee
+    commit: true
+- soca:
+    branch: 92519ab72b89a4c3b802501e71b7b66349fc8cc8
+    commit: true
+- iodaconv:
+    branch: e8235068c663029b1f2cb4abf4e86f476c2e6af0
+    commit: true
+- gsw:
+    branch: 697cbeb7605d70ed3857664c5f54a5c05346e31f
+    commit: true
+- ufo-data:
+    branch: 70464f6da0c306828d820004375d151c33c7c53d
+    commit: true
+- icepack:
+    branch: 73136ee8dcdbe378821e540488a5980a03d8abe6
+    commit: true