From 1c77e3a4e434f0afb73949e2051cdb0714a9b1ac Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 21 Oct 2022 15:46:43 -0600 Subject: [PATCH 1/8] Pretty-print cache file, and add beginning of a cache unittest script. --- cime_config/cam_build_cache.py | 18 ++- .../build_cache_files/example_build_cache.xml | 23 +++ test/unit/test_build_cache.py | 136 ++++++++++++++++++ 3 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 test/unit/sample_files/build_cache_files/example_build_cache.xml create mode 100644 test/unit/test_build_cache.py diff --git a/cime_config/cam_build_cache.py b/cime_config/cam_build_cache.py index e9b3fc8d..6efa2fa7 100644 --- a/cime_config/cam_build_cache.py +++ b/cime_config/cam_build_cache.py @@ -13,14 +13,15 @@ import os import hashlib import xml.etree.ElementTree as ET +from xml.dom import minidom #Used to pretty-print cache file # Find and include the ccpp-framework scripts directory # Assume we are in /cime_config and SPIN is in /ccpp_framework __CURRDIR = os.path.abspath(os.path.dirname(__file__)) __CAMROOT = os.path.abspath(os.path.join(__CURRDIR, os.pardir)) -__SPINSCRIPTS = os.path.join(__CAMROOT, "ccpp_framework", 'scripts') -if __SPINSCRIPTS not in sys.path: - sys.path.append(__SPINSCRIPTS) +__CCPP_FRAMEWORK = os.path.join(__CAMROOT, "ccpp_framework", 'scripts') +if __CCPP_FRAMEWORK not in sys.path: + sys.path.append(__CCPP_FRAMEWORK) # end if # CCPP framework imports @@ -386,8 +387,17 @@ def write(self): kind_type = ET.SubElement(ccpp, 'kind_type') kind_type.text = f"{kind_def}={kind_type}" # end for + + #Combine elments into an Element Tree object: new_cache_tree = ET.ElementTree(new_cache) - new_cache_tree.write(self.__build_cache) + + #Convert Element Tree to a Document Object Model (DOM) XML object: + dom_xml = minidom.parseString(ET.tostring(new_cache_tree.getroot())) + + #Write XML in a "pretty-print" format: + with open(self.__build_cache, "w", encoding="UTF-8") as xml_file: + xml_file.write(dom_xml.toprettyxml(indent=" ")) + #End with def registry_mismatch(self, gen_reg_file, registry_source_files, dycore, config): diff --git a/test/unit/sample_files/build_cache_files/example_build_cache.xml b/test/unit/sample_files/build_cache_files/example_build_cache.xml new file mode 100644 index 00000000..e6d89565 --- /dev/null +++ b/test/unit/sample_files/build_cache_files/example_build_cache.xml @@ -0,0 +1,23 @@ + + + + + + + none + + /yellow/brick/road/munchkin.meta + brain + heart + + + + + + + + toto dog + UNSET + kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + + diff --git a/test/unit/test_build_cache.py b/test/unit/test_build_cache.py new file mode 100644 index 00000000..cce2ee60 --- /dev/null +++ b/test/unit/test_build_cache.py @@ -0,0 +1,136 @@ +""" +Python unit testing collection for the various +cam_build_cache.py functions, including their +error-handling processes. Please note that +these tests will only work with Python 3.7 +or later. + +To run these unit tests, simply type: + +python test_build_cache.py + +or (for more verbose output): + +python test_build_cache.py -v + +which will currently run XX tests, all of which should pass. +""" + +#---------------------------------------- +#Import required python libraries/modules: +#---------------------------------------- +import sys +import os.path + +#Python unit-testing library: +import unittest + +#Add directory to python path: +_CURRDIR = os.path.abspath(os.path.dirname(__file__)) +_CAM_ROOT_DIR = os.path.join(_CURRDIR, os.pardir, os.pardir) +_CAM_CONF_DIR = os.path.abspath(os.path.join(_CAM_ROOT_DIR, "cime_config")) +_PRE_TMP_DIR = os.path.join(_CURRDIR, "tmp") +_TMP_DIR = os.path.join(_PRE_TMP_DIR, "cam_build_cache") +_SAMPLES_DIR = os.path.join(_CURRDIR, "sample_files", "build_cache_files") + +#Check for all necessary directories: +if not os.path.exists(_CAM_CONF_DIR): + _EMSG = f"ERROR: Cannot find 'cime_config' directory in '{_CAM_CONF_DIR}'" + raise ImportError(_EMSG) +#End if + +#Add "cime_config" directory to python path: +sys.path.append(_CAM_CONF_DIR) + +#Import CAM Build Cache object: +# pylint: disable=wrong-import-position +from cam_build_cache import BuildCacheCAM +# pylint: enable=wrong-import-position + +#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +#Main cam_build_cache testing routine, used when script is run directly +#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +class CamBuildCacheTestRoutine(unittest.TestCase): + + """ + Runs all CAM build cache (cam_build_cache) tests, + to ensure that the scripts and error-handling methods + are running properly. + """ + + #+++++++++++++++++++++++++++++++++++ + #BuildCacheCAM object creation tests + #+++++++++++++++++++++++++++++++++++ + + def test_build_cache_cam_no_file(self): + + """ + Check that the BuildCacheCAM object is + created successfully when the cache + file does not exist. + """ + + #Create non-existent cache file path: + cache_file = os.path.join(_TMP_DIR, "empty_test_cache.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Check that outputs match expected defaults: + self.assertEqual(test_cache.scheme_nl_metadata(), []) + self.assertEqual(test_cache.scheme_nl_groups(), []) + self.assertEqual(test_cache.reg_file_list(), []) + self.assertEqual(test_cache.ic_names(), {}) + + #++++++++++++++++ + + def test_build_cache_cam_file(self): + + """ + Check that the BuildCacheCAM object is + create successfully when an already + existing cache file is used. + """ + + #Set expected outputs: + nl_meta_list = ['/not/in/kansas/wizard_nl.meta'] + nl_group_list = ['toto', 'dog'] + reg_file_list = ['/yellow/brick/road/munchkin.meta'] + ic_names_dict = {'Only_had_a': ['brain', 'heart']} + + #Set path to already-existing cache file: + cache_file = os.path.join(_SAMPLES_DIR, "example_build_cache.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Check that outputs match values listed in cache file: + self.assertEqual(test_cache.scheme_nl_metadata(), nl_meta_list) + self.assertEqual(test_cache.scheme_nl_groups(), nl_group_list) + self.assertEqual(test_cache.reg_file_list(), reg_file_list) + self.assertEqual(test_cache.ic_names(), ic_names_dict) + + #+++++++++++++++++++++++++ + #Registry generation tests + #+++++++++++++++++++++++++ + + #++++++++++++++++++++++++++++++++++++ + #CCPP Physics suites generation tests + #++++++++++++++++++++++++++++++++++++ + + + #++++++++++++++++++++++++++++++++++++++++ + #Initialization routines generation tests + #++++++++++++++++++++++++++++++++++++++++ + +################################################# +#Run unit tests if this script is called directly +################################################# + +if __name__ == "__main__": + unittest.main() + +############ +#End of file +############ From 168cd03e58333359e9da395cd82600a36caf3ff7 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 23 Dec 2022 15:18:26 -0700 Subject: [PATCH 2/8] Convert build_cache doctests to unittest tests, and remove extra cam_build_cache test variables. --- cime_config/cam_build_cache.py | 215 -------- .../bad_ccpp_tag_build_cache.xml | 24 + .../bad_reg_tag_build_cache.xml | 24 + .../bad_section_tag_build_cache.xml | 26 + .../write_init_files/suite_bad.xml | 13 - test/unit/test_build_cache.py | 494 +++++++++++++++++- test/unit/test_cam_autogen.py | 2 +- 7 files changed, 568 insertions(+), 230 deletions(-) create mode 100644 test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml create mode 100644 test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml create mode 100644 test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml delete mode 100644 test/unit/sample_files/write_init_files/suite_bad.xml diff --git a/cime_config/cam_build_cache.py b/cime_config/cam_build_cache.py index 6efa2fa7..6e1d5ec0 100644 --- a/cime_config/cam_build_cache.py +++ b/cime_config/cam_build_cache.py @@ -129,42 +129,6 @@ class BuildCacheCAM: metadata and source code files for all schemes specified in the SDFs. Any host model metadata file changes will also trigger a CCPP rebuild so a registry-file creation implies a CCPP Framework run. - - doctests - - 1. Check that the proper error is generated when wrong file is input: - - >>> BuildCacheCAM(TEST_SCHEME) - Traceback (most recent call last): - ... - parse_source.CCPPError: read_xml_file: Cannot read ...temp_adjust_scalar.meta, syntax error: line 1, column 0 - - 2. Check that the proper error is generated when build_cache has an invalid tag: - - >>> BuildCacheCAM(BAD_BUILD_CACHE) - Traceback (most recent call last): - ... - ValueError: ERROR: Unknown section tag, 'test' - - 3. Check that the proper error is generated when build_cache has an invalid registry tag: - - >>> BuildCacheCAM(BAD_BUILD_CACHE_REG) - Traceback (most recent call last): - ... - ValueError: ERROR: Unknown registry tag, 'test' - - 4. Check that the proper error is generated when build_cache has an invalid ccpp tag: - - >>> BuildCacheCAM(BAD_BUILD_CACHE_CCPP) - Traceback (most recent call last): - ... - ValueError: ERROR: Unknown CCPP tag, 'test' - - 5. Check that parsing works (no errors) when input is valid: - - >>> BuildCacheCAM(BUILD_CACHE)._BuildCacheCAM__dycore - 'none' - """ def __init__(self, build_cache): @@ -404,34 +368,6 @@ def registry_mismatch(self, gen_reg_file, registry_source_files, """ Determine if the registry input data differs from the data stored in our cache. Return True if the data differs. - - doctests - - 1. Check that the function returns False when there are no changes: - - >>> BUILD_CACHE_CAM.registry_mismatch(REGISTRY_FILE, [REGISTRY_FILE], NULL_DYCORE, NONE_CONFIG) - False - - 2. Check that the function returns True when the dycore has changed: - - >>> BUILD_CACHE_CAM.registry_mismatch(REGISTRY_FILE, [REGISTRY_FILE], SE_DYCORE, NONE_CONFIG) - True - - 3. Check that the function returns True when the registry has been updated: - - >>> BUILD_CACHE_CAM.registry_mismatch(REGISTRY_FILE, [TEST_SDF], NULL_DYCORE, NONE_CONFIG) - True - - 4. Check that the function returns True when the gen_reg_file has been updated: - - >>> BUILD_CACHE_CAM.registry_mismatch(TEST_SDF, [REGISTRY_FILE], NULL_DYCORE, NONE_CONFIG) - True - - 5. Check that the function returns True when config changes: - - >>> BUILD_CACHE_CAM.registry_mismatch(REGISTRY_FILE, [REGISTRY_FILE], NULL_DYCORE, TEST_CHANGE) - True - """ mismatch = False mismatch = (not self.__dycore) or (self.__dycore != dycore) @@ -462,34 +398,6 @@ def ccpp_mismatch(self, sdfs, scheme_files, host_files, """ Determine if the CCPP input data differs from the data stored in our cache. Return True if the data differs. - - doctests - - 1. Check that the function returns False when no changes were made: - - >>> BUILD_CACHE_CAM.ccpp_mismatch([TEST_SDF], [TEST_SCHEME], [], PREPROC_DEFS, KIND_PHYS) - False - - 2. Check that the function returns True when the preproc_defs changes: - - >>> BUILD_CACHE_CAM.ccpp_mismatch([TEST_SDF], [TEST_SCHEME], [], TEST_CHANGE, KIND_PHYS) - True - - 3. Check that the function returns True when kind_phys changes: - - >>> BUILD_CACHE_CAM.ccpp_mismatch([TEST_SDF], [TEST_SCHEME], [], PREPROC_DEFS, KPHYS_CHANGE) - True - - 4. Check that the function returns True when an SDF changes: - - >>> BUILD_CACHE_CAM.ccpp_mismatch([REGISTRY_FILE], [TEST_SCHEME], [], PREPROC_DEFS, KIND_PHYS) - True - - 5. Check that the function returns True when a scheme changes: - - >>> BUILD_CACHE_CAM.ccpp_mismatch([TEST_SDF], [REGISTRY_FILE], [], PREPROC_DEFS, KIND_PHYS) - True - """ mismatch = ((not self.__preproc_defs) or (self.__preproc_defs != preproc_defs)) @@ -587,19 +495,6 @@ def init_write_mismatch(self, gen_init_file): Determine if the init_files writer (write_init_files.py) differs from the data stored in our cache. Return True if the data differs. - - doctests - - 1. Check that the function returns False when nothing has changed: - - >>> BUILD_CACHE_CAM.init_write_mismatch(REGISTRY_FILE) - False - - 2. Check that the function returns True when the file has changed: - - >>> BUILD_CACHE_CAM.init_write_mismatch(TEST_SDF) - True - """ #Initialize variable: @@ -629,115 +524,5 @@ def ic_names(self): """Return a copy of the registry initial conditions dictionary""" return dict(self.__ic_names) -############################################################################### -# IGNORE EVERYTHING BELOW HERE UNLESS RUNNING TESTS ON CAM_BUILD_CACHE! -############################################################################### - -# Call testing routine, if script is run directly -if __name__ == "__main__": - - # Import modules needed for testing: - import doctest - import logging - import shutil - - _LOGGER = logging.getLogger(__name__) - - #++++++++++++++++++++++++++++++++++++++++++ - # Determine current working directory: - TEST_AUTO_DIR = os.path.dirname(os.path.abspath(__file__)) - TEST_ATM_ROOT = os.path.abspath(os.path.join(TEST_AUTO_DIR, os.pardir)) - - TEST_SOURCE_MODS_DIR = os.path.join(TEST_ATM_ROOT, "SourceMods") - - # Remove old test directories if they exist: - if os.path.exists(TEST_SOURCE_MODS_DIR): - shutil.rmtree(TEST_SOURCE_MODS_DIR) - - # Create variables for testing: - NULL_DYCORE = 'none' - SE_DYCORE = 'se' - NONE_CONFIG = None - KIND_PHYS = ["kind_phys = REAL64"] - KPHYS_CHANGE = ["kind_phys = REAL32"] - PREPROC_DEFS = "UNSET" - TEST_CHANGE = "TEST" - - # Create "SourceMods directory: - os.mkdir(TEST_SOURCE_MODS_DIR) - - # Set logger to fatal, to avoid log messages: - _LOGGER.setLevel(logging.FATAL) - - # Set test CCPP suite paths: - SUITE_TEST_PATH = os.path.join(TEST_ATM_ROOT, "test", "unit", "sample_files", - "write_init_files") - TEST_SDF = os.path.join(SUITE_TEST_PATH, "suite_simple.xml") - TEST_SCHEME = os.path.join(SUITE_TEST_PATH, "temp_adjust_scalar.meta") - BUILD_CACHE = os.path.join(SUITE_TEST_PATH, "simple_build_cache_template.xml") - REGISTRY_FILE = os.path.join(SUITE_TEST_PATH, "simple_reg.xml") - - # Copy test CCPP suite into SourceMods directory: - shutil.copy2(TEST_SDF, TEST_SOURCE_MODS_DIR) - shutil.copy2(BUILD_CACHE, os.path.join(TEST_SOURCE_MODS_DIR, "simple_build_cache.xml")) - shutil.copy2(BUILD_CACHE, os.path.join(TEST_SOURCE_MODS_DIR, "bad_simple_build_cache.xml")) - shutil.copy2(BUILD_CACHE, os.path.join(TEST_SOURCE_MODS_DIR, "bad_simple_build_cache_reg.xml")) - shutil.copy2(BUILD_CACHE, os.path.join(TEST_SOURCE_MODS_DIR, "bad_simple_build_cache_ccpp.xml")) - shutil.copy2(REGISTRY_FILE, TEST_SOURCE_MODS_DIR) - shutil.copy2(TEST_SCHEME, TEST_SOURCE_MODS_DIR) - - REGISTRY_FILE = os.path.join(TEST_SOURCE_MODS_DIR, "simple_reg.xml") - BUILD_CACHE = os.path.join(TEST_SOURCE_MODS_DIR, "simple_build_cache.xml") - BAD_BUILD_CACHE = os.path.join(TEST_SOURCE_MODS_DIR, "bad_simple_build_cache.xml") - BAD_BUILD_CACHE_REG = os.path.join(TEST_SOURCE_MODS_DIR, "bad_simple_build_cache_reg.xml") - BAD_BUILD_CACHE_CCPP = os.path.join(TEST_SOURCE_MODS_DIR, "bad_simple_build_cache_ccpp.xml") - TEST_SDF = os.path.join(TEST_SOURCE_MODS_DIR, "suite_simple.xml") - TEST_SCHEME = os.path.join(TEST_SOURCE_MODS_DIR, "temp_adjust_scalar.meta") - - # Generate test build caches from template: - f1 = open(BUILD_CACHE, 'rt', encoding='utf-8') - data = f1.read() - data = data.replace("TAG1", "").replace("TAG2", "").replace("TAG3", "") - f1.close() - f1 = open(BUILD_CACHE, 'w', encoding='utf-8') - f1.write(data) - f1.close() - - f1 = open(BAD_BUILD_CACHE, 'rt', encoding='utf-8') - data = f1.read() - data = data.replace("TAG1", "").replace("TAG2", "").replace("TAG3", "") - f1.close() - f1 = open(BAD_BUILD_CACHE, 'w', encoding='utf-8') - f1.write(data) - f1.close() - - f1 = open(BAD_BUILD_CACHE_REG, 'rt', encoding='utf-8') - data = f1.read() - data = data.replace("TAG1", "").replace("TAG2", "").replace("TAG3", "") - f1.close() - f1 = open(BAD_BUILD_CACHE_REG, 'w', encoding='utf-8') - f1.write(data) - f1.close() - - f1 = open(BAD_BUILD_CACHE_CCPP, 'rt', encoding='utf-8') - data = f1.read() - data = data.replace("TAG1", "").replace("TAG2", "").replace("TAG3", "") - f1.close() - f1 = open(BAD_BUILD_CACHE_CCPP, 'w', encoding='utf-8') - f1.write(data) - f1.close() - - BUILD_CACHE_CAM = BuildCacheCAM(BUILD_CACHE) - - # Run doctests: - OPTIONS = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE - TEST_SUCCESS = doctest.testmod(optionflags=OPTIONS)[0] - - # Remove testing directories: - shutil.rmtree(TEST_SOURCE_MODS_DIR) - - # Exit script with error code matching number of failed tests: - sys.exit(TEST_SUCCESS) - ############# # End of file diff --git a/test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml b/test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml new file mode 100644 index 00000000..fdb51432 --- /dev/null +++ b/test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml @@ -0,0 +1,24 @@ + + + + + + + none + + /yellow/brick/road/munchkin.meta + brain + heart + + + + + + + + toto dog + UNSET + kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + of the North + + diff --git a/test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml b/test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml new file mode 100644 index 00000000..9330a5bd --- /dev/null +++ b/test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml @@ -0,0 +1,24 @@ + + + + + + + none + + /yellow/brick/road/munchkin.meta + brain + heart + Oz + + + + + + + + toto dog + UNSET + kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + + diff --git a/test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml b/test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml new file mode 100644 index 00000000..619939e3 --- /dev/null +++ b/test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml @@ -0,0 +1,26 @@ + + + + + + + none + + /yellow/brick/road/munchkin.meta + brain + heart + + + + + + + + toto dog + UNSET + kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + + + + + diff --git a/test/unit/sample_files/write_init_files/suite_bad.xml b/test/unit/sample_files/write_init_files/suite_bad.xml deleted file mode 100644 index 447b131b..00000000 --- a/test/unit/sample_files/write_init_files/suite_bad.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - bad_scheme - - diff --git a/test/unit/test_build_cache.py b/test/unit/test_build_cache.py index cce2ee60..010cffde 100644 --- a/test/unit/test_build_cache.py +++ b/test/unit/test_build_cache.py @@ -20,6 +20,7 @@ #Import required python libraries/modules: #---------------------------------------- import sys +import os import os.path #Python unit-testing library: @@ -32,19 +33,31 @@ _PRE_TMP_DIR = os.path.join(_CURRDIR, "tmp") _TMP_DIR = os.path.join(_PRE_TMP_DIR, "cam_build_cache") _SAMPLES_DIR = os.path.join(_CURRDIR, "sample_files", "build_cache_files") +_WRITE_INIT_DIR = os.path.join(_CURRDIR, "sample_files", "write_init_files") +_CCPP_FRAMEWORK = os.path.join(_CAM_ROOT_DIR, "ccpp_framework", 'scripts') #Check for all necessary directories: if not os.path.exists(_CAM_CONF_DIR): - _EMSG = f"ERROR: Cannot find 'cime_config' directory in '{_CAM_CONF_DIR}'" + _EMSG = f"ERROR: Cannot find required '{_CAM_ROOT_DIR}' directory" + raise ImportError(_EMSG) +#End if +if not os.path.exists(_CCPP_FRAMEWORK): + _EMSG = f"ERROR: Cannot find CCPP-framework routines in '{_CCPP_FRAMEWORK}'" raise ImportError(_EMSG) #End if #Add "cime_config" directory to python path: sys.path.append(_CAM_CONF_DIR) +#Add "ccpp_framework" scripts directory to python path: +sys.path.append(_CCPP_FRAMEWORK) + #Import CAM Build Cache object: # pylint: disable=wrong-import-position from cam_build_cache import BuildCacheCAM + +#Import CCPP Error type: +from parse_source import CCPPError # pylint: enable=wrong-import-position #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -111,19 +124,498 @@ def test_build_cache_cam_file(self): self.assertEqual(test_cache.reg_file_list(), reg_file_list) self.assertEqual(test_cache.ic_names(), ic_names_dict) + #++++++++++++++++ + + def test_build_cache_wrong_file_type(self): + + """ + Check that the correct error is raised when + the BuildCacheCAM object is initialized with + the wrong file type. + """ + + #Set path to "bad" file: + non_xml_file = os.path.join(_SAMPLES_DIR, os.pardir, "ref_pres.meta") + + #Set expected error message: + emsg = f"read_xml_file: Cannot read {non_xml_file}, syntax error: line 1, column 0" + + #Expect CCPP error when initializing object: + with self.assertRaises(CCPPError) as cerr: + _ = BuildCacheCAM(non_xml_file) + #End with + + #Check that error message matches what is expected: + self.assertEqual(emsg, str(cerr.exception)) + + def test_build_cache_wrong_section_tag(self): + + """ + Check that the correct error is raised when + the BuildCacheCAM object is initialized with a + cache file that contains an unkown section tag. + """ + + #Set path to cache file with "bad" section tag: + bad_section_tag_file = os.path.join(_SAMPLES_DIR, "bad_section_tag_build_cache.xml") + + #Set expected error message: + emsg = "ERROR: Unknown section tag, 'Wicked_Witch'" + + #Expect Value error when initializing object: + with self.assertRaises(ValueError) as verr: + _ = BuildCacheCAM(bad_section_tag_file) + #End with + + #Check that error message matches what is expected: + self.assertEqual(emsg, str(verr.exception)) + + #++++++++++++++++ + + def test_build_cache_wrong_reg_tag(self): + + """ + Check that the correct error is raised when + the BuildCacheCAM object is initialized with + a cache file that has a bad registry tag. + """ + + #Set path to cache file with "bad" registry tag: + bad_reg_tag_file = os.path.join(_SAMPLES_DIR, "bad_reg_tag_build_cache.xml") + + #Set expected error message: + emsg = "ERROR: Unknown registry tag, 'Wizard'" + + #Expect Value error when initializing object: + with self.assertRaises(ValueError) as verr: + _ = BuildCacheCAM(bad_reg_tag_file) + #End with + + #Check that error message matches what is expected: + self.assertEqual(emsg, str(verr.exception)) + + #++++++++++++++++ + + def test_build_cache_wrong_ccpp_tag(self): + + """ + Check that the correct error is raised when + the BuildCacheCAM object is initialized with + a cache file that has a bad CCPP tag. + """ + + #Set path to cache file with "bad" registry tag: + bad_ccpp_tag_file = os.path.join(_SAMPLES_DIR, "bad_ccpp_tag_build_cache.xml") + + #Set expected error message: + emsg = "ERROR: Unknown CCPP tag, 'Good_Witch'" + + #Expect Value error when initializing object: + with self.assertRaises(ValueError) as verr: + _ = BuildCacheCAM(bad_ccpp_tag_file) + #End with + + #Check that error message matches what is expected: + self.assertEqual(emsg, str(verr.exception)) + #+++++++++++++++++++++++++ #Registry generation tests #+++++++++++++++++++++++++ + def test_registry_mismatch_good_match(self): + + """ + Check that the 'registry_mismatch' + function returns False when there + is no change in the registry. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to registry generator file listed in build_cache file. + #Please note that in this sample file the registry XML file is listed, + #and not a python file as would normally be the case: + reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + reg_match = test_cache.registry_mismatch(reg_file, [reg_file], "none", None) + + #Check that function returns False: + self.assertFalse(reg_match) + + #++++++++++++++++ + + def test_registry_mismatch_diff_dycore(self): + + """ + Check that the 'registry_mismatch' + function returns True when there is + a change in the dycore being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to registry generator file listed in build_cache file. + #Please note that in this sample file the registry XML file is listed, + #and not a python file as would normally be the case: + reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Set "new" dycore value: + new_dycore = "se" + + #Run registry_mismatch function: + reg_match = test_cache.registry_mismatch(reg_file, [reg_file], new_dycore, None) + + #Check that function returns True: + self.assertTrue(reg_match) + + #++++++++++++++++ + + def test_registry_mismatch_diff_reg_file(self): + + """ + Check that the 'registry_mismatch' + function returns True when there is + a change in the registry file being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to registry generator file listed in build_cache file. + #Please note that in this sample file the registry XML file is listed, + #and not a python file as would normally be the case: + reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Set path to "new" registry file: + new_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + reg_match = test_cache.registry_mismatch(reg_file, [new_file], "none", None) + + #Check that function returns True: + self.assertTrue(reg_match) + + #++++++++++++++++ + + def test_registry_mismatch_diff_gen_file(self): + + """ + Check that the 'registry_mismatch' + function returns True when there is a + change in the registry generator being + used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to registry generator file listed in build_cache file. + #Please note that in this sample file the registry XML file is listed, + #and not a python file as would normally be the case: + reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Set path to "new" registry file: + new_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + reg_match = test_cache.registry_mismatch(new_file, [reg_file], "none", None) + + #Check that function returns True: + self.assertTrue(reg_match) + + #++++++++++++++++ + + def test_registry_mismatch_diff_config(self): + + """ + Check that the 'registry_mismatch' + function returns True when the there is + a change in the registry config options + being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to registry generator file listed in build_cache file. + #Please note that in this sample file the registry XML file is listed, + #and not a python file as would normally be the case: + reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + reg_match = test_cache.registry_mismatch(reg_file, [reg_file], "none", "banana") + + #Check that function returns True: + self.assertTrue(reg_match) + #++++++++++++++++++++++++++++++++++++ #CCPP Physics suites generation tests #++++++++++++++++++++++++++++++++++++ + def test_ccpp_mismatch_good_match(self): + + """ + Check that the 'ccpp_mismatch' + function returns False when there + is no change to the CCPP framework + or physics schemes. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to Suite Definition File (SDF) listed in build_cache file: + sdf_file = os.path.join(_WRITE_INIT_DIR, "suite_simple.xml") + + #Set path to physics scheme meta file: + scheme_meta_file = os.path.join(_WRITE_INIT_DIR, "temp_adjust_scalar.meta") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + ccpp_match = test_cache.ccpp_mismatch([sdf_file], [scheme_meta_file], [], + "UNSET", ["kind_phys = REAL64"]) + + #Check that function returns False: + self.assertFalse(ccpp_match) + + #++++++++++++++++ + + def test_ccpp_mismatch_diff_preproc(self): + + """ + Check that the 'ccpp_mismatch' + function returns True when there + is a change in the pre-processor + definitions being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to Suite Definition File (SDF) listed in build_cache file: + sdf_file = os.path.join(_WRITE_INIT_DIR, "suite_simple.xml") + + #Set path to physics scheme meta file: + scheme_meta_file = os.path.join(_WRITE_INIT_DIR, "temp_adjust_scalar.meta") + + #Set "new" pre-processor definition: + preproc_def = "BANANA" + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + ccpp_match = test_cache.ccpp_mismatch([sdf_file], [scheme_meta_file], [], + preproc_def, ["kind_phys = REAL64"]) + + #Check that function returns True: + self.assertTrue(ccpp_match) + + #++++++++++++++++ + + def test_ccpp_mismatch_diff_kind(self): + + """ + Check that the 'ccpp_mismatch' + function returns True when there + is a change in the kind types + being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to Suite Definition File (SDF) listed in build_cache file: + sdf_file = os.path.join(_WRITE_INIT_DIR, "suite_simple.xml") + + #Set path to physics scheme meta file: + scheme_meta_file = os.path.join(_WRITE_INIT_DIR, "temp_adjust_scalar.meta") + + #Set "new" physics kind value: + new_kind_phys_def = ["kind_phys = REAL32"] + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + ccpp_match = test_cache.ccpp_mismatch([sdf_file], [scheme_meta_file], [], + "UNSET", new_kind_phys_def) + + #Check that function returns True: + self.assertTrue(ccpp_match) + + #++++++++++++++++ + + def test_ccpp_mismatch_diff_sdf(self): + + """ + Check that the 'ccpp_mismatch' + function returns True when there + is a change in the SDF being + used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to "new" Suite Definition File (SDF), which in this case is actually + #just a registry file: + new_sdf_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Set path to physics scheme meta file: + scheme_meta_file = os.path.join(_WRITE_INIT_DIR, "temp_adjust_scalar.meta") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + ccpp_match = test_cache.ccpp_mismatch([new_sdf_file], [scheme_meta_file], [], + "UNSET", ["kind_phys = REAL64"]) + + #Check that function returns True: + self.assertTrue(ccpp_match) + + #++++++++++++++++ + + def test_ccpp_mismatch_diff_scheme(self): + + """ + Check that the 'ccpp_mismatch' + function returns True when there + is a change in the scheme being + used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to Suite Definition File (SDF) listed in build_cache file: + sdf_file = os.path.join(_WRITE_INIT_DIR, "suite_simple.xml") + + #Set path to "new" physics scheme meta file: + new_scheme_meta_file = os.path.join(_WRITE_INIT_DIR, "temp_adjust_param.meta") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + ccpp_match = test_cache.ccpp_mismatch([sdf_file], [new_scheme_meta_file], [], + "UNSET", ["kind_phys = REAL64"]) + + #Check that function returns True: + self.assertTrue(ccpp_match) + + #++++++++++++++++ + + def test_ccpp_mismatch_diff_host(self): + + """ + Check that the 'ccpp_mismatch' + function returns True when there + is a change in the host files + being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to Suite Definition File (SDF) listed in build_cache file: + sdf_file = os.path.join(_WRITE_INIT_DIR, "suite_simple.xml") + + #Set path to physics scheme meta file: + scheme_meta_file = os.path.join(_WRITE_INIT_DIR, "temp_adjust_scalar.meta") + + #Set path to "new" host model meta file: + new_host_file = os.path.join(_WRITE_INIT_DIR, "simple_host.meta") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + ccpp_match = test_cache.ccpp_mismatch([sdf_file], [scheme_meta_file], + [new_host_file], "UNSET", + ["kind_phys = REAL64"]) + + #Check that function returns True: + self.assertTrue(ccpp_match) #++++++++++++++++++++++++++++++++++++++++ #Initialization routines generation tests #++++++++++++++++++++++++++++++++++++++++ + def test_init_write_mismatch_good_match(self): + + """ + Check that the 'init_write_mismatch' + function returns False when there + is no change in the init files writing + function being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to initialization files generator file listed in build_cache file. + #Please note that in this sample file the registry XML file is listed, + #and not a python file as would normally be the case: + init_gen_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + init_match = test_cache.init_write_mismatch(init_gen_file) + + #Check that function returns False: + self.assertFalse(init_match) + + #++++++++++++++++ + + def test_init_write_diff_host(self): + + """ + Check that the 'init_write_mismatch' + function returns True when there + is a change in the init files writing + function being used. + """ + + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + + #Set path to "new" initialization files generator, + #which is just a different XML file in this case: + new_init_gen_file = os.path.join(_WRITE_INIT_DIR, "suite_simple.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Run registry_mismatch function: + init_match = test_cache.init_write_mismatch(new_init_gen_file) + + #Check that function returns True: + self.assertTrue(init_match) + ################################################# #Run unit tests if this script is called directly ################################################# diff --git a/test/unit/test_cam_autogen.py b/test/unit/test_cam_autogen.py index 39fea9d2..b5a59628 100644 --- a/test/unit/test_cam_autogen.py +++ b/test/unit/test_cam_autogen.py @@ -38,7 +38,7 @@ #Check for all necessary directories: if not os.path.exists(_CAM_CONF_DIR): - _EMSG = f"ERROR: Cannot find 'cime_config' directory in '{_CAM_CONF_DIR}'" + _EMSG = f"ERROR: Cannot find required '{_CAM_CONF_DIR}' directory" raise ImportError(_EMSG) #End if if not os.path.exists(_CCPP_DIR): From 6d8613a1f8ae60922a28967e4d2fa1dca5bcc017 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 23 Dec 2022 15:38:04 -0700 Subject: [PATCH 3/8] Remove need to manually add python tests to Github workflow, and update run_tests shell script. --- .github/workflows/python_unit_tests.yml | 15 ++------------- test/run_tests.sh | 4 ++-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/python_unit_tests.yml b/.github/workflows/python_unit_tests.yml index cd835a6a..9ca8a8db 100644 --- a/.github/workflows/python_unit_tests.yml +++ b/.github/workflows/python_unit_tests.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: #All of these python versions will be used to run tests: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] fail-fast: false steps: # Acquire github action routines: @@ -50,18 +50,7 @@ jobs: - name: python unit tests run: | # Run all python doctests in cime_config: - # CAM config classes doctests: - pytest cime_config/cam_config_classes.py --doctest-modules - # CAM config doctests: - pytest cime_config/cam_config.py --doctest-modules - # CAM autogen doctests: - pytest cime_config/cam_autogen.py --doctest-modules - # CAM build cache doctests: - python cime_config/cam_build_cache.py - # Namelist reader generator doctests: - pytest cime_config/create_readnl_files.py --doctest-modules - # ParamGen atm_in namelist writer doctests: - pytest cime_config/atm_in_paramgen.py --doctest-modules + pytest cime_config --doctest-modules # Run all python doctests in src/data: pytest src/data --doctest-modules diff --git a/test/run_tests.sh b/test/run_tests.sh index 29b04632..7be758b5 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -58,8 +58,6 @@ run_doctest cime_config/cam_config_classes.py run_doctest cime_config/cam_config.py # CAM autogen doctests: run_doctest cime_config/cam_autogen.py -# CAM build cache doctests: -run_doctest cime_config/cam_build_cache.py # Namelist reader generator doctests: run_doctest cime_config/create_readnl_files.py # Registry generator doctests: @@ -70,6 +68,8 @@ run_doctest cime_config/atm_in_paramgen.py run_unittest test/unit/test_cam_config.py # CAM autogen unit tests: run_unittest test/unit/test_cam_autogen.py +# CAM build_cache unit tests: +run_unittest test/unit/test_build_cache.py # Registry generator unit tests: run_unittest test/unit/test_registry.py # Namelist reader autogeneration unit tests From 4c085d681ae43d382870ba8a9cc7c73bbb813baa Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 4 Jan 2023 16:01:46 -0700 Subject: [PATCH 4/8] Increase test coverage of the build cache system. Also added extra test files, and fixed some bugs found while testing. Finally, fixed a bug found in cam_autogen which would cause the CCPP caps to be re-generated whenever a namelist definition file was found for a scheme, regardless of whether that file was different from what was used previously. --- cime_config/cam_autogen.py | 11 +- cime_config/cam_build_cache.py | 74 ++- test/run_tests.sh | 4 +- .../bad_ccpp_tag_build_cache.xml | 5 +- .../bad_reg_tag_build_cache.xml | 5 +- .../bad_section_tag_build_cache.xml | 5 +- .../build_cache_files/example_build_cache.xml | 5 +- .../update_ccpp_build_cache.xml | 24 + .../update_init_gen_build_cache.xml | 24 + .../update_reg_build_cache.xml | 24 + test/unit/test_build_cache.py | 479 +++++++++++++++++- 11 files changed, 629 insertions(+), 31 deletions(-) create mode 100644 test/unit/sample_files/build_cache_files/update_ccpp_build_cache.xml create mode 100644 test/unit/sample_files/build_cache_files/update_init_gen_build_cache.xml create mode 100644 test/unit/sample_files/build_cache_files/update_reg_build_cache.xml diff --git a/cime_config/cam_autogen.py b/cime_config/cam_autogen.py index b2b3907c..39227aa7 100644 --- a/cime_config/cam_autogen.py +++ b/cime_config/cam_autogen.py @@ -533,14 +533,19 @@ def generate_physics_suites(build_cache, preproc_defs, host_name, nl_groups = namelist_obj.groups() # include generated metadata files scheme_nl_meta_files = namelist_obj.meta_files() + # Finally, make sure the CCPP caps are grenerated + # (although this may not really be necessary): + do_gen_ccpp = True + else: - # Not running namelist generator, collect metadata files and - # namelist group names from the build cache + # Not running namelist generator, collect metadata files + # from the build cache nl_groups = build_cache.scheme_nl_groups() scheme_nl_meta_files = build_cache.scheme_nl_metadata() # end if + #Add the namelist meta files to the host files list + #if present: if scheme_nl_meta_files: - do_gen_ccpp = True host_files.extend(scheme_nl_meta_files) # end if (no else) diff --git a/cime_config/cam_build_cache.py b/cime_config/cam_build_cache.py index 6e1d5ec0..00b970a2 100644 --- a/cime_config/cam_build_cache.py +++ b/cime_config/cam_build_cache.py @@ -32,7 +32,15 @@ ############################################################################### def new_xml_entry(parent, tag, path, file_hash): ############################################################################### - """Create a new file type entry in the cache""" + """ + Create a new file type entry in the cache + + doctest: + >>> parent_xml = ET.Element("test") + >>> test_xml = new_xml_entry(parent_xml, "test", "/no/where", "banana") + >>> ET.dump(test_xml) + + """ new_entry = ET.SubElement(parent, tag) new_entry.set('file_path', path) new_entry.set('hash', file_hash) @@ -43,6 +51,38 @@ def new_entry_from_xml(item): ############################################################################### """Create a new FileStatus entry from XML entry, . ValueError is thrown in error. + + doctests: + + 1. Make sure function works as expected: + >>> parent_xml = ET.Element("test") + >>> test_xml = new_xml_entry(parent_xml, "test", "/no/where", "banana") + >>> new_entry_from_xml(test_xml).file_hash + 'banana' + + 2. Make sure function throws an error if no hash entry is found: + >>> test_xml = ET.Element("test") + >>> test_xml.set("file_path", "/no/where") + >>> new_entry_from_xml(test_xml) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + ValueError: ERROR: No hash for 'test', '/no/where' + + 3. Make sure function throws an error if no file path entry is found: + >>> test_xml = ET.Element("test") + >>> test_xml.set("hash", "banana") + >>> new_entry_from_xml(test_xml) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + ValueError: ERROR: No path for 'test' XML item + + 4. Make sure function throws an error if neither a file path or hash is found: + >>> test_xml = ET.Element("test") + >>> new_entry_from_xml(test_xml) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + ValueError: ERROR: Invalid 'test' XML item + """ path = item.get('file_path', default=None) file_hash = item.get('hash', default=None) @@ -50,13 +90,13 @@ def new_entry_from_xml(item): if path and file_hash: new_entry = FileStatus(path, item.tag, file_hash) elif path: - emsg = f"ERROR: No hash for {item.tag}, '{path}'" + emsg = f"ERROR: No hash for '{item.tag}', '{path}'" raise ValueError(emsg) elif file_hash: - emsg = f"ERROR: No path for {item.tag} XML item" + emsg = f"ERROR: No path for '{item.tag}' XML item" raise ValueError(emsg) else: - raise ValueError(f"ERROR: Invalid {item.tag} XML item") + raise ValueError(f"ERROR: Invalid '{item.tag}' XML item") # end if return new_entry @@ -75,8 +115,8 @@ def __init__(self, file_path, description, file_hash=None): elif os.path.exists(self.__path): self.__hash = FileStatus.sha1sum(self.__path) else: - emsg = "ERROR: {}, '{}', does not exist" - raise ValueError(emsg.format(description, file_path)) + emsg = f"ERROR: '{description}', '{file_path}', does not exist" + raise ValueError(emsg) # end if def hash_mismatch(self, file_path): @@ -108,6 +148,10 @@ def gen_key(cls, file_path): def sha1sum(cls, filename): """Efficient SHA hash of a file. SHA1 is good enough for this cache Stolen from Georg Sauthoff (https://stackoverflow.com/users/427158) + + Relevant Stack Overflow post: + + https://stackoverflow.com/questions/22058048/hashing-a-file-in-python/44873382#44873382 """ file_hash = hashlib.sha1() barray = bytearray(128*1024) @@ -204,8 +248,7 @@ def __init__(self, build_cache): new_entry = new_entry_from_xml(item) self.__xml_files[new_entry.key] = new_entry elif item.tag == 'scheme_namelist_meta_file': - new_entry = new_entry_from_xml(item) - self.__scheme_nl_metadata.append(new_entry) + self.__scheme_nl_metadata.append(item.text.strip()) elif item.tag == 'scheme_namelist_groups': group_list = [x for x in item.text.strip().split(' ') if x] @@ -244,8 +287,9 @@ def update_registry(self, gen_reg_file, registry_source_files, # end for # reg_file_list contains the files generated from the registry self.__reg_gen_files = reg_file_list - # ic_names are the initial condition variable names from the registry - self.__ic_names = dict(ic_names) + # ic_names are the initial condition variable names from the registry, + # and should already be of type dict: + self.__ic_names = ic_names def update_ccpp(self, suite_definition_files, scheme_files, host_files, xml_files, namelist_meta_files, namelist_groups, @@ -335,8 +379,8 @@ def write(self): new_xml_entry(ccpp, 'xml_file', sfile.file_path, sfile.file_hash) # end for for sfile in self.__scheme_nl_metadata: - new_xml_entry(ccpp, 'scheme_namelist_meta_file', sfile, - FileStatus.sha1sum(sfile)) + scheme_nlmeta = ET.SubElement(ccpp, 'scheme_namelist_meta_file') + scheme_nlmeta.text = sfile # end for if self.__scheme_nl_groups: scheme_nlgroups = ET.SubElement(ccpp, 'scheme_namelist_groups') @@ -348,8 +392,8 @@ def write(self): preproc = ET.SubElement(ccpp, 'preproc_defs') preproc.text = self.__preproc_defs for kind_def, kind_type in self.__kind_types.items(): - kind_type = ET.SubElement(ccpp, 'kind_type') - kind_type.text = f"{kind_def}={kind_type}" + kind_elem = ET.SubElement(ccpp, 'kind_type') + kind_elem.text = f"{kind_def}={kind_type}" # end for #Combine elments into an Element Tree object: @@ -508,7 +552,7 @@ def init_write_mismatch(self, gen_init_file): def scheme_nl_metadata(self): """Return the stored list of scheme namelist metadata files""" - return [x.file_path for x in self.__scheme_nl_metadata] + return self.__scheme_nl_metadata def scheme_nl_groups(self): """Return the stored list of scheme namelist groups""" diff --git a/test/run_tests.sh b/test/run_tests.sh index 7be758b5..8d30fc25 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -58,6 +58,8 @@ run_doctest cime_config/cam_config_classes.py run_doctest cime_config/cam_config.py # CAM autogen doctests: run_doctest cime_config/cam_autogen.py +# CAM build cache doctests: +run_doctest cime_config/cam_build_cache.py # Namelist reader generator doctests: run_doctest cime_config/create_readnl_files.py # Registry generator doctests: @@ -68,7 +70,7 @@ run_doctest cime_config/atm_in_paramgen.py run_unittest test/unit/test_cam_config.py # CAM autogen unit tests: run_unittest test/unit/test_cam_autogen.py -# CAM build_cache unit tests: +# CAM build cache unit tests: run_unittest test/unit/test_build_cache.py # Registry generator unit tests: run_unittest test/unit/test_registry.py diff --git a/test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml b/test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml index fdb51432..bb1b5d21 100644 --- a/test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml +++ b/test/unit/sample_files/build_cache_files/bad_ccpp_tag_build_cache.xml @@ -15,10 +15,11 @@ - + + /not/in/kansas/wizard_nl.meta toto dog UNSET - kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + kind_phys=REAL64 of the North diff --git a/test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml b/test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml index 9330a5bd..0f30ad1b 100644 --- a/test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml +++ b/test/unit/sample_files/build_cache_files/bad_reg_tag_build_cache.xml @@ -16,9 +16,10 @@ - + + /not/in/kansas/wizard_nl.meta toto dog UNSET - kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + kind_phys=REAL64 diff --git a/test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml b/test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml index 619939e3..7d454ab6 100644 --- a/test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml +++ b/test/unit/sample_files/build_cache_files/bad_section_tag_build_cache.xml @@ -15,10 +15,11 @@ - + + /not/in/kansas/wizard_nl.meta toto dog UNSET - kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + kind_phys=REAL64 diff --git a/test/unit/sample_files/build_cache_files/example_build_cache.xml b/test/unit/sample_files/build_cache_files/example_build_cache.xml index e6d89565..4c7ac6ff 100644 --- a/test/unit/sample_files/build_cache_files/example_build_cache.xml +++ b/test/unit/sample_files/build_cache_files/example_build_cache.xml @@ -15,9 +15,10 @@ - + + /not/in/kansas/wizard_nl.meta toto dog UNSET - kind_phys=<Element 'kind_type' at 0x7f6f680c4450> + kind_phys=REAL64 diff --git a/test/unit/sample_files/build_cache_files/update_ccpp_build_cache.xml b/test/unit/sample_files/build_cache_files/update_ccpp_build_cache.xml new file mode 100644 index 00000000..d25d8d4a --- /dev/null +++ b/test/unit/sample_files/build_cache_files/update_ccpp_build_cache.xml @@ -0,0 +1,24 @@ + + + + + + + none + + /yellow/brick/road/munchkin.meta + brain + heart + + + + + + + /dragon/fruit/kumquat_namelist.meta + orange lemon + + -DBANANA + kind_phys=REAL32 + + diff --git a/test/unit/sample_files/build_cache_files/update_init_gen_build_cache.xml b/test/unit/sample_files/build_cache_files/update_init_gen_build_cache.xml new file mode 100644 index 00000000..b431b5d1 --- /dev/null +++ b/test/unit/sample_files/build_cache_files/update_init_gen_build_cache.xml @@ -0,0 +1,24 @@ + + + + + + + none + + /yellow/brick/road/munchkin.meta + brain + heart + + + + + + + /not/in/kansas/wizard_nl.meta + toto dog + + UNSET + kind_phys=REAL64 + + diff --git a/test/unit/sample_files/build_cache_files/update_reg_build_cache.xml b/test/unit/sample_files/build_cache_files/update_reg_build_cache.xml new file mode 100644 index 00000000..f394d8f3 --- /dev/null +++ b/test/unit/sample_files/build_cache_files/update_reg_build_cache.xml @@ -0,0 +1,24 @@ + + + + + + + banana + + tmp/cam_build_cache/test_reg.xml + heart + brain + + + + + + + /not/in/kansas/wizard_nl.meta + toto dog + + UNSET + kind_phys=REAL64 + + diff --git a/test/unit/test_build_cache.py b/test/unit/test_build_cache.py index 010cffde..9a66d21d 100644 --- a/test/unit/test_build_cache.py +++ b/test/unit/test_build_cache.py @@ -13,7 +13,7 @@ python test_build_cache.py -v -which will currently run XX tests, all of which should pass. +which will currently run 30 tests, all of which should pass. """ #---------------------------------------- @@ -22,11 +22,15 @@ import sys import os import os.path +import glob +import shutil +import filecmp #Python unit-testing library: import unittest #Add directory to python path: +_CWD = os.getcwd() _CURRDIR = os.path.abspath(os.path.dirname(__file__)) _CAM_ROOT_DIR = os.path.join(_CURRDIR, os.pardir, os.pardir) _CAM_CONF_DIR = os.path.abspath(os.path.join(_CAM_ROOT_DIR, "cime_config")) @@ -54,12 +58,24 @@ #Import CAM Build Cache object: # pylint: disable=wrong-import-position -from cam_build_cache import BuildCacheCAM +from cam_build_cache import FileStatus, BuildCacheCAM #Import CCPP Error type: from parse_source import CCPPError # pylint: enable=wrong-import-position +#++++++++++++++++ +#Helper functions +#++++++++++++++++ + +def remove_files(file_list): + """Remove files in if they exist""" + for fpath in file_list: + if os.path.exists(fpath): + os.remove(fpath) + #End if + #End for + #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #Main cam_build_cache testing routine, used when script is run directly #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -72,6 +88,161 @@ class CamBuildCacheTestRoutine(unittest.TestCase): are running properly. """ + #++++++++++++++++++++++ + #Test environment setup + #++++++++++++++++++++++ + + @classmethod + def setUpClass(cls): + """Clean output directory (tmp) before running tests""" + #Does "tmp" directory exist? If not then create it: + if not os.path.exists(_PRE_TMP_DIR): + os.mkdir(_PRE_TMP_DIR) + #end if + #Now check if "write_init_files" directory exists: + if not os.path.exists(_TMP_DIR): + os.mkdir(_TMP_DIR) + #end if + + #Clear out all files: + remove_files(glob.iglob(os.path.join(_TMP_DIR, '*.*'))) + + #Set path to test registry file, which will be used in + #various tests below as a stand-in for all teeded files: + test_reg_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Copy test registry file to local "tmp" directory: + shutil.copy2(test_reg_file, os.path.join(_TMP_DIR, "test_reg.xml")) + + #Finally, need to change the current working directory in order + #for the relative paths to work: + os.chdir(_CURRDIR) + + #Run inherited setup method: + super(cls, CamBuildCacheTestRoutine).setUpClass() + + #++++++++++++++++ + + @classmethod + def tearDownClass(cls): + """ + Return to original working directory + now that these tests are finished. + """ + + #Return to original working working directory: + os.chdir(_CWD) + + #Run inherited teardown method: + super(cls, CamBuildCacheTestRoutine).tearDownClass() + + #++++++++++++++++++++++++++++++++ + #FileStatus object creation tests + #++++++++++++++++++++++++++++++++ + + def test_file_status_cache_file_hash(self): + + """ + Check that the FileStatus object is + created successfully when given the + proper inputs, incluing a file hash. + """ + + #Set path to test registry file: + test_reg_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Create new FileStatus object with assigned hashs: + test_status = FileStatus(test_reg_file, "test", file_hash="orange") + + #Check that new status object has correct properties: + self.assertEqual(test_status.file_path, test_reg_file) + self.assertEqual(test_status.file_hash, "orange") + self.assertEqual(test_status.key, "param_reg.xml") + + #++++++++++++++++ + + def test_file_status_cache_file_no_hash(self): + + """ + Check that the FileStatus object is + created successfully when given the + proper inputs, except a file hash. + """ + + #Set path to test registry file: + test_reg_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Set expected sha256 hash value: + test_hash = "584c7f0992d4af811afb2069c752f279c51a1ac4" + + #Create new FileStatus object with assigned hashs: + test_status = FileStatus(test_reg_file, "test") + + #Check that new status object has correct properties: + self.assertEqual(test_status.file_path, test_reg_file) + self.assertEqual(test_status.file_hash, test_hash) + self.assertEqual(test_status.key, "param_reg.xml") + + #++++++++++++++++ + + def test_file_status_no_file(self): + + """ + Check that the correct error is raised when + the FileStatus object is initialized with + no provided hash and a non-existent file. + """ + + #Set path to non-existent file: + missing_file = os.path.join(_WRITE_INIT_DIR, "scooby_dooby.doo") + + #Set expected error message: + emsg = f"ERROR: 'test', '{missing_file}', does not exist" + + #Expect Value error when initializing object: + with self.assertRaises(ValueError) as verr: + _ = FileStatus(missing_file, "test") + #End with + + #Check that error message matches what is expected: + self.assertEqual(emsg, str(verr.exception)) + + #++++++++++++++++ + + def test_file_status_hash_mismatch(self): + + """ + Check that the "hash_mismatch" method + works as expected. + """ + + #Set path to test registry file: + test_reg_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Set path to temporary copy of registry file: + mod_reg_file = os.path.join(_TMP_DIR, "param_reg_mod.xml") + + #Create new FileStatus object: + test_status = FileStatus(test_reg_file, "test") + + #Open test file: + with open(test_reg_file, "r", encoding="utf-8") as test_fil: + #Read in file: + file_lines = test_fil.readlines() + + #Modify third line: + file_lines[2] = '' + #end with + + #Now create new file with modified lines: + with open(mod_reg_file, "w", encoding="utf-8") as mod_fil: + #Write lines to new file: + mod_fil.writelines(file_lines) + #End with + + #Check that hash_mismatch returns the correct result: + self.assertTrue(test_status.hash_mismatch(mod_reg_file)) + #+++++++++++++++++++++++++++++++++++ #BuildCacheCAM object creation tests #+++++++++++++++++++++++++++++++++++ @@ -222,6 +393,55 @@ def test_build_cache_wrong_ccpp_tag(self): #Registry generation tests #+++++++++++++++++++++++++ + def test_update_registry(self): + + """ + Check that the "update_registry" + method successfully updates the build + cache, using the "write" method to validate + the change. + """ + + #Set path to already-existing cache file: + cache_file = os.path.join(_SAMPLES_DIR, "example_build_cache.xml") + + #Set path to expected cache file: + expected_file = os.path.join(_SAMPLES_DIR, "update_reg_build_cache.xml") + + #Set path to "new" build cache file: + test_file = os.path.join(_TMP_DIR, "update_reg_build_cache.xml") + + #Set path to test registry file, which is used in the provided cache file, + #making sure to use the relative path in order to exactly match the cache: + tmp_test_reg = os.path.join("tmp", "cam_build_cache", "test_reg.xml") + + #Copy build cache file to temporary directory: + shutil.copy2(cache_file, test_file) + + #Create new build cache object: + test_cache = BuildCacheCAM(test_file) + + #Set non-file update_registry inputs: + ic_names = {"Only_had_a": ["heart", "brain"]} + dycore = "banana" + + #Update registry fields: + test_cache.update_registry(tmp_test_reg, [tmp_test_reg], + dycore, None, [tmp_test_reg], ic_names) + + #Write updated fields to build cache file: + test_cache.write() + + #Create assertion message for file comparison: + amsg = f"Test file '{test_file}' does not match '{expected_file}'" + + #Check that the newly written build cache file matches + #what is expected: + self.assertTrue(filecmp.cmp(test_file, expected_file, + shallow=False), msg=amsg) + + #++++++++++++++++ + def test_registry_mismatch_good_match(self): """ @@ -370,6 +590,61 @@ def test_registry_mismatch_diff_config(self): #CCPP Physics suites generation tests #++++++++++++++++++++++++++++++++++++ + def test_update_ccpp(self): + + """ + Check that the "update_ccpp" + method successfully updates the build + cache, using the "write" method to validate + the change. + """ + + #Set path to already-existing cache file: + cache_file = os.path.join(_SAMPLES_DIR, "example_build_cache.xml") + + #Set path to expected cache file: + expected_file = os.path.join(_SAMPLES_DIR, "update_ccpp_build_cache.xml") + + #Set path to "new" build cache file: + test_file = os.path.join(_TMP_DIR, "update_ccpp_build_cache.xml") + + #Set path to test registry file, which is used in the provided cache file, + #making sure to use the relative path in order to exactly match the cache: + tmp_test_reg = os.path.join("tmp", "cam_build_cache", "test_reg.xml") + + #Copy build cache file to temporary directory: + shutil.copy2(cache_file, test_file) + + #Create new build cache object: + test_cache = BuildCacheCAM(test_file) + + #Set file-based update_ccpp inputs: + xml_nl_file_dict = {"starfruit": tmp_test_reg} + + #Set non-file update_ccpp inputs: + nl_meta = ["/dragon/fruit/kumquat_namelist.meta"] + nl_groups = ["orange", "lemon"] + preproc_defs = "-DBANANA" + kind_types = ["kind_phys=REAL32"] + + #Update ccpp fields: + test_cache.update_ccpp([tmp_test_reg], [tmp_test_reg], [tmp_test_reg], + xml_nl_file_dict, nl_meta, nl_groups, + tmp_test_reg, preproc_defs, kind_types) + + #Write updated fields to build cache file: + test_cache.write() + + #Create assertion message for file comparison: + amsg = f"Test file '{test_file}' does not match '{expected_file}'" + + #Check that the newly written build cache file matches + #what is expected: + self.assertTrue(filecmp.cmp(test_file, expected_file, + shallow=False), msg=amsg) + + #++++++++++++++++ + def test_ccpp_mismatch_good_match(self): """ @@ -559,10 +834,206 @@ def test_ccpp_mismatch_diff_host(self): #Check that function returns True: self.assertTrue(ccpp_match) + #++++++++++++++++++++++++++++++++ + #CCPP scheme namelist files tests + #++++++++++++++++++++++++++++++++ + + def test_xml_nl_good_match(self): + + """ + Check that the 'xml_nl_mismatch' + function returns False when there + is no change in the init files writing + function being used. + """ + + #Set path to already-existing cache file used by "update" tests: + cache_file = os.path.join(_SAMPLES_DIR, "update_ccpp_build_cache.xml") + + #Set path to test registry file, which is used in the provided cache file, + #making sure to use the relative path in order to exactly match the cache: + tmp_test_reg = os.path.join("tmp", "cam_build_cache", "test_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Create xml namelist files dictionary: + xml_nl_file_dict = {"starfruit": tmp_test_reg} + + #Run xml_nl_mismatch function: + nl_match = test_cache.xml_nl_mismatch(tmp_test_reg, xml_nl_file_dict) + + #Check that function returns False: + self.assertFalse(nl_match) + + #++++++++++++++++ + + def test_xml_nl_diff_create_nl_file(self): + + """ + Check that the 'xml_nl_mismatch' + function returns True when there + is a change in the namelist files + generation function being used. + """ + + #Set path to already-existing cache file used by "update" tests: + cache_file = os.path.join(_SAMPLES_DIR, "update_ccpp_build_cache.xml") + + #Set path to test registry file, which is used in the provided cache file, + #making sure to use the relative path in order to exactly match the cache: + tmp_test_reg = os.path.join("tmp", "cam_build_cache", "test_reg.xml") + + #Set path to "new" file being used as the namelist generator file. Please + #note that for simplicity this is still just another registry xml file: + new_create_nl_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Create xml namelist files dictionary: + xml_nl_file_dict = {"starfruit": tmp_test_reg} + + #Run xml_nl_mismatch function: + nl_match = test_cache.xml_nl_mismatch(new_create_nl_file, xml_nl_file_dict) + + #Check that function returns False: + self.assertTrue(nl_match) + + #++++++++++++++++ + + def test_xml_nl_diff_xml_file_path(self): + + """ + Check that the 'xml_nl_mismatch' + function returns True when there + is a difference in the namelist xml + file path, even if the files themselves + are the same. + """ + + #Set path to already-existing cache file used by "update" tests: + cache_file = os.path.join(_SAMPLES_DIR, "update_ccpp_build_cache.xml") + + #Set path to test registry file, which is used in the provided cache file: + tmp_test_reg = os.path.join("tmp", "cam_build_cache", "test_reg.xml") + + #Also set path to original registry file, which should have the same + #file contents (and thus the same hash), but a different file path: + test_reg_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Create xml namelist files dictionary, but this time + #pointing to the original, non-cached file path: + xml_nl_file_dict = {"starfruit": test_reg_file} + + #Run xml_nl_mismatch function: + nl_match = test_cache.xml_nl_mismatch(tmp_test_reg, xml_nl_file_dict) + + #Check that function returns False: + self.assertTrue(nl_match) + + #++++++++++++++++ + + def test_xml_nl_diff_xml_file(self): + + """ + Check that the 'xml_nl_mismatch' + function returns True when there + is a difference in the namelist xml + file, even if the file paths are the + same. + """ + + #Set path to already-existing cache file used by "update" tests: + cache_file = os.path.join(_SAMPLES_DIR, "update_ccpp_build_cache.xml") + + #Set path to test registry file, which is used in the provided cache file, + #making sure to use the relative path in order to exactly match the cache: + tmp_test_reg = os.path.join("tmp", "cam_build_cache", "test_reg.xml") + + #Set path to "new" xml file, which is different from the one provided + #in the cache file: + test_reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + + #Copy "new" xml file to local "tmp" directory, so that it has the + #same path and name as the cached file, but different contents/hash: + shutil.copy2(test_reg_file, os.path.join(_CURRDIR, tmp_test_reg)) + + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) + + #Create xml namelist files dictionary, but this time + #pointing to the original, non-cached file path: + xml_nl_file_dict = {"starfruit": tmp_test_reg} + + #Run xml_nl_mismatch function: + nl_match = test_cache.xml_nl_mismatch(tmp_test_reg, xml_nl_file_dict) + + #Check that function returns False: + self.assertTrue(nl_match) + + #Reset the "test_reg.xml" file to it's original values, + #in order to avoid test failures elsewhere: + orig_reg_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + shutil.copy2(orig_reg_file, os.path.join(_CURRDIR, tmp_test_reg)) + #++++++++++++++++++++++++++++++++++++++++ #Initialization routines generation tests #++++++++++++++++++++++++++++++++++++++++ + def test_update_init_gen(self): + + """ + Check that the "update_init_gen" + method successfully updates the build + cache, using the "write" method to validate + the change. + """ + + #Set path to already-existing cache file: + cache_file = os.path.join(_SAMPLES_DIR, "example_build_cache.xml") + + #Set path to expected cache file: + expected_file = os.path.join(_SAMPLES_DIR, "update_init_gen_build_cache.xml") + + #Set path to "new" build cache file: + test_file = os.path.join(_TMP_DIR, "update_init_gen_build_cache.xml") + + #Set path to test registry file, which will be used in the + #"updated" build cache as a stand-in for all needed files: + test_reg_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + + #Copy build cache file to temporary directory: + shutil.copy2(cache_file, test_file) + + #Copy registry file to local "tmp" directory, which + #removes the need to match absolute paths when + #checking the contents of the output file: + tmp_test_reg = os.path.join("tmp", "cam_build_cache", "test_reg.xml") + shutil.copy2(test_reg_file, os.path.join(os.curdir, tmp_test_reg)) + + #Create new build cache object: + test_cache = BuildCacheCAM(test_file) + + #Update init routines generator: + test_cache.update_init_gen(tmp_test_reg) + + #Write updated fields to build cache file: + test_cache.write() + + #Create assertion message for file comparison: + amsg = f"Test file '{test_file}' does not match '{expected_file}'" + + #Check that the newly written build cache file matches + #what is expected: + self.assertTrue(filecmp.cmp(test_file, expected_file, + shallow=False), msg=amsg) + + #++++++++++++++++ + def test_init_write_mismatch_good_match(self): """ @@ -583,7 +1054,7 @@ def test_init_write_mismatch_good_match(self): #Create new build cache object: test_cache = BuildCacheCAM(cache_file) - #Run registry_mismatch function: + #Run init_write_mismatch function: init_match = test_cache.init_write_mismatch(init_gen_file) #Check that function returns False: @@ -610,7 +1081,7 @@ def test_init_write_diff_host(self): #Create new build cache object: test_cache = BuildCacheCAM(cache_file) - #Run registry_mismatch function: + #Run init_write_mismatch function: init_match = test_cache.init_write_mismatch(new_init_gen_file) #Check that function returns True: From d1e2a9f2403f75da14b64e07ac9c43ce8ad8bdc1 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 5 Jan 2023 22:01:51 -0700 Subject: [PATCH 5/8] Fix test failures, and remove python 3.11 testing for now. --- .github/workflows/python_unit_tests.yml | 2 +- test/unit/test_build_cache.py | 66 ++++++++++++------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/python_unit_tests.yml b/.github/workflows/python_unit_tests.yml index 9ca8a8db..48a79a6a 100644 --- a/.github/workflows/python_unit_tests.yml +++ b/.github/workflows/python_unit_tests.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: #All of these python versions will be used to run tests: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10"] fail-fast: false steps: # Acquire github action routines: diff --git a/test/unit/test_build_cache.py b/test/unit/test_build_cache.py index 9a66d21d..81b399d4 100644 --- a/test/unit/test_build_cache.py +++ b/test/unit/test_build_cache.py @@ -69,12 +69,12 @@ #++++++++++++++++ def remove_files(file_list): - """Remove files in if they exist""" - for fpath in file_list: - if os.path.exists(fpath): - os.remove(fpath) - #End if - #End for + """Remove files in if they exist""" + for fpath in file_list: + if os.path.exists(fpath): + os.remove(fpath) + #End if + #End for #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ #Main cam_build_cache testing routine, used when script is run directly @@ -125,16 +125,16 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - """ - Return to original working directory - now that these tests are finished. - """ + """ + Return to original working directory + now that these tests are finished. + """ - #Return to original working working directory: - os.chdir(_CWD) + #Return to original working working directory: + os.chdir(_CWD) - #Run inherited teardown method: - super(cls, CamBuildCacheTestRoutine).tearDownClass() + #Run inherited teardown method: + super(cls, CamBuildCacheTestRoutine).tearDownClass() #++++++++++++++++++++++++++++++++ #FileStatus object creation tests @@ -501,31 +501,31 @@ def test_registry_mismatch_diff_dycore(self): def test_registry_mismatch_diff_reg_file(self): - """ - Check that the 'registry_mismatch' - function returns True when there is - a change in the registry file being used. - """ + """ + Check that the 'registry_mismatch' + function returns True when there is + a change in the registry file being used. + """ - #Set path to already-existing cache file used by test_write_init_files: - cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") + #Set path to already-existing cache file used by test_write_init_files: + cache_file = os.path.join(_WRITE_INIT_DIR, "simple_build_cache_template.xml") - #Set path to registry generator file listed in build_cache file. - #Please note that in this sample file the registry XML file is listed, - #and not a python file as would normally be the case: - reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") + #Set path to registry generator file listed in build_cache file. + #Please note that in this sample file the registry XML file is listed, + #and not a python file as would normally be the case: + reg_file = os.path.join(_WRITE_INIT_DIR, "simple_reg.xml") - #Set path to "new" registry file: - new_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") + #Set path to "new" registry file: + new_file = os.path.join(_WRITE_INIT_DIR, "param_reg.xml") - #Create new build cache object: - test_cache = BuildCacheCAM(cache_file) + #Create new build cache object: + test_cache = BuildCacheCAM(cache_file) - #Run registry_mismatch function: - reg_match = test_cache.registry_mismatch(reg_file, [new_file], "none", None) + #Run registry_mismatch function: + reg_match = test_cache.registry_mismatch(reg_file, [new_file], "none", None) - #Check that function returns True: - self.assertTrue(reg_match) + #Check that function returns True: + self.assertTrue(reg_match) #++++++++++++++++ From 4aa0b27e2f608bd6c2eb43737d4d5618756118eb Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 24 Jan 2023 13:25:55 -0700 Subject: [PATCH 6/8] Update CIME Tools paths (Github issue #205). --- cime_config/buildlib | 2 +- cime_config/buildnml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/buildlib b/cime_config/buildlib index fdbd2b8f..7eb30164 100755 --- a/cime_config/buildlib +++ b/cime_config/buildlib @@ -17,7 +17,7 @@ from cam_config import ConfigCAM # CAM's configure structure __CIMEROOT = os.environ.get("CIMEROOT") if __CIMEROOT is None: raise SystemExit("ERROR: must set CIMEROOT environment variable") -sys.path.append(os.path.join(__CIMEROOT, "scripts", "lib")) +sys.path.append(os.path.join(__CIMEROOT, "CIME", "Tools")) #pylint: disable=wrong-import-position # CIME imports diff --git a/cime_config/buildnml b/cime_config/buildnml index 370d9879..c34bdda1 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -13,7 +13,7 @@ _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT is None: raise SystemExit("ERROR: must set CIMEROOT environment variable") -_LIBDIR = os.path.join(_CIMEROOT, "scripts", "Tools") +_LIBDIR = os.path.join(_CIMEROOT, "CIME", "Tools") sys.path.append(_LIBDIR) # pylint: disable=wildcard-import, wrong-import-position # pylint: disable=unused-wildcard-import From 59c9f9aa03b783a88910cf16c894ff9296b74d2c Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 14 Feb 2023 07:57:37 -0700 Subject: [PATCH 7/8] Code review updates. --- cime_config/cam_autogen.py | 4 ++-- cime_config/cam_build_cache.py | 2 +- test/unit/test_build_cache.py | 14 ++++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cime_config/cam_autogen.py b/cime_config/cam_autogen.py index 39227aa7..4de3f8a6 100644 --- a/cime_config/cam_autogen.py +++ b/cime_config/cam_autogen.py @@ -539,9 +539,9 @@ def generate_physics_suites(build_cache, preproc_defs, host_name, else: # Not running namelist generator, collect metadata files - # from the build cache - nl_groups = build_cache.scheme_nl_groups() + # and namelist group names from the build cache scheme_nl_meta_files = build_cache.scheme_nl_metadata() + nl_groups = build_cache.scheme_nl_groups() # end if #Add the namelist meta files to the host files list #if present: diff --git a/cime_config/cam_build_cache.py b/cime_config/cam_build_cache.py index 00b970a2..6577d6d8 100644 --- a/cime_config/cam_build_cache.py +++ b/cime_config/cam_build_cache.py @@ -16,7 +16,7 @@ from xml.dom import minidom #Used to pretty-print cache file # Find and include the ccpp-framework scripts directory -# Assume we are in /cime_config and SPIN is in /ccpp_framework +# Assume we are in /cime_config and CCPP_FRAMEWORK is in /ccpp_framework __CURRDIR = os.path.abspath(os.path.dirname(__file__)) __CAMROOT = os.path.abspath(os.path.join(__CURRDIR, os.pardir)) __CCPP_FRAMEWORK = os.path.join(__CAMROOT, "ccpp_framework", 'scripts') diff --git a/test/unit/test_build_cache.py b/test/unit/test_build_cache.py index 81b399d4..2e748a78 100644 --- a/test/unit/test_build_cache.py +++ b/test/unit/test_build_cache.py @@ -42,7 +42,7 @@ #Check for all necessary directories: if not os.path.exists(_CAM_CONF_DIR): - _EMSG = f"ERROR: Cannot find required '{_CAM_ROOT_DIR}' directory" + _EMSG = f"ERROR: Cannot find required '{_CAM_CONF_DIR}' directory" raise ImportError(_EMSG) #End if if not os.path.exists(_CCPP_FRAMEWORK): @@ -175,7 +175,7 @@ def test_file_status_cache_file_no_hash(self): #Set expected sha256 hash value: test_hash = "584c7f0992d4af811afb2069c752f279c51a1ac4" - #Create new FileStatus object with assigned hashs: + #Create new FileStatus object with assigned hash: test_status = FileStatus(test_reg_file, "test") #Check that new status object has correct properties: @@ -266,6 +266,7 @@ def test_build_cache_cam_no_file(self): self.assertEqual(test_cache.scheme_nl_groups(), []) self.assertEqual(test_cache.reg_file_list(), []) self.assertEqual(test_cache.ic_names(), {}) + self.assertEqual(test_cache. #++++++++++++++++ @@ -843,8 +844,9 @@ def test_xml_nl_good_match(self): """ Check that the 'xml_nl_mismatch' function returns False when there - is no change in the init files writing - function being used. + is no change in the namelist files + or readnl generation function being + used. """ #Set path to already-existing cache file used by "update" tests: @@ -873,8 +875,8 @@ def test_xml_nl_diff_create_nl_file(self): """ Check that the 'xml_nl_mismatch' function returns True when there - is a change in the namelist files - generation function being used. + is a change in the readnl generation + function being used. """ #Set path to already-existing cache file used by "update" tests: From b4eada5230667690628d8772679c2b7a0c1d8269 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 14 Feb 2023 08:00:34 -0700 Subject: [PATCH 8/8] Commited wrong file. Now using the correct one. --- test/unit/test_build_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/test_build_cache.py b/test/unit/test_build_cache.py index 2e748a78..0b4cdc74 100644 --- a/test/unit/test_build_cache.py +++ b/test/unit/test_build_cache.py @@ -266,7 +266,6 @@ def test_build_cache_cam_no_file(self): self.assertEqual(test_cache.scheme_nl_groups(), []) self.assertEqual(test_cache.reg_file_list(), []) self.assertEqual(test_cache.ic_names(), {}) - self.assertEqual(test_cache. #++++++++++++++++