From fb8453766aef9b9b273fdb2de21233ac08fe505b Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Thu, 28 Mar 2024 12:31:41 +1100 Subject: [PATCH 1/2] assign ww3 data to wave realm in access-om3 builder --- src/access_nri_intake/source/builders.py | 4 +++- tests/test_builders.py | 27 ++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/access_nri_intake/source/builders.py b/src/access_nri_intake/source/builders.py index 4b485af..2d60d84 100644 --- a/src/access_nri_intake/source/builders.py +++ b/src/access_nri_intake/source/builders.py @@ -323,8 +323,10 @@ def parser(file): variable_units_list, ) = parse_access_ncfile(file) - if ("mom6" in filename) or ("ww3" in filename): + if "mom6" in filename: realm = "ocean" + elif "ww3" in filename: + realm = "wave" elif "cice" in filename: realm = "seaIce" else: diff --git a/tests/test_builders.py b/tests/test_builders.py index e180919..c4774a0 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -11,9 +11,9 @@ @pytest.mark.parametrize( - "basedirs, builder, kwargs, num_assets, num_valid_assets, num_datasets", + "basedirs, builder, kwargs, num_assets, num_valid_assets, num_datasets, realms", [ - (["access-om2"], "AccessOm2Builder", {}, 12, 12, 6), + (["access-om2"], "AccessOm2Builder", {}, 12, 12, 6, ["ocean", "seaIce"]), ( ["access-cm2/by578", "access-cm2/by578a"], "AccessCm2Builder", @@ -21,9 +21,26 @@ 18, 14, 7, + ["atmos", "ocean", "seaIce"], + ), + ( + ["access-esm1-5"], + "AccessEsm15Builder", + {"ensemble": False}, + 11, + 11, + 11, + ["atmos", "ocean", "seaIce"], + ), + ( + ["access-om3"], + "AccessOm3Builder", + {}, + 12, + 12, + 6, + ["ocean", "wave", "seaIce"], ), - (["access-esm1-5"], "AccessEsm15Builder", {"ensemble": False}, 11, 11, 11), - (["access-om3"], "AccessOm3Builder", {}, 12, 12, 6), ], ) def test_builder_build( @@ -35,6 +52,7 @@ def test_builder_build( num_assets, num_valid_assets, num_datasets, + realms, ): """ Test the various steps of the build process @@ -60,6 +78,7 @@ def test_builder_build( ) assert len(cat.df) == num_valid_assets assert len(cat) == num_datasets + assert sorted(cat.unique()["realm"]) == sorted(realms) def test_builder_columns_with_iterables(test_data): From 84a0202aa76c8c9e39b3d98f6053170b4c2b1c58 Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Thu, 28 Mar 2024 13:12:27 +1100 Subject: [PATCH 2/2] add specific tests for builder parsers --- src/access_nri_intake/source/builders.py | 8 +- tests/test_builders.py | 104 ++++++++++++++++++----- 2 files changed, 83 insertions(+), 29 deletions(-) diff --git a/src/access_nri_intake/source/builders.py b/src/access_nri_intake/source/builders.py index 2d60d84..bd46232 100644 --- a/src/access_nri_intake/source/builders.py +++ b/src/access_nri_intake/source/builders.py @@ -221,12 +221,8 @@ def __init__(self, path): @staticmethod def parser(file): try: - match_groups = re.match( - r".*/([^/]*)/([^/]*)/output\d+/([^/]*)/.*\.nc", file - ).groups() - # configuration = match_groups[0] - # exp_id = match_groups[1] - realm = match_groups[2] + match_groups = re.match(r".*/output\d+/([^/]*)/.*\.nc", file).groups() + realm = match_groups[0] if realm == "ice": realm = "seaIce" diff --git a/tests/test_builders.py b/tests/test_builders.py index c4774a0..878dd11 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -11,9 +11,9 @@ @pytest.mark.parametrize( - "basedirs, builder, kwargs, num_assets, num_valid_assets, num_datasets, realms", + "basedirs, builder, kwargs, num_assets, num_valid_assets, num_datasets", [ - (["access-om2"], "AccessOm2Builder", {}, 12, 12, 6, ["ocean", "seaIce"]), + (["access-om2"], "AccessOm2Builder", {}, 12, 12, 6), ( ["access-cm2/by578", "access-cm2/by578a"], "AccessCm2Builder", @@ -21,26 +21,9 @@ 18, 14, 7, - ["atmos", "ocean", "seaIce"], - ), - ( - ["access-esm1-5"], - "AccessEsm15Builder", - {"ensemble": False}, - 11, - 11, - 11, - ["atmos", "ocean", "seaIce"], - ), - ( - ["access-om3"], - "AccessOm3Builder", - {}, - 12, - 12, - 6, - ["ocean", "wave", "seaIce"], ), + (["access-esm1-5"], "AccessEsm15Builder", {"ensemble": False}, 11, 11, 11), + (["access-om3"], "AccessOm3Builder", {}, 12, 12, 6), ], ) def test_builder_build( @@ -52,7 +35,6 @@ def test_builder_build( num_assets, num_valid_assets, num_datasets, - realms, ): """ Test the various steps of the build process @@ -78,7 +60,83 @@ def test_builder_build( ) assert len(cat.df) == num_valid_assets assert len(cat) == num_datasets - assert sorted(cat.unique()["realm"]) == sorted(realms) + + +@pytest.mark.parametrize( + "filename, builder, realm, member, file_id", + [ + ( + "access-om2/output000/ocean/ocean.nc", + "AccessOm2Builder", + "ocean", + None, + "ocean", + ), + ( + "access-om2/output000/ice/OUTPUT/iceh.1900-01.nc", + "AccessOm2Builder", + "seaIce", + None, + "iceh_XXXX_XX", + ), + ( + "access-cm2/by578/history/atm/netCDF/by578a.pd201501_dai.nc", + "AccessCm2Builder", + "atmos", + "by578", + "a_pdXXXXXX_dai", + ), + ( + "access-cm2/by578a/history/atm/netCDF/by578aa.pd201501_dai.nc", + "AccessCm2Builder", + "atmos", + "by578a", + "a_pdXXXXXX_dai", + ), + ( + "access-cm2/by578/history/ice/iceh_d.2015-01.nc", + "AccessCm2Builder", + "seaIce", + "by578", + "iceh_d_XXXX_XX", + ), + ( + "access-cm2/by578/history/ocn/ocean_daily.nc-20150630", + "AccessCm2Builder", + "ocean", + "by578", + "ocean_daily", + ), + ( + "access-om3/output000/GMOM_JRA_WD.mom6.h.sfc_1900_01_02.nc", + "AccessOm3Builder", + "ocean", + None, + "GMOM_JRA_WD_mom6_h_sfc_XXXX_XX_XX", + ), + ( + "access-om3/output000/GMOM_JRA_WD.cice.h.1900-01-01.nc", + "AccessOm3Builder", + "seaIce", + None, + "GMOM_JRA_WD_cice_h_XXXX_XX_XX", + ), + ( + "access-om3/output000/GMOM_JRA_WD.ww3.hi.1900-01-02-00000.nc", + "AccessOm3Builder", + "wave", + None, + "GMOM_JRA_WD_ww3_hi_XXXX_XX_XX_XXXXX", + ), + ], +) +def test_builder_parser(test_data, filename, builder, realm, member, file_id): + Builder = getattr(builders, builder) + info = Builder.parser(str(test_data / filename)) + assert info["realm"] == realm + if member: + assert info["member"] == member + assert info["file_id"] == file_id def test_builder_columns_with_iterables(test_data):