From ecbb538d4aad858d0cf0edd9c72157c89f45be9c Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Fri, 30 Jun 2023 15:53:08 +1000 Subject: [PATCH] add Builder tests --- src/access_nri_intake/catalog/__init__.py | 2 +- src/access_nri_intake/source/__init__.py | 4 +- src/access_nri_intake/source/utils.py | 3 + .../history/atm/netCDF/by578a.p7201501_6h.nc | Bin .../history/atm/netCDF/by578a.p8201501_3h.nc | Bin .../history/atm/netCDF/by578a.pd201501_dai.nc | Bin .../history/atm/netCDF/by578a.pm201501_mon.nc | Bin .../history/ice/iceh_d.2015-01.nc | Bin .../history/ice/iceh_m.2015-01.nc | Bin .../history/ocn/ocean_daily.nc-20150630 | Bin .../history/ocn/ocean_month.nc-20150630 | Bin .../history/ocn/ocean_scalar.nc-20150630 | Bin .../access-cm2/{mem1 => by578}/metadata.yaml | 0 .../restart/atm/by578.xhist-19700630 | 0 .../restart/atm/by578.xhist-20191231 | 0 .../restart/atm/by578a.da19700701_00 | 0 .../restart/atm/by578a.da20200101_00 | 0 .../restart/cpl/a2i.nc-19700630 | 0 .../restart/cpl/a2i.nc-20191231 | 0 .../restart/cpl/i2a.nc-19700630 | 0 .../restart/cpl/i2a.nc-20191231 | 0 .../restart/cpl/o2i.nc-19700630 | 0 .../restart/cpl/o2i.nc-20191231 | 0 .../restart/ice/ice.restart_file-19700630 | 0 .../restart/ice/ice.restart_file-20191231 | 0 .../restart/ice/iced.1970-07-01-00000.nc | 0 .../restart/ice/iced.2020-01-01-00000.nc | 0 .../restart/ice/mice.nc-19700630 | 0 .../restart/ice/mice.nc-20191231 | 0 .../restart/ocn/restart-19700630.tar | 0 .../restart/ocn/restart-20191231.tar | 0 .../access-cm2/{mem1 => by578}/tmp_comp.txt | 0 .../access-cm2/{mem1 => by578}/tmp_new.txt | 0 .../atm/netCDF/by578aa.p7201501_6h.nc} | Bin .../atm/netCDF/by578aa.p8201501_3h.nc} | Bin .../atm/netCDF/by578aa.pd201501_dai.nc} | Bin .../atm/netCDF/by578aa.pm201501_mon.nc} | Bin .../history/ice/iceh_d.2015-01.nc | Bin .../history/ice/iceh_m.2015-01.nc | Bin .../history/ocn/ocean_daily.nc-20150630 | Bin .../history/ocn/ocean_month.nc-20150630 | Bin .../history/ocn/ocean_scalar.nc-20150630 | Bin .../access-cm2/{mem2 => by578a}/metadata.yaml | 0 .../restart/atm/by578.xhist-19700630 | 0 .../restart/atm/by578.xhist-20191231 | 0 .../restart/atm/by578a.da19700701_00 | 0 .../restart/atm/by578a.da20200101_00 | 0 .../restart/cpl/a2i.nc-19700630 | 0 .../restart/cpl/a2i.nc-20191231 | 0 .../restart/cpl/i2a.nc-19700630 | 0 .../restart/cpl/i2a.nc-20191231 | 0 .../restart/cpl/o2i.nc-19700630 | 0 .../restart/cpl/o2i.nc-20191231 | 0 .../restart/ice/ice.restart_file-19700630 | 0 .../restart/ice/ice.restart_file-20191231 | 0 .../restart/ice/iced.1970-07-01-00000.nc | 0 .../restart/ice/iced.2020-01-01-00000.nc | 0 .../restart/ice/mice.nc-19700630 | 0 .../restart/ice/mice.nc-20191231 | 0 .../restart/ocn/restart-19700630.tar | 0 .../restart/ocn/restart-20191231.tar | 0 .../access-cm2/{mem2 => by578a}/tmp_comp.txt | 0 .../access-cm2/{mem2 => by578a}/tmp_new.txt | 0 tests/test_builders.py | 72 +++++++++++++++--- tests/test_source_utils.py | 8 +- 65 files changed, 72 insertions(+), 17 deletions(-) rename tests/data/access-cm2/{mem1 => by578}/history/atm/netCDF/by578a.p7201501_6h.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/history/atm/netCDF/by578a.p8201501_3h.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/history/atm/netCDF/by578a.pd201501_dai.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/history/atm/netCDF/by578a.pm201501_mon.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/history/ice/iceh_d.2015-01.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/history/ice/iceh_m.2015-01.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/history/ocn/ocean_daily.nc-20150630 (100%) rename tests/data/access-cm2/{mem1 => by578}/history/ocn/ocean_month.nc-20150630 (100%) rename tests/data/access-cm2/{mem1 => by578}/history/ocn/ocean_scalar.nc-20150630 (100%) rename tests/data/access-cm2/{mem1 => by578}/metadata.yaml (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/atm/by578.xhist-19700630 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/atm/by578.xhist-20191231 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/atm/by578a.da19700701_00 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/atm/by578a.da20200101_00 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/cpl/a2i.nc-19700630 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/cpl/a2i.nc-20191231 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/cpl/i2a.nc-19700630 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/cpl/i2a.nc-20191231 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/cpl/o2i.nc-19700630 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/cpl/o2i.nc-20191231 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ice/ice.restart_file-19700630 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ice/ice.restart_file-20191231 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ice/iced.1970-07-01-00000.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ice/iced.2020-01-01-00000.nc (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ice/mice.nc-19700630 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ice/mice.nc-20191231 (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ocn/restart-19700630.tar (100%) rename tests/data/access-cm2/{mem1 => by578}/restart/ocn/restart-20191231.tar (100%) rename tests/data/access-cm2/{mem1 => by578}/tmp_comp.txt (100%) rename tests/data/access-cm2/{mem1 => by578}/tmp_new.txt (100%) rename tests/data/access-cm2/{mem2/history/atm/netCDF/by578a.p7201501_6h.nc => by578a/history/atm/netCDF/by578aa.p7201501_6h.nc} (100%) rename tests/data/access-cm2/{mem2/history/atm/netCDF/by578a.p8201501_3h.nc => by578a/history/atm/netCDF/by578aa.p8201501_3h.nc} (100%) rename tests/data/access-cm2/{mem2/history/atm/netCDF/by578a.pd201501_dai.nc => by578a/history/atm/netCDF/by578aa.pd201501_dai.nc} (100%) rename tests/data/access-cm2/{mem2/history/atm/netCDF/by578a.pm201501_mon.nc => by578a/history/atm/netCDF/by578aa.pm201501_mon.nc} (100%) rename tests/data/access-cm2/{mem2 => by578a}/history/ice/iceh_d.2015-01.nc (100%) rename tests/data/access-cm2/{mem2 => by578a}/history/ice/iceh_m.2015-01.nc (100%) rename tests/data/access-cm2/{mem2 => by578a}/history/ocn/ocean_daily.nc-20150630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/history/ocn/ocean_month.nc-20150630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/history/ocn/ocean_scalar.nc-20150630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/metadata.yaml (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/atm/by578.xhist-19700630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/atm/by578.xhist-20191231 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/atm/by578a.da19700701_00 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/atm/by578a.da20200101_00 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/cpl/a2i.nc-19700630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/cpl/a2i.nc-20191231 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/cpl/i2a.nc-19700630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/cpl/i2a.nc-20191231 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/cpl/o2i.nc-19700630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/cpl/o2i.nc-20191231 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ice/ice.restart_file-19700630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ice/ice.restart_file-20191231 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ice/iced.1970-07-01-00000.nc (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ice/iced.2020-01-01-00000.nc (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ice/mice.nc-19700630 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ice/mice.nc-20191231 (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ocn/restart-19700630.tar (100%) rename tests/data/access-cm2/{mem2 => by578a}/restart/ocn/restart-20191231.tar (100%) rename tests/data/access-cm2/{mem2 => by578a}/tmp_comp.txt (100%) rename tests/data/access-cm2/{mem2 => by578a}/tmp_new.txt (100%) diff --git a/src/access_nri_intake/catalog/__init__.py b/src/access_nri_intake/catalog/__init__.py index 025d02be..65ae0fa1 100644 --- a/src/access_nri_intake/catalog/__init__.py +++ b/src/access_nri_intake/catalog/__init__.py @@ -18,7 +18,7 @@ NAME_COLUMN = "name" TRANSLATOR_GROUPBY_COLUMNS = ["model", "realm", "frequency"] -SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/4e3d10e563d7c1c9f66e9ab92a2926cdec3d6893/experiment_asset.json" +SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/d4da77a0e627775c11ba394c0a3f72a2c654971c/experiment_asset.json" SCHEMA_HASH = "b18cf5bdd06a6f5bcdc71dfc80f7336c63eb49f6d6f75c2cd3371e59eee5488b" EXP_JSONSCHEMA, CATALOG_JSONSCHEMA = get_jsonschema( diff --git a/src/access_nri_intake/source/__init__.py b/src/access_nri_intake/source/__init__.py index 1799cc73..3b54329e 100644 --- a/src/access_nri_intake/source/__init__.py +++ b/src/access_nri_intake/source/__init__.py @@ -17,8 +17,8 @@ PATH_COLUMN = "path" VARIABLE_COLUMN = "variable" -SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/4e3d10e563d7c1c9f66e9ab92a2926cdec3d6893/file_asset.json" -SCHEMA_HASH = "2a09030653f495939c90a22e95dd1c4587c8695f7f07e17b9129a6491469f9fc" +SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/d4da77a0e627775c11ba394c0a3f72a2c654971c/file_asset.json" +SCHEMA_HASH = "7f1f58e1ae419faf8e24f15e937ef5717fa872920a06758ee2983506fcaf70fc" _, ESM_JSONSCHEMA = get_jsonschema( url=SCHEMA_URL, known_hash=SCHEMA_HASH, required=CORE_COLUMNS diff --git a/src/access_nri_intake/source/utils.py b/src/access_nri_intake/source/utils.py index 4c4f0b63..52c6db2f 100644 --- a/src/access_nri_intake/source/utils.py +++ b/src/access_nri_intake/source/utils.py @@ -189,6 +189,9 @@ def parse_access_ncfile(file): start_date, end_date, frequency = get_timeinfo(ds) + if not variable_list: + raise EmptyFileError("This file contains no variables") + if filename_frequency: if filename_frequency != frequency: msg = ( diff --git a/tests/data/access-cm2/mem1/history/atm/netCDF/by578a.p7201501_6h.nc b/tests/data/access-cm2/by578/history/atm/netCDF/by578a.p7201501_6h.nc similarity index 100% rename from tests/data/access-cm2/mem1/history/atm/netCDF/by578a.p7201501_6h.nc rename to tests/data/access-cm2/by578/history/atm/netCDF/by578a.p7201501_6h.nc diff --git a/tests/data/access-cm2/mem1/history/atm/netCDF/by578a.p8201501_3h.nc b/tests/data/access-cm2/by578/history/atm/netCDF/by578a.p8201501_3h.nc similarity index 100% rename from tests/data/access-cm2/mem1/history/atm/netCDF/by578a.p8201501_3h.nc rename to tests/data/access-cm2/by578/history/atm/netCDF/by578a.p8201501_3h.nc diff --git a/tests/data/access-cm2/mem1/history/atm/netCDF/by578a.pd201501_dai.nc b/tests/data/access-cm2/by578/history/atm/netCDF/by578a.pd201501_dai.nc similarity index 100% rename from tests/data/access-cm2/mem1/history/atm/netCDF/by578a.pd201501_dai.nc rename to tests/data/access-cm2/by578/history/atm/netCDF/by578a.pd201501_dai.nc diff --git a/tests/data/access-cm2/mem1/history/atm/netCDF/by578a.pm201501_mon.nc b/tests/data/access-cm2/by578/history/atm/netCDF/by578a.pm201501_mon.nc similarity index 100% rename from tests/data/access-cm2/mem1/history/atm/netCDF/by578a.pm201501_mon.nc rename to tests/data/access-cm2/by578/history/atm/netCDF/by578a.pm201501_mon.nc diff --git a/tests/data/access-cm2/mem1/history/ice/iceh_d.2015-01.nc b/tests/data/access-cm2/by578/history/ice/iceh_d.2015-01.nc similarity index 100% rename from tests/data/access-cm2/mem1/history/ice/iceh_d.2015-01.nc rename to tests/data/access-cm2/by578/history/ice/iceh_d.2015-01.nc diff --git a/tests/data/access-cm2/mem1/history/ice/iceh_m.2015-01.nc b/tests/data/access-cm2/by578/history/ice/iceh_m.2015-01.nc similarity index 100% rename from tests/data/access-cm2/mem1/history/ice/iceh_m.2015-01.nc rename to tests/data/access-cm2/by578/history/ice/iceh_m.2015-01.nc diff --git a/tests/data/access-cm2/mem1/history/ocn/ocean_daily.nc-20150630 b/tests/data/access-cm2/by578/history/ocn/ocean_daily.nc-20150630 similarity index 100% rename from tests/data/access-cm2/mem1/history/ocn/ocean_daily.nc-20150630 rename to tests/data/access-cm2/by578/history/ocn/ocean_daily.nc-20150630 diff --git a/tests/data/access-cm2/mem1/history/ocn/ocean_month.nc-20150630 b/tests/data/access-cm2/by578/history/ocn/ocean_month.nc-20150630 similarity index 100% rename from tests/data/access-cm2/mem1/history/ocn/ocean_month.nc-20150630 rename to tests/data/access-cm2/by578/history/ocn/ocean_month.nc-20150630 diff --git a/tests/data/access-cm2/mem1/history/ocn/ocean_scalar.nc-20150630 b/tests/data/access-cm2/by578/history/ocn/ocean_scalar.nc-20150630 similarity index 100% rename from tests/data/access-cm2/mem1/history/ocn/ocean_scalar.nc-20150630 rename to tests/data/access-cm2/by578/history/ocn/ocean_scalar.nc-20150630 diff --git a/tests/data/access-cm2/mem1/metadata.yaml b/tests/data/access-cm2/by578/metadata.yaml similarity index 100% rename from tests/data/access-cm2/mem1/metadata.yaml rename to tests/data/access-cm2/by578/metadata.yaml diff --git a/tests/data/access-cm2/mem1/restart/atm/by578.xhist-19700630 b/tests/data/access-cm2/by578/restart/atm/by578.xhist-19700630 similarity index 100% rename from tests/data/access-cm2/mem1/restart/atm/by578.xhist-19700630 rename to tests/data/access-cm2/by578/restart/atm/by578.xhist-19700630 diff --git a/tests/data/access-cm2/mem1/restart/atm/by578.xhist-20191231 b/tests/data/access-cm2/by578/restart/atm/by578.xhist-20191231 similarity index 100% rename from tests/data/access-cm2/mem1/restart/atm/by578.xhist-20191231 rename to tests/data/access-cm2/by578/restart/atm/by578.xhist-20191231 diff --git a/tests/data/access-cm2/mem1/restart/atm/by578a.da19700701_00 b/tests/data/access-cm2/by578/restart/atm/by578a.da19700701_00 similarity index 100% rename from tests/data/access-cm2/mem1/restart/atm/by578a.da19700701_00 rename to tests/data/access-cm2/by578/restart/atm/by578a.da19700701_00 diff --git a/tests/data/access-cm2/mem1/restart/atm/by578a.da20200101_00 b/tests/data/access-cm2/by578/restart/atm/by578a.da20200101_00 similarity index 100% rename from tests/data/access-cm2/mem1/restart/atm/by578a.da20200101_00 rename to tests/data/access-cm2/by578/restart/atm/by578a.da20200101_00 diff --git a/tests/data/access-cm2/mem1/restart/cpl/a2i.nc-19700630 b/tests/data/access-cm2/by578/restart/cpl/a2i.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem1/restart/cpl/a2i.nc-19700630 rename to tests/data/access-cm2/by578/restart/cpl/a2i.nc-19700630 diff --git a/tests/data/access-cm2/mem1/restart/cpl/a2i.nc-20191231 b/tests/data/access-cm2/by578/restart/cpl/a2i.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem1/restart/cpl/a2i.nc-20191231 rename to tests/data/access-cm2/by578/restart/cpl/a2i.nc-20191231 diff --git a/tests/data/access-cm2/mem1/restart/cpl/i2a.nc-19700630 b/tests/data/access-cm2/by578/restart/cpl/i2a.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem1/restart/cpl/i2a.nc-19700630 rename to tests/data/access-cm2/by578/restart/cpl/i2a.nc-19700630 diff --git a/tests/data/access-cm2/mem1/restart/cpl/i2a.nc-20191231 b/tests/data/access-cm2/by578/restart/cpl/i2a.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem1/restart/cpl/i2a.nc-20191231 rename to tests/data/access-cm2/by578/restart/cpl/i2a.nc-20191231 diff --git a/tests/data/access-cm2/mem1/restart/cpl/o2i.nc-19700630 b/tests/data/access-cm2/by578/restart/cpl/o2i.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem1/restart/cpl/o2i.nc-19700630 rename to tests/data/access-cm2/by578/restart/cpl/o2i.nc-19700630 diff --git a/tests/data/access-cm2/mem1/restart/cpl/o2i.nc-20191231 b/tests/data/access-cm2/by578/restart/cpl/o2i.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem1/restart/cpl/o2i.nc-20191231 rename to tests/data/access-cm2/by578/restart/cpl/o2i.nc-20191231 diff --git a/tests/data/access-cm2/mem1/restart/ice/ice.restart_file-19700630 b/tests/data/access-cm2/by578/restart/ice/ice.restart_file-19700630 similarity index 100% rename from tests/data/access-cm2/mem1/restart/ice/ice.restart_file-19700630 rename to tests/data/access-cm2/by578/restart/ice/ice.restart_file-19700630 diff --git a/tests/data/access-cm2/mem1/restart/ice/ice.restart_file-20191231 b/tests/data/access-cm2/by578/restart/ice/ice.restart_file-20191231 similarity index 100% rename from tests/data/access-cm2/mem1/restart/ice/ice.restart_file-20191231 rename to tests/data/access-cm2/by578/restart/ice/ice.restart_file-20191231 diff --git a/tests/data/access-cm2/mem1/restart/ice/iced.1970-07-01-00000.nc b/tests/data/access-cm2/by578/restart/ice/iced.1970-07-01-00000.nc similarity index 100% rename from tests/data/access-cm2/mem1/restart/ice/iced.1970-07-01-00000.nc rename to tests/data/access-cm2/by578/restart/ice/iced.1970-07-01-00000.nc diff --git a/tests/data/access-cm2/mem1/restart/ice/iced.2020-01-01-00000.nc b/tests/data/access-cm2/by578/restart/ice/iced.2020-01-01-00000.nc similarity index 100% rename from tests/data/access-cm2/mem1/restart/ice/iced.2020-01-01-00000.nc rename to tests/data/access-cm2/by578/restart/ice/iced.2020-01-01-00000.nc diff --git a/tests/data/access-cm2/mem1/restart/ice/mice.nc-19700630 b/tests/data/access-cm2/by578/restart/ice/mice.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem1/restart/ice/mice.nc-19700630 rename to tests/data/access-cm2/by578/restart/ice/mice.nc-19700630 diff --git a/tests/data/access-cm2/mem1/restart/ice/mice.nc-20191231 b/tests/data/access-cm2/by578/restart/ice/mice.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem1/restart/ice/mice.nc-20191231 rename to tests/data/access-cm2/by578/restart/ice/mice.nc-20191231 diff --git a/tests/data/access-cm2/mem1/restart/ocn/restart-19700630.tar b/tests/data/access-cm2/by578/restart/ocn/restart-19700630.tar similarity index 100% rename from tests/data/access-cm2/mem1/restart/ocn/restart-19700630.tar rename to tests/data/access-cm2/by578/restart/ocn/restart-19700630.tar diff --git a/tests/data/access-cm2/mem1/restart/ocn/restart-20191231.tar b/tests/data/access-cm2/by578/restart/ocn/restart-20191231.tar similarity index 100% rename from tests/data/access-cm2/mem1/restart/ocn/restart-20191231.tar rename to tests/data/access-cm2/by578/restart/ocn/restart-20191231.tar diff --git a/tests/data/access-cm2/mem1/tmp_comp.txt b/tests/data/access-cm2/by578/tmp_comp.txt similarity index 100% rename from tests/data/access-cm2/mem1/tmp_comp.txt rename to tests/data/access-cm2/by578/tmp_comp.txt diff --git a/tests/data/access-cm2/mem1/tmp_new.txt b/tests/data/access-cm2/by578/tmp_new.txt similarity index 100% rename from tests/data/access-cm2/mem1/tmp_new.txt rename to tests/data/access-cm2/by578/tmp_new.txt diff --git a/tests/data/access-cm2/mem2/history/atm/netCDF/by578a.p7201501_6h.nc b/tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.p7201501_6h.nc similarity index 100% rename from tests/data/access-cm2/mem2/history/atm/netCDF/by578a.p7201501_6h.nc rename to tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.p7201501_6h.nc diff --git a/tests/data/access-cm2/mem2/history/atm/netCDF/by578a.p8201501_3h.nc b/tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.p8201501_3h.nc similarity index 100% rename from tests/data/access-cm2/mem2/history/atm/netCDF/by578a.p8201501_3h.nc rename to tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.p8201501_3h.nc diff --git a/tests/data/access-cm2/mem2/history/atm/netCDF/by578a.pd201501_dai.nc b/tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.pd201501_dai.nc similarity index 100% rename from tests/data/access-cm2/mem2/history/atm/netCDF/by578a.pd201501_dai.nc rename to tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.pd201501_dai.nc diff --git a/tests/data/access-cm2/mem2/history/atm/netCDF/by578a.pm201501_mon.nc b/tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.pm201501_mon.nc similarity index 100% rename from tests/data/access-cm2/mem2/history/atm/netCDF/by578a.pm201501_mon.nc rename to tests/data/access-cm2/by578a/history/atm/netCDF/by578aa.pm201501_mon.nc diff --git a/tests/data/access-cm2/mem2/history/ice/iceh_d.2015-01.nc b/tests/data/access-cm2/by578a/history/ice/iceh_d.2015-01.nc similarity index 100% rename from tests/data/access-cm2/mem2/history/ice/iceh_d.2015-01.nc rename to tests/data/access-cm2/by578a/history/ice/iceh_d.2015-01.nc diff --git a/tests/data/access-cm2/mem2/history/ice/iceh_m.2015-01.nc b/tests/data/access-cm2/by578a/history/ice/iceh_m.2015-01.nc similarity index 100% rename from tests/data/access-cm2/mem2/history/ice/iceh_m.2015-01.nc rename to tests/data/access-cm2/by578a/history/ice/iceh_m.2015-01.nc diff --git a/tests/data/access-cm2/mem2/history/ocn/ocean_daily.nc-20150630 b/tests/data/access-cm2/by578a/history/ocn/ocean_daily.nc-20150630 similarity index 100% rename from tests/data/access-cm2/mem2/history/ocn/ocean_daily.nc-20150630 rename to tests/data/access-cm2/by578a/history/ocn/ocean_daily.nc-20150630 diff --git a/tests/data/access-cm2/mem2/history/ocn/ocean_month.nc-20150630 b/tests/data/access-cm2/by578a/history/ocn/ocean_month.nc-20150630 similarity index 100% rename from tests/data/access-cm2/mem2/history/ocn/ocean_month.nc-20150630 rename to tests/data/access-cm2/by578a/history/ocn/ocean_month.nc-20150630 diff --git a/tests/data/access-cm2/mem2/history/ocn/ocean_scalar.nc-20150630 b/tests/data/access-cm2/by578a/history/ocn/ocean_scalar.nc-20150630 similarity index 100% rename from tests/data/access-cm2/mem2/history/ocn/ocean_scalar.nc-20150630 rename to tests/data/access-cm2/by578a/history/ocn/ocean_scalar.nc-20150630 diff --git a/tests/data/access-cm2/mem2/metadata.yaml b/tests/data/access-cm2/by578a/metadata.yaml similarity index 100% rename from tests/data/access-cm2/mem2/metadata.yaml rename to tests/data/access-cm2/by578a/metadata.yaml diff --git a/tests/data/access-cm2/mem2/restart/atm/by578.xhist-19700630 b/tests/data/access-cm2/by578a/restart/atm/by578.xhist-19700630 similarity index 100% rename from tests/data/access-cm2/mem2/restart/atm/by578.xhist-19700630 rename to tests/data/access-cm2/by578a/restart/atm/by578.xhist-19700630 diff --git a/tests/data/access-cm2/mem2/restart/atm/by578.xhist-20191231 b/tests/data/access-cm2/by578a/restart/atm/by578.xhist-20191231 similarity index 100% rename from tests/data/access-cm2/mem2/restart/atm/by578.xhist-20191231 rename to tests/data/access-cm2/by578a/restart/atm/by578.xhist-20191231 diff --git a/tests/data/access-cm2/mem2/restart/atm/by578a.da19700701_00 b/tests/data/access-cm2/by578a/restart/atm/by578a.da19700701_00 similarity index 100% rename from tests/data/access-cm2/mem2/restart/atm/by578a.da19700701_00 rename to tests/data/access-cm2/by578a/restart/atm/by578a.da19700701_00 diff --git a/tests/data/access-cm2/mem2/restart/atm/by578a.da20200101_00 b/tests/data/access-cm2/by578a/restart/atm/by578a.da20200101_00 similarity index 100% rename from tests/data/access-cm2/mem2/restart/atm/by578a.da20200101_00 rename to tests/data/access-cm2/by578a/restart/atm/by578a.da20200101_00 diff --git a/tests/data/access-cm2/mem2/restart/cpl/a2i.nc-19700630 b/tests/data/access-cm2/by578a/restart/cpl/a2i.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem2/restart/cpl/a2i.nc-19700630 rename to tests/data/access-cm2/by578a/restart/cpl/a2i.nc-19700630 diff --git a/tests/data/access-cm2/mem2/restart/cpl/a2i.nc-20191231 b/tests/data/access-cm2/by578a/restart/cpl/a2i.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem2/restart/cpl/a2i.nc-20191231 rename to tests/data/access-cm2/by578a/restart/cpl/a2i.nc-20191231 diff --git a/tests/data/access-cm2/mem2/restart/cpl/i2a.nc-19700630 b/tests/data/access-cm2/by578a/restart/cpl/i2a.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem2/restart/cpl/i2a.nc-19700630 rename to tests/data/access-cm2/by578a/restart/cpl/i2a.nc-19700630 diff --git a/tests/data/access-cm2/mem2/restart/cpl/i2a.nc-20191231 b/tests/data/access-cm2/by578a/restart/cpl/i2a.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem2/restart/cpl/i2a.nc-20191231 rename to tests/data/access-cm2/by578a/restart/cpl/i2a.nc-20191231 diff --git a/tests/data/access-cm2/mem2/restart/cpl/o2i.nc-19700630 b/tests/data/access-cm2/by578a/restart/cpl/o2i.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem2/restart/cpl/o2i.nc-19700630 rename to tests/data/access-cm2/by578a/restart/cpl/o2i.nc-19700630 diff --git a/tests/data/access-cm2/mem2/restart/cpl/o2i.nc-20191231 b/tests/data/access-cm2/by578a/restart/cpl/o2i.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem2/restart/cpl/o2i.nc-20191231 rename to tests/data/access-cm2/by578a/restart/cpl/o2i.nc-20191231 diff --git a/tests/data/access-cm2/mem2/restart/ice/ice.restart_file-19700630 b/tests/data/access-cm2/by578a/restart/ice/ice.restart_file-19700630 similarity index 100% rename from tests/data/access-cm2/mem2/restart/ice/ice.restart_file-19700630 rename to tests/data/access-cm2/by578a/restart/ice/ice.restart_file-19700630 diff --git a/tests/data/access-cm2/mem2/restart/ice/ice.restart_file-20191231 b/tests/data/access-cm2/by578a/restart/ice/ice.restart_file-20191231 similarity index 100% rename from tests/data/access-cm2/mem2/restart/ice/ice.restart_file-20191231 rename to tests/data/access-cm2/by578a/restart/ice/ice.restart_file-20191231 diff --git a/tests/data/access-cm2/mem2/restart/ice/iced.1970-07-01-00000.nc b/tests/data/access-cm2/by578a/restart/ice/iced.1970-07-01-00000.nc similarity index 100% rename from tests/data/access-cm2/mem2/restart/ice/iced.1970-07-01-00000.nc rename to tests/data/access-cm2/by578a/restart/ice/iced.1970-07-01-00000.nc diff --git a/tests/data/access-cm2/mem2/restart/ice/iced.2020-01-01-00000.nc b/tests/data/access-cm2/by578a/restart/ice/iced.2020-01-01-00000.nc similarity index 100% rename from tests/data/access-cm2/mem2/restart/ice/iced.2020-01-01-00000.nc rename to tests/data/access-cm2/by578a/restart/ice/iced.2020-01-01-00000.nc diff --git a/tests/data/access-cm2/mem2/restart/ice/mice.nc-19700630 b/tests/data/access-cm2/by578a/restart/ice/mice.nc-19700630 similarity index 100% rename from tests/data/access-cm2/mem2/restart/ice/mice.nc-19700630 rename to tests/data/access-cm2/by578a/restart/ice/mice.nc-19700630 diff --git a/tests/data/access-cm2/mem2/restart/ice/mice.nc-20191231 b/tests/data/access-cm2/by578a/restart/ice/mice.nc-20191231 similarity index 100% rename from tests/data/access-cm2/mem2/restart/ice/mice.nc-20191231 rename to tests/data/access-cm2/by578a/restart/ice/mice.nc-20191231 diff --git a/tests/data/access-cm2/mem2/restart/ocn/restart-19700630.tar b/tests/data/access-cm2/by578a/restart/ocn/restart-19700630.tar similarity index 100% rename from tests/data/access-cm2/mem2/restart/ocn/restart-19700630.tar rename to tests/data/access-cm2/by578a/restart/ocn/restart-19700630.tar diff --git a/tests/data/access-cm2/mem2/restart/ocn/restart-20191231.tar b/tests/data/access-cm2/by578a/restart/ocn/restart-20191231.tar similarity index 100% rename from tests/data/access-cm2/mem2/restart/ocn/restart-20191231.tar rename to tests/data/access-cm2/by578a/restart/ocn/restart-20191231.tar diff --git a/tests/data/access-cm2/mem2/tmp_comp.txt b/tests/data/access-cm2/by578a/tmp_comp.txt similarity index 100% rename from tests/data/access-cm2/mem2/tmp_comp.txt rename to tests/data/access-cm2/by578a/tmp_comp.txt diff --git a/tests/data/access-cm2/mem2/tmp_new.txt b/tests/data/access-cm2/by578a/tmp_new.txt similarity index 100% rename from tests/data/access-cm2/mem2/tmp_new.txt rename to tests/data/access-cm2/by578a/tmp_new.txt diff --git a/tests/test_builders.py b/tests/test_builders.py index af324570..6aa85a83 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -1,23 +1,75 @@ # Copyright 2023 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. # SPDX-License-Identifier: Apache-2.0 +import intake +import pandas as pd import pytest -from access_nri_intake.source import builders +from access_nri_intake.source import CORE_COLUMNS, builders @pytest.mark.parametrize( - "basedir, builder, kwargs", + "basedirs, builder, kwargs, num_assets, num_valid_assets, num_datasets", [ - ("access-om2", "AccessOm2Builder", {}), - ("access-cm2", "AccessCm2Builder", {"ensemble": False}), - ("access-cm2", "AccessCm2Builder", {"ensemble": True}), - ("access-esm1-5", "AccessEsm15Builder", {"ensemble": False}), - ("access-esm1-5", "AccessEsm15Builder", {"ensemble": True}), + (["access-om2"], "AccessOm2Builder", {}, 12, 12, 6), + ( + ["access-cm2/by578", "access-cm2/by578a"], + "AccessCm2Builder", + {"ensemble": True}, + 18, + 14, + 7, + ), + (["access-esm1-5"], "AccessEsm15Builder", {"ensemble": False}, 11, 11, 11), ], ) -def test_builder(test_data, basedir, builder, kwargs): +def test_builder_build( + tmp_path, + test_data, + basedirs, + builder, + kwargs, + num_assets, + num_valid_assets, + num_datasets, +): + """ + Test the various steps of the build process + """ Builder = getattr(builders, builder) - path = str(test_data / basedir) + path = [str(test_data / basedir) for basedir in basedirs] builder = Builder(path, **kwargs) - print(builder.get_assets()) + + builder.get_assets() + assert isinstance(builder.assets, list) + assert len(builder.assets) == num_assets + + builder.build() + assert isinstance(builder.df, pd.DataFrame) + assert len(builder.df) == num_valid_assets + assert all([col in builder.df.columns for col in CORE_COLUMNS]) + + builder.save(name="test", description="test datastore", directory=str(tmp_path)) + + cat = intake.open_esm_datastore( + str(tmp_path / "test.json"), + columns_with_iterables=builder.columns_with_iterables, + ) + assert len(cat.df) == num_valid_assets + assert len(cat) == num_datasets + + +def test_builder_columns_with_iterables(test_data): + builder = builders.AccessOm2Builder(str(test_data / "access-om2")) + assert not builder.columns_with_iterables + builder.build() + assert sorted(list(builder.columns_with_iterables)) == sorted( + [ + col + for col, val in builder.df.applymap(type) + .isin([list, tuple, set]) + .any() + .items() + if val + ] + ) diff --git a/tests/test_source_utils.py b/tests/test_source_utils.py index 1bbd2ed6..807f4c6d 100644 --- a/tests/test_source_utils.py +++ b/tests/test_source_utils.py @@ -159,7 +159,7 @@ def test_parse_access_filename(filename, expected): ), ), ( - "access-cm2/mem1/history/atm/netCDF/by578a.pd201501_dai.nc", + "access-cm2/by578/history/atm/netCDF/by578a.pd201501_dai.nc", ( "by578a.pd201501_dai.nc", "by578a_pdXXXXXX_dai", @@ -174,7 +174,7 @@ def test_parse_access_filename(filename, expected): ), ), ( - "access-cm2/mem1/history/ice/iceh_d.2015-01.nc", + "access-cm2/by578/history/ice/iceh_d.2015-01.nc", ( "iceh_d.2015-01.nc", "iceh_d_XXXX_XX", @@ -195,7 +195,7 @@ def test_parse_access_filename(filename, expected): ), ), ( - "access-cm2/mem1/history/ocn/ocean_daily.nc-20150630", + "access-cm2/by578/history/ocn/ocean_daily.nc-20150630", ( "ocean_daily.nc-20150630", "ocean_daily", @@ -210,7 +210,7 @@ def test_parse_access_filename(filename, expected): ), ), ( - "access-cm2/mem1/history/ocn/ocean_scalar.nc-20150630", + "access-cm2/by578/history/ocn/ocean_scalar.nc-20150630", ( "ocean_scalar.nc-20150630", "ocean_scalar",