Skip to content

Commit

Permalink
add om3 test data and tests
Browse files Browse the repository at this point in the history
  • Loading branch information
dougiesquire committed Feb 29, 2024
1 parent bdf7f6b commit bbd8b12
Show file tree
Hide file tree
Showing 127 changed files with 327 additions and 1 deletion.
2 changes: 1 addition & 1 deletion src/access_nri_intake/source/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ def parse_access_filename(filename):
r"^ocean.*[^\d]_(\d{2})$", # A few wierd files in ACCESS-OM2 01deg_jra55v13_ryf9091
r"^.*\.p.(\d{6})_.*", # ACCESS-CM2 atmosphere
r"^.*\.p.-(\d{6})_.*", # ACCESS-ESM1.5 atmosphere
rf"[^\.]*\.{om3_components}\..*({ymds}|{ymd})$", # ACCESS-OM3
rf"[^\.]*\.{om3_components}\..*({ymds}|{ymd}|{ym})$", # ACCESS-OM3
]
# Frequency translations
frequencies = {
Expand Down
1 change: 1 addition & 0 deletions tests/data/access-om3/README
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
These files were copied from a test run using an early version of ACCESS-OM3 and subsequently stripped to a test dataset that includes only single variables/grid points
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
7 changes: 7 additions & 0 deletions tests/data/access-om3/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
experiment_uuid: 4cf0c4ee-09c9-4675-ae1f-ce46f0d848ed
created: '2024-02-27'
name: MOM6-CICE6-WW3-1deg_jra55do_ryf-4cf0c4ee
model: ACCESS-OM3
url: [email protected]:COSIMA/MOM6-CICE6-WW3.git
contact: dougiesquire
email: [email protected]
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
7 changes: 7 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.e109393186
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Loading parallelio/2.5.10
Loading requirement: hdf5/1.14.1-2 netcdf-c/4.9.2 netcdf-fortran/4.6.0
Currently Loaded Modulefiles:
1) openmpi/4.1.4(default) 4) hdf5/1.14.1-2 7) parallelio/2.5.10
2) pbs 5) netcdf-c/4.9.2
3) intel-compiler/2021.6.0 6) netcdf-fortran/4.6.0
payu: Model exited with error code 233; aborting.
7 changes: 7 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.e109393326
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Loading parallelio/2.5.10
Loading requirement: hdf5/1.14.1-2 netcdf-c/4.9.2 netcdf-fortran/4.6.0
Currently Loaded Modulefiles:
1) openmpi/4.1.4(default) 4) hdf5/1.14.1-2 7) parallelio/2.5.10
2) pbs 5) netcdf-c/4.9.2
3) intel-compiler/2021.6.0 6) netcdf-fortran/4.6.0
payu: Model exited with error code 14; aborting.
6 changes: 6 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.e109493795
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Loading parallelio/2.5.10
Loading requirement: hdf5/1.14.1-2 netcdf-c/4.9.2 netcdf-fortran/4.6.0
Currently Loaded Modulefiles:
1) openmpi/4.1.4(default) 4) hdf5/1.14.1-2 7) parallelio/2.5.10
2) pbs 5) netcdf-c/4.9.2
3) intel-compiler/2021.6.0 6) netcdf-fortran/4.6.0
6 changes: 6 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.e109494482
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Loading parallelio/2.5.10
Loading requirement: hdf5/1.14.1-2 netcdf-c/4.9.2 netcdf-fortran/4.6.0
Currently Loaded Modulefiles:
1) pbs 4) hdf5/1.14.1-2 7) parallelio/2.5.10
2) openmpi/4.1.4(default) 5) netcdf-c/4.9.2
3) intel-compiler/2021.6.0 6) netcdf-fortran/4.6.0
35 changes: 35 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.o109393186
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
laboratory path: /scratch/tm70/ds0092/access-om3
binary path: /scratch/tm70/ds0092/access-om3/bin
input path: /scratch/tm70/ds0092/access-om3/input
work path: /scratch/tm70/ds0092/access-om3/work
archive path: /scratch/tm70/ds0092/access-om3/archive
Pre-existing archive found at: /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3. Experiment name will remain: MOM6-CICE6-WW3
nruns: 1 nruns_per_submit: 1 subrun: 1
Loading input manifest: manifests/input.yaml
Loading restart manifest: manifests/restart.yaml
Loading exe manifest: manifests/exe.yaml
Setting up access-om3
Checking exe and input manifests
Updating full hashes for 1 files in manifests/exe.yaml
Creating restart manifest
Writing manifests/restart.yaml
Writing manifests/exe.yaml
payu: Found modules in /opt/Modules/v4.3.0
mpirun -wdir /scratch/tm70/ds0092/access-om3/work/MOM6-CICE6-WW3 -np 48 /scratch/tm70/ds0092/access-om3/work/MOM6-CICE6-WW3/access-om3-MOM6-CICE6-WW3-3965e25
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/access-om3.err /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/access-om3.109393186.gadi-pbs.err
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/access-om3.out /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/access-om3.109393186.gadi-pbs.out
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/job.yaml /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/job.109393186.gadi-pbs.yaml
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/env.yaml /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/env.109393186.gadi-pbs.yaml

======================================================================================
Resource Usage on 2024-02-27 13:57:27:
Job Id: 109393186.gadi-pbs
Project: tm70
Exit Status: 1
Service Units: 1.01
NCPUs Requested: 48 NCPUs Used: 48
CPU Time Used: 00:04:23
Memory Requested: 192.0GB Memory Used: 20.95GB
Walltime requested: 01:00:00 Walltime Used: 00:00:38
JobFS requested: 10.0GB JobFS used: 0B
======================================================================================
34 changes: 34 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.o109393326
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
laboratory path: /scratch/tm70/ds0092/access-om3
binary path: /scratch/tm70/ds0092/access-om3/bin
input path: /scratch/tm70/ds0092/access-om3/input
work path: /scratch/tm70/ds0092/access-om3/work
archive path: /scratch/tm70/ds0092/access-om3/archive
nruns: 1 nruns_per_submit: 1 subrun: 1
Loading input manifest: manifests/input.yaml
Loading restart manifest: manifests/restart.yaml
Loading exe manifest: manifests/exe.yaml
Setting up access-om3
Checking exe and input manifests
Updating full hashes for 1 files in manifests/exe.yaml
Creating restart manifest
Writing manifests/restart.yaml
Writing manifests/exe.yaml
payu: Found modules in /opt/Modules/v4.3.0
mpirun -wdir /scratch/tm70/ds0092/access-om3/work/MOM6-CICE6-WW3 -np 48 /scratch/tm70/ds0092/access-om3/work/MOM6-CICE6-WW3/access-om3-MOM6-CICE6-WW3-3965e25
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/access-om3.err /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/access-om3.109393326.gadi-pbs.err
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/access-om3.out /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/access-om3.109393326.gadi-pbs.out
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/job.yaml /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/job.109393326.gadi-pbs.yaml
/g/data/tm70/ds0092/model/payu/MOM6-CICE6-WW3/env.yaml /scratch/tm70/ds0092/access-om3/archive/MOM6-CICE6-WW3/error_logs/env.109393326.gadi-pbs.yaml

======================================================================================
Resource Usage on 2024-02-27 14:02:41:
Job Id: 109393326.gadi-pbs
Project: tm70
Exit Status: 1
Service Units: 0.88
NCPUs Requested: 48 NCPUs Used: 48
CPU Time Used: 00:11:19
Memory Requested: 192.0GB Memory Used: 34.53GB
Walltime requested: 01:00:00 Walltime Used: 00:00:33
JobFS requested: 10.0GB JobFS used: 0B
======================================================================================
26 changes: 26 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.o109493795
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
laboratory path: /scratch/tm70/ds0092/access-om3
binary path: /scratch/tm70/ds0092/access-om3/bin
input path: /scratch/tm70/ds0092/access-om3/input
work path: /scratch/tm70/ds0092/access-om3/work
archive path: /scratch/tm70/ds0092/access-om3/archive
nruns: 1 nruns_per_submit: 1 subrun: 1
Loading input manifest: manifests/input.yaml
Loading restart manifest: manifests/restart.yaml
Loading exe manifest: manifests/exe.yaml
Setting up access-om3
Checking exe and input manifests
Updating full hashes for 1 files in manifests/exe.yaml
Creating restart manifest

======================================================================================
Resource Usage on 2024-02-28 12:33:28:
Job Id: 109493795.gadi-pbs
Project: tm70
Exit Status: 271 (Linux Signal 15 SIGTERM Termination)
Service Units: 9.84
NCPUs Requested: 48 NCPUs Used: 48
CPU Time Used: 04:38:57
Memory Requested: 192.0GB Memory Used: 47.51GB
Walltime requested: 01:00:00 Walltime Used: 00:06:09
JobFS requested: 10.0GB JobFS used: 8.16MB
======================================================================================
30 changes: 30 additions & 0 deletions tests/data/access-om3/pbs_logs/1deg_jra55do_ry.o109494482
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
laboratory path: /scratch/tm70/ds0092/access-om3
binary path: /scratch/tm70/ds0092/access-om3/bin
input path: /scratch/tm70/ds0092/access-om3/input
work path: /scratch/tm70/ds0092/access-om3/work
archive path: /scratch/tm70/ds0092/access-om3/archive
nruns: 1 nruns_per_submit: 1 subrun: 1
Loading input manifest: manifests/input.yaml
Loading restart manifest: manifests/restart.yaml
Loading exe manifest: manifests/exe.yaml
Setting up access-om3
Checking exe and input manifests
Updating full hashes for 1 files in manifests/exe.yaml
Creating restart manifest
Writing manifests/restart.yaml
Writing manifests/exe.yaml
payu: Found modules in /opt/Modules/v4.3.0
mpirun -wdir /scratch/tm70/ds0092/access-om3/work/MOM6-CICE6-WW3 -np 48 /scratch/tm70/ds0092/access-om3/work/MOM6-CICE6-WW3/access-om3-MOM6-CICE6-WW3

======================================================================================
Resource Usage on 2024-02-28 12:37:35:
Job Id: 109494482.gadi-pbs
Project: tm70
Exit Status: 0
Service Units: 4.13
NCPUs Requested: 48 NCPUs Used: 48
CPU Time Used: 01:51:24
Memory Requested: 192.0GB Memory Used: 46.95GB
Walltime requested: 01:00:00 Walltime Used: 00:02:35
JobFS requested: 10.0GB JobFS used: 8.16MB
======================================================================================
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
1 change: 1 addition & 0 deletions tests/data/access-om3/restart000/rpointer.atm
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.datm.r.1900-01-02-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart000/rpointer.cpl
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.cpl.r.1900-01-02-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart000/rpointer.ice
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
./GMOM_JRA_WD.cice.r.1900-01-02-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart000/rpointer.ocn
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.mom6.r.1900-01-02-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart000/rpointer.rof
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.drof.r.1900-01-02-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart000/rpointer.wav
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.ww3.r.1900-01-02-00000
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
1 change: 1 addition & 0 deletions tests/data/access-om3/restart001/rpointer.atm
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.datm.r.1900-01-03-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart001/rpointer.cpl
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.cpl.r.1900-01-03-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart001/rpointer.ice
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
./GMOM_JRA_WD.cice.r.1900-01-03-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart001/rpointer.ocn
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.mom6.r.1900-01-03-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart001/rpointer.rof
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.drof.r.1900-01-03-00000.nc
1 change: 1 addition & 0 deletions tests/data/access-om3/restart001/rpointer.wav
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
GMOM_JRA_WD.ww3.r.1900-01-03-00000
1 change: 1 addition & 0 deletions tests/test_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
7,
),
(["access-esm1-5"], "AccessEsm15Builder", {"ensemble": False}, 11, 11, 11),
(["access-om3"], "AccessOm3Builder", {}, 12, 12, 6),
],
)
def test_builder_build(
Expand Down
2 changes: 2 additions & 0 deletions tests/test_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
AccessCm2Builder,
AccessEsm15Builder,
AccessOm2Builder,
AccessOm3Builder,
)


Expand All @@ -35,6 +36,7 @@ def test_CatalogManager_init(tmp_path):
(AccessOm2Builder, "access-om2", {}),
(AccessCm2Builder, "access-cm2/by578", {"ensemble": False}),
(AccessEsm15Builder, "access-esm1-5", {"ensemble": False}),
(AccessOm3Builder, "access-om3", {}),
],
)
def test_CatalogManager_build_esm(tmp_path, test_data, builder, basedir, kwargs):
Expand Down
152 changes: 152 additions & 0 deletions tests/test_source_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,14 @@
None,
),
),
(
"GMOM_JRA.mom6.sfc_1900_01",
(
"GMOM_JRA_mom6_sfc_XXXX_XX",
"1900_01",
None,
),
),
(
"GMOM_JRA.mom6.static",
(
Expand Down Expand Up @@ -401,6 +409,150 @@ def test_parse_access_filename(filename, expected):
["mmol/m^3", "days"],
),
),
(
"access-om3/output000/GMOM_JRA_WD.mom6.h.native_1900_01.nc",
(
"GMOM_JRA_WD.mom6.h.native_1900_01.nc",
"GMOM_JRA_WD_mom6_h_native_XXXX_XX",
"1900_01",
"1mon",
"1900-01-01, 00:00:00",
"1900-02-01, 00:00:00",
["average_DT", "average_T1", "average_T2", "thetao", "time_bnds"],
[
"Length of average period",
"Start time for average period",
"End time for average period",
"Sea Water Potential Temperature",
"time axis boundaries",
],
["", "", "", "sea_water_potential_temperature", ""],
["", "", "", "area:mean zl:mean yh:mean xh:mean time: mean", ""],
[
"days",
"days since 0001-01-01 00:00:00",
"days since 0001-01-01 00:00:00",
"degC",
"days since 0001-01-01 00:00:00",
],
),
),
(
"access-om3/output000/GMOM_JRA_WD.mom6.h.sfc_1900_01_02.nc",
(
"GMOM_JRA_WD.mom6.h.sfc_1900_01_02.nc",
"GMOM_JRA_WD_mom6_h_sfc_XXXX_XX_XX",
"1900_01_02",
"1day",
"1900-01-01, 00:00:00",
"1900-01-02, 00:00:00",
["average_DT", "average_T1", "average_T2", "time_bnds", "tos"],
[
"Length of average period",
"Start time for average period",
"End time for average period",
"time axis boundaries",
"Sea Surface Temperature",
],
["", "", "", "", "sea_surface_temperature"],
["", "", "", "", "area:mean yh:mean xh:mean time: mean"],
[
"days",
"days since 0001-01-01 00:00:00",
"days since 0001-01-01 00:00:00",
"days since 0001-01-01 00:00:00",
"degC",
],
),
),
(
"access-om3/output000/GMOM_JRA_WD.mom6.h.static.nc",
(
"GMOM_JRA_WD.mom6.h.static.nc",
"GMOM_JRA_WD_mom6_h_static",
None,
"fx",
"none",
"none",
["geolat", "geolon"],
["Latitude of tracer (T) points", "Longitude of tracer (T) points"],
["", ""],
["time: point", "time: point"],
["degrees_north", "degrees_east"],
),
),
(
"access-om3/output000/GMOM_JRA_WD.mom6.h.z_1900_01.nc",
(
"GMOM_JRA_WD.mom6.h.z_1900_01.nc",
"GMOM_JRA_WD_mom6_h_z_XXXX_XX",
"1900_01",
"1mon",
"1900-01-01, 00:00:00",
"1900-02-01, 00:00:00",
["average_DT", "average_T1", "average_T2", "thetao", "time_bnds"],
[
"Length of average period",
"Start time for average period",
"End time for average period",
"Sea Water Potential Temperature",
"time axis boundaries",
],
["", "", "", "sea_water_potential_temperature", ""],
["", "", "", "area:mean z_l:mean yh:mean xh:mean time: mean", ""],
[
"days",
"days since 0001-01-01 00:00:00",
"days since 0001-01-01 00:00:00",
"degC",
"days since 0001-01-01 00:00:00",
],
),
),
(
"access-om3/output000/GMOM_JRA_WD.cice.h.1900-01-01.nc",
(
"GMOM_JRA_WD.cice.h.1900-01-01.nc",
"GMOM_JRA_WD_cice_h_XXXX_XX_XX",
"1900-01-01",
"1day",
"1900-01-01, 00:00:00",
"1900-01-02, 00:00:00",
["TLAT", "TLON", "aice", "tarea", "time_bounds"],
[
"T grid center latitude",
"T grid center longitude",
"ice area (aggregate)",
"area of T grid cells",
"time interval endpoints",
],
["", "", "", "", ""],
["", "", "time: mean", "", ""],
[
"degrees_north",
"degrees_east",
"1",
"m^2",
"days since 0000-01-01 00:00:00",
],
),
),
(
"access-om3/output000/GMOM_JRA_WD.ww3.hi.1900-01-02-00000.nc",
(
"GMOM_JRA_WD.ww3.hi.1900-01-02-00000.nc",
"GMOM_JRA_WD_ww3_hi_XXXX_XX_XX_XXXXX",
"1900-01-02-00000",
"fx", # WW3 provides no time bounds
"none",
"none",
["EF", "mapsta"],
["1D spectral density", "map status"],
["", ""],
["", ""],
["m2 s", "unitless"],
),
),
],
)
def test_parse_access_ncfile(test_data, filename, expected):
Expand Down

0 comments on commit bbd8b12

Please sign in to comment.