Skip to content

Commit

Permalink
Merge pull request #119 from ACCESS-NRI/Nones_when_attrs_missing
Browse files Browse the repository at this point in the history 
Add units, input Nones for missing variable attrs
  • Loading branch information
@dougiesquire
dougiesquire committed Sep 28, 2023
2 parents b891b5c + f91a2aa commit f46ab55
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 21 deletions.
4 changes: 2 additions & 2 deletions src/access_nri_intake/source/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
PATH_COLUMN = "path"
VARIABLE_COLUMN = "variable"

SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/d4da77a0e627775c11ba394c0a3f72a2c654971c/file_asset.json"
SCHEMA_HASH = "7f1f58e1ae419faf8e24f15e937ef5717fa872920a06758ee2983506fcaf70fc"
SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/25643eb437e95ee48b3fa6b620c7a0986c2c3bb0/file_asset.json"
SCHEMA_HASH = "d7b5fcab71861f6c4b319e64cfde75f36de2bdc797f13b5b4f7029b41ce51e5a"

_, ESM_JSONSCHEMA = get_jsonschema(
url=SCHEMA_URL, known_hash=SCHEMA_HASH, required=CORE_COLUMNS
Expand Down
4 changes: 4 additions & 0 deletions src/access_nri_intake/source/builders.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ def parser(file):
variable_long_name_list,
variable_standard_name_list,
variable_cell_methods_list,
variable_units_list,
) = parse_access_ncfile(file)

info = {
Expand All @@ -254,6 +255,7 @@ def parser(file):
"variable_long_name": variable_long_name_list,
"variable_standard_name": variable_standard_name_list,
"variable_cell_methods": variable_cell_methods_list,
"variable_units": variable_units_list,
"filename": filename,
"file_id": file_id,
}
Expand Down Expand Up @@ -330,6 +332,7 @@ def parser(file):
variable_long_name_list,
variable_standard_name_list,
variable_cell_methods_list,
variable_units_list,
) = parse_access_ncfile(file)

# Remove exp_id from file id so that members can be part of the same dataset
Expand All @@ -346,6 +349,7 @@ def parser(file):
"variable_long_name": variable_long_name_list,
"variable_standard_name": variable_standard_name_list,
"variable_cell_methods": variable_cell_methods_list,
"variable_units": variable_units_list,
"filename": filename,
"file_id": file_id,
}
Expand Down
10 changes: 10 additions & 0 deletions src/access_nri_intake/source/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,15 +248,24 @@ def parse_access_ncfile(file, time_dim="time"):
variable_long_name_list = []
variable_standard_name_list = []
variable_cell_methods_list = []
variable_units_list = []
for var in ds.data_vars:
attrs = ds[var].attrs
if "long_name" in attrs:
variable_list.append(var)
variable_long_name_list.append(attrs["long_name"])
if "standard_name" in attrs:
variable_standard_name_list.append(attrs["standard_name"])
else:
variable_standard_name_list.append(None)
if "cell_methods" in attrs:
variable_cell_methods_list.append(attrs["cell_methods"])
else:
variable_cell_methods_list.append(None)
if "units" in attrs:
variable_units_list.append(attrs["units"])
else:
variable_units_list.append(None)

start_date, end_date, frequency = get_timeinfo(ds, filename_frequency, time_dim)

Expand All @@ -274,6 +283,7 @@ def parse_access_ncfile(file, time_dim="time"):
variable_long_name_list,
variable_standard_name_list,
variable_cell_methods_list,
variable_units_list,
)

return outputs
69 changes: 50 additions & 19 deletions tests/test_source_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,9 @@ def test_parse_access_filename(filename, expected):
"none",
["geolat_t", "geolon_t"],
["tracer latitude", "tracer longitude"],
[],
[None, None],
["time: point", "time: point"],
["degrees_N", "degrees_E"],
),
),
(
Expand All @@ -110,8 +111,9 @@ def test_parse_access_filename(filename, expected):
"1910-01-01, 00:00:00",
["temp", "time_bounds"],
["Conservative temperature", "time axis boundaries"],
["sea_water_conservative_temperature"],
["time: mean"],
["sea_water_conservative_temperature", None],
["time: mean", None],
["K", "days"],
),
),
(
Expand All @@ -128,8 +130,9 @@ def test_parse_access_filename(filename, expected):
"mixed layer depth determined by density criteria",
"time axis boundaries",
],
["ocean_mixed_layer_thickness_defined_by_sigma_t"],
["time: mean"],
["ocean_mixed_layer_thickness_defined_by_sigma_t", None],
["time: mean", None],
["m", "days"],
),
),
(
Expand All @@ -145,6 +148,7 @@ def test_parse_access_filename(filename, expected):
["mixed layer depth determined by density criteria"],
["ocean_mixed_layer_thickness_defined_by_sigma_t"],
["time: mean"],
["m"],
),
),
(
Expand All @@ -164,8 +168,15 @@ def test_parse_access_filename(filename, expected):
"area of T grid cells",
"boundaries for time-averaging interval",
],
[],
["time: mean"],
[None, None, None, None, None],
[None, None, "time: mean", None, None],
[
"degrees_north",
"degrees_east",
"1",
"m^2",
"days since 1900-01-01 00:00:00",
],
),
),
(
Expand All @@ -181,6 +192,7 @@ def test_parse_access_filename(filename, expected):
["TEMPERATURE AT 1.5M"],
["air_temperature"],
["time: mean"],
["K"],
),
),
(
Expand All @@ -200,8 +212,15 @@ def test_parse_access_filename(filename, expected):
"area of T grid cells",
"boundaries for time-averaging interval",
],
[],
["time: mean"],
[None, None, None, None, None],
[None, None, "time: mean", None, None],
[
"degrees_north",
"degrees_east",
"1",
"m^2",
"days since 1850-01-01 00:00:00",
],
),
),
(
Expand All @@ -215,8 +234,9 @@ def test_parse_access_filename(filename, expected):
"2015-07-01, 00:00:00",
["sst", "time_bounds"],
["Potential temperature", "time axis boundaries"],
["sea_surface_temperature"],
["time: mean"],
["sea_surface_temperature", None],
["time: mean", None],
["K", "days"],
),
),
(
Expand All @@ -230,8 +250,9 @@ def test_parse_access_filename(filename, expected):
"2015-07-01, 00:00:00",
["temp_global_ave", "time_bounds"],
["Global mean temp in liquid seawater", "time axis boundaries"],
["sea_water_potential_temperature"],
["time: mean"],
["sea_water_potential_temperature", None],
["time: mean", None],
["deg_C", "days"],
),
),
(
Expand All @@ -247,6 +268,7 @@ def test_parse_access_filename(filename, expected):
["TEMPERATURE AT 1.5M"],
["air_temperature"],
["time: mean"],
["K"],
),
),
(
Expand All @@ -266,8 +288,15 @@ def test_parse_access_filename(filename, expected):
"area of T grid cells",
"boundaries for time-averaging interval",
],
[],
["time: mean"],
[None, None, None, None, None],
[None, None, "time: mean", None, None],
[
"degrees_north",
"degrees_east",
"1",
"m^2",
"days since 0001-01-01 00:00:00",
],
),
),
(
Expand All @@ -281,8 +310,9 @@ def test_parse_access_filename(filename, expected):
"1850-12-30, 00:00:00",
["fgco2_raw", "time_bounds"],
["Flux into ocean - DIC, inc. anth.", "time axis boundaries"],
[],
["time: mean"],
[None, None],
["time: mean", None],
["mmol/m^2/s", "days"],
),
),
(
Expand All @@ -296,8 +326,9 @@ def test_parse_access_filename(filename, expected):
"1850-12-30, 00:00:00",
["o2", "time_bounds"],
["o2", "time axis boundaries"],
[],
["time: mean"],
[None, None],
["time: mean", None],
["mmol/m^3", "days"],
),
),
],
Expand Down

0 comments on commit f46ab55

Please sign in to comment.