Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into 175-data-request-add-…
Browse files Browse the repository at this point in the history
…om4_025jra_ryf
  • Loading branch information
marc-white committed Sep 24, 2024
2 parents b6e9d41 + e67d2ba commit 2956b7b
Show file tree
Hide file tree
Showing 10 changed files with 344 additions and 34 deletions.
6 changes: 5 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ requires = [
where = ["src"]

[tool.setuptools.package-data]
access_nri_intake = ["data/catalog.yaml"]
access_nri_intake = [
"data/catalog.yaml",
"data/metadata_schema_experiment.json",
"data/metadata_schema_file.json",
]

[tool.versioneer]
VCS = "git"
Expand Down
4 changes: 1 addition & 3 deletions src/access_nri_intake/catalog/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@
NAME_COLUMN = "name"
TRANSLATOR_GROUPBY_COLUMNS = ["model", "realm", "frequency"]

SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/e9055da95093ec2faa555c090fc5af17923d1566/au.org.access-nri/model/output/experiment-metadata/1-0-2.json"
SCHEMA_HASH = "ecb72c1adde3679896ceeca96aa6500d07ea2e05810155ec7a5dc301593c1dc7"

EXP_JSONSCHEMA, CATALOG_JSONSCHEMA = get_jsonschema(
url=SCHEMA_URL, known_hash=SCHEMA_HASH, required=CORE_COLUMNS
metadata_file="data/metadata_schema_experiment.json", required=CORE_COLUMNS
)

COLUMNS_WITH_ITERABLES = [
Expand Down
5 changes: 2 additions & 3 deletions src/access_nri_intake/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from .catalog import EXP_JSONSCHEMA, translators
from .catalog.manager import CatalogManager
from .source import builders
from .utils import _can_be_array, load_metadata_yaml
from .utils import _can_be_array, get_catalog_fp, load_metadata_yaml


class MetadataCheckError(Exception):
Expand Down Expand Up @@ -214,9 +214,8 @@ def _get_project(path):
# Save the catalog
cm.save()

_here = os.path.abspath(os.path.dirname(__file__))
if update:
with open(os.path.join(_here, "data", "catalog.yaml"), "w") as fobj:
with get_catalog_fp().open(mode="w") as fobj:
yaml.dump(yaml_dict, fobj)


Expand Down
7 changes: 3 additions & 4 deletions src/access_nri_intake/data/__init__.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
# Copyright 2023 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
# SPDX-License-Identifier: Apache-2.0

import os

import intake

_here = os.path.abspath(os.path.dirname(__file__))
data = intake.open_catalog(os.path.join(_here, "catalog.yaml")).access_nri
from access_nri_intake.utils import get_catalog_fp

data = intake.open_catalog(get_catalog_fp()).access_nri
169 changes: 169 additions & 0 deletions src/access_nri_intake/data/metadata_schema_experiment.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
{
"$id": "https://raw.githubusercontent.com/ACCESS-NRI/schema/main/au.org.access-nri/model/output/experiment-metadata/1-0-2.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Experiment metadata",
"description": "The metadata associated with a model experiment",
"type": "object",
"properties": {
"schema_version": {
"const": "1-0-2",
"description": "The version of the schema (string)"
},
"name": {
"type": "string",
"description": "The name of the experiment (string)"
},
"experiment_uuid": {
"type": "string",
"format": "uuid",
"description": "Unique uuid for the experiment (string)"
},
"description": {
"type": "string",
"description": "Short description of the experiment (string, < 150 char)"
},
"long_description": {
"type": "string",
"description": "Long description of the experiment (string)"
},
"model": {
"oneOf": [
{"type": ["string", "null"]},
{
"type": "array",
"items": {"type": ["string", "null"]}
}
],
"description": "The name(s) of the model(s) used in the experiment (string)"
},
"realm": {
"type": "array",
"items": {
"oneOf": [
{"type": "null"},
{
"type": "string",
"enum": [
"aerosol",
"atmos",
"atmosChem",
"land",
"landIce",
"none",
"ocean",
"ocnBgchem",
"seaIce",
"unknown",
"wave"
]
}
]
},
"description": "The realm(s) included in the experiment (string)"
},
"frequency": {
"type": "array",
"items": {
"oneOf": [
{"type": "null"},
{
"type": "string",
"oneOf": [
{
"pattern": "^fx$"
},
{
"pattern": "^subhr$"
},
{
"pattern": "^\\d+hr$"
},
{
"pattern": "^\\d+day$"
},
{
"pattern": "^\\d+mon$"
},
{
"pattern": "^\\d+yr$"
},
{
"pattern": "^\\d+dec$"
}
]
}
]
},
"description": "The frequency(/ies) included in the experiment (string)"
},
"variable": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The variable(s) included in the experiment (string)"
},
"nominal_resolution": {
"type": "array",
"items": {"type": ["string", "null"]},
"description": "The nominal resolution(s) of model(s) used in the experiment (string)"
},
"version": {
"type": ["number", "string", "null"],
"description": "The version of the experiment (number, string)"
},
"contact": {
"type": ["string", "null"],
"description": "Contact name for the experiment (string)"
},
"email": {
"type": ["string", "null"],
"description": "Email address of the contact for the experiment (string)"
},
"created": {
"type": ["string", "null"],
"description": "Initial creation date of experiment (string)"
},
"reference": {
"type": ["string", "null"],
"description": "Citation or reference information (string)"
},
"license": {
"type": ["string", "null"],
"description": "License of the experiment (string)"
},
"url": {
"type": ["string", "null"],
"description": "Relevant url, e.g. github repo for experiment configuration (string)"
},
"parent_experiment": {
"type": ["string", "null"],
"description": "experiment_uuid for parent experiment if appropriate (string)"
},
"related_experiments": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "experiment_uuids for any related experiment(s) (string)"
},
"notes": {
"type": ["string", "null"],
"description": "Additional notes (string)"
},
"keywords": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "Keywords to associated with experiment (string)"
}
},
"required": [
"name",
"experiment_uuid",
"description",
"long_description"
],
"additionalProperties": false
}
132 changes: 132 additions & 0 deletions src/access_nri_intake/data/metadata_schema_file.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
{
"$id": "https://raw.githubusercontent.com/ACCESS-NRI/schema/main/au.org.access-nri/model/output/file-metadata/1-0-1.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "File metadata",
"description": "The metadata associated with a file containing or referencing climate model data",
"type": "object",
"properties": {
"schema_version": {
"const": "1-0-1",
"description": "The version of the schema (string)"
},
"path": {
"type": "string",
"description": "The path to the file asset"
},
"realm": {
"type": "string",
"enum": [
"aerosol",
"atmos",
"atmosChem",
"land",
"landIce",
"none",
"ocean",
"ocnBgchem",
"seaIce",
"unknown",
"wave"
],
"description": "The realm of the data in the file asset"
},
"variable": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The variable(s) in the file asset"
},
"variable_long_name": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The long_name(s) of the variable(s) in the file asset"
},
"variable_standard_name": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The standard_names(s) of the variable(s) in the file asset"
},
"variable_cell_methods": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The cell_methods(s) of the variable(s) in the file asset"
},
"variable_units": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The units of the variable(s) in the file asset"
},
"frequency": {
"type": "string",
"oneOf": [
{
"pattern": "^fx$"
},
{
"pattern": "^subhr$"
},
{
"pattern": "^\\d+hr$"
},
{
"pattern": "^\\d+day$"
},
{
"pattern": "^\\d+mon$"
},
{
"pattern": "^\\d+yr$"
},
{
"pattern": "^\\d+dec$"
}
],
"description": "The frequency of the variable(s) in the file asset"
},
"start_date": {
"type": "string",
"oneOf": [
{
"pattern": "^\\d\\d\\d\\d-\\d\\d-\\d\\d,\\s\\d\\d:\\d\\d:\\d\\d$"
},
{
"pattern": "none"
}
],
"description": "The start date of the variable(s) in the file asset"
},
"end_date": {
"type": "string",
"oneOf": [
{
"pattern": "^\\d\\d\\d\\d-\\d\\d-\\d\\d,\\s\\d\\d:\\d\\d:\\d\\d$"
},
{
"pattern": "none"
}
],
"description": "The end date of the variable(s) in the file asset"
},
"nominal_resolution": {
"type": "string",
"description": "The nominal resolution of the variable(s) in the file asset"
}
},
"required": [
"path",
"realm",
"variable",
"frequency",
"start_date",
"end_date"
]
}
4 changes: 1 addition & 3 deletions src/access_nri_intake/source/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
PATH_COLUMN = "path"
VARIABLE_COLUMN = "variable"

SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/e9055da95093ec2faa555c090fc5af17923d1566/au.org.access-nri/model/output/file-metadata/1-0-1.json"
SCHEMA_HASH = "8f2f069fa06d81ff086b91daa6503f75615aa90385ab61ee2d1a7956dc96f9a6"

_, ESM_JSONSCHEMA = get_jsonschema(
url=SCHEMA_URL, known_hash=SCHEMA_HASH, required=CORE_COLUMNS
metadata_file="data/metadata_schema_file.json", required=CORE_COLUMNS
)
Loading

0 comments on commit 2956b7b

Please sign in to comment.