Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

185 ship schema with package #189

Merged
merged 7 commits into from
Sep 13, 2024
Merged

185 ship schema with package #189

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b487a34
Incorporated metadata schemas as package data
marc-white Aug 29, 2024
7341436
Updates where old-style data file import was located
marc-white Aug 29, 2024
26f7395
Return $id to metadata files, remove URLs from __init__ files
marc-white Sep 8, 2024
7fcbd55
Helper function for getting catalog.yaml (plus test)
marc-white Sep 9, 2024
0283850
Correct $id values in schema JSON
marc-white Sep 9, 2024
20f6e36
Delete stray comments
marc-white Sep 12, 2024
bf43d6d
Remove even more stray comments
marc-white Sep 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ requires = [
where = ["src"]

[tool.setuptools.package-data]
access_nri_intake = ["data/catalog.yaml"]
access_nri_intake = [
"data/catalog.yaml",
"data/metadata_schema_experiment.json",
"data/metadata_schema_file.json",
]

[tool.versioneer]
VCS = "git"
Expand Down
4 changes: 1 addition & 3 deletions src/access_nri_intake/catalog/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@
NAME_COLUMN = "name"
TRANSLATOR_GROUPBY_COLUMNS = ["model", "realm", "frequency"]

SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/e9055da95093ec2faa555c090fc5af17923d1566/au.org.access-nri/model/output/experiment-metadata/1-0-2.json"
SCHEMA_HASH = "ecb72c1adde3679896ceeca96aa6500d07ea2e05810155ec7a5dc301593c1dc7"

EXP_JSONSCHEMA, CATALOG_JSONSCHEMA = get_jsonschema(
url=SCHEMA_URL, known_hash=SCHEMA_HASH, required=CORE_COLUMNS
metadata_file="data/metadata_schema_experiment.json", required=CORE_COLUMNS
)

COLUMNS_WITH_ITERABLES = [
Expand Down
5 changes: 2 additions & 3 deletions src/access_nri_intake/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from .catalog import EXP_JSONSCHEMA, translators
from .catalog.manager import CatalogManager
from .source import builders
from .utils import _can_be_array, load_metadata_yaml
from .utils import _can_be_array, get_catalog_fp, load_metadata_yaml


class MetadataCheckError(Exception):
Expand Down Expand Up @@ -214,9 +214,8 @@
# Save the catalog
cm.save()

_here = os.path.abspath(os.path.dirname(__file__))
if update:
with open(os.path.join(_here, "data", "catalog.yaml"), "w") as fobj:
with get_catalog_fp().open(mode="w") as fobj:

Check warning on line 218 in src/access_nri_intake/cli.py

View check run for this annotation

Codecov / codecov/patch

src/access_nri_intake/cli.py#L218 

Added line #L218 was not covered by tests
yaml.dump(yaml_dict, fobj)


Expand Down
13 changes: 9 additions & 4 deletions src/access_nri_intake/data/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
# Copyright 2023 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
# SPDX-License-Identifier: Apache-2.0

import os

# import importlib
# import importlib.resources
marc-white marked this conversation as resolved.
Show resolved Hide resolved
import intake

_here = os.path.abspath(os.path.dirname(__file__))
data = intake.open_catalog(os.path.join(_here, "catalog.yaml")).access_nri
from access_nri_intake.utils import get_catalog_fp

data = intake.open_catalog(
# importlib.resources.files("access_nri_intake").joinpath("data/catalog.yaml")
# os.path.join(_here, "catalog.yaml")
get_catalog_fp()
).access_nri
marc-white marked this conversation as resolved.
Show resolved Hide resolved
169 changes: 169 additions & 0 deletions src/access_nri_intake/data/metadata_schema_experiment.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
{
marc-white marked this conversation as resolved.
Show resolved Hide resolved
"$id": "https://raw.githubusercontent.com/ACCESS-NRI/schema/main/au.org.access-nri/model/output/experiment-metadata/1-0-2.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Experiment metadata",
"description": "The metadata associated with a model experiment",
"type": "object",
"properties": {
"schema_version": {
"const": "1-0-2",
"description": "The version of the schema (string)"
},
"name": {
"type": "string",
"description": "The name of the experiment (string)"
},
"experiment_uuid": {
"type": "string",
"format": "uuid",
"description": "Unique uuid for the experiment (string)"
},
"description": {
"type": "string",
"description": "Short description of the experiment (string, < 150 char)"
},
"long_description": {
"type": "string",
"description": "Long description of the experiment (string)"
},
"model": {
"oneOf": [
{"type": ["string", "null"]},
{
"type": "array",
"items": {"type": ["string", "null"]}
}
],
"description": "The name(s) of the model(s) used in the experiment (string)"
},
"realm": {
"type": "array",
"items": {
"oneOf": [
{"type": "null"},
{
"type": "string",
"enum": [
"aerosol",
"atmos",
"atmosChem",
"land",
"landIce",
"none",
"ocean",
"ocnBgchem",
"seaIce",
"unknown",
"wave"
]
}
]
},
"description": "The realm(s) included in the experiment (string)"
},
"frequency": {
"type": "array",
"items": {
"oneOf": [
{"type": "null"},
{
"type": "string",
"oneOf": [
{
"pattern": "^fx$"
},
{
"pattern": "^subhr$"
},
{
"pattern": "^\\d+hr$"
},
{
"pattern": "^\\d+day$"
},
{
"pattern": "^\\d+mon$"
},
{
"pattern": "^\\d+yr$"
},
{
"pattern": "^\\d+dec$"
}
]
}
]
},
"description": "The frequency(/ies) included in the experiment (string)"
},
"variable": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The variable(s) included in the experiment (string)"
},
"nominal_resolution": {
"type": "array",
"items": {"type": ["string", "null"]},
"description": "The nominal resolution(s) of model(s) used in the experiment (string)"
},
"version": {
"type": ["number", "string", "null"],
"description": "The version of the experiment (number, string)"
},
"contact": {
"type": ["string", "null"],
"description": "Contact name for the experiment (string)"
},
"email": {
"type": ["string", "null"],
"description": "Email address of the contact for the experiment (string)"
},
"created": {
"type": ["string", "null"],
"description": "Initial creation date of experiment (string)"
},
"reference": {
"type": ["string", "null"],
"description": "Citation or reference information (string)"
},
"license": {
"type": ["string", "null"],
"description": "License of the experiment (string)"
},
"url": {
"type": ["string", "null"],
"description": "Relevant url, e.g. github repo for experiment configuration (string)"
},
"parent_experiment": {
"type": ["string", "null"],
"description": "experiment_uuid for parent experiment if appropriate (string)"
},
"related_experiments": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "experiment_uuids for any related experiment(s) (string)"
},
"notes": {
"type": ["string", "null"],
"description": "Additional notes (string)"
},
"keywords": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "Keywords to associated with experiment (string)"
}
},
"required": [
"name",
"experiment_uuid",
"description",
"long_description"
],
"additionalProperties": false
}
132 changes: 132 additions & 0 deletions src/access_nri_intake/data/metadata_schema_file.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
{
"$id": "https://raw.githubusercontent.com/ACCESS-NRI/schema/main/au.org.access-nri/model/output/file-metadata/1-0-1.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "File metadata",
"description": "The metadata associated with a file containing or referencing climate model data",
"type": "object",
"properties": {
"schema_version": {
"const": "1-0-1",
"description": "The version of the schema (string)"
},
"path": {
"type": "string",
"description": "The path to the file asset"
},
"realm": {
"type": "string",
"enum": [
"aerosol",
"atmos",
"atmosChem",
"land",
"landIce",
"none",
"ocean",
"ocnBgchem",
"seaIce",
"unknown",
"wave"
],
"description": "The realm of the data in the file asset"
},
"variable": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The variable(s) in the file asset"
},
"variable_long_name": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The long_name(s) of the variable(s) in the file asset"
},
"variable_standard_name": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The standard_names(s) of the variable(s) in the file asset"
},
"variable_cell_methods": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The cell_methods(s) of the variable(s) in the file asset"
},
"variable_units": {
"type": "array",
"items": {
"type": ["string", "null"]
},
"description": "The units of the variable(s) in the file asset"
},
"frequency": {
"type": "string",
"oneOf": [
{
"pattern": "^fx$"
},
{
"pattern": "^subhr$"
},
{
"pattern": "^\\d+hr$"
},
{
"pattern": "^\\d+day$"
},
{
"pattern": "^\\d+mon$"
},
{
"pattern": "^\\d+yr$"
},
{
"pattern": "^\\d+dec$"
}
],
"description": "The frequency of the variable(s) in the file asset"
},
"start_date": {
"type": "string",
"oneOf": [
{
"pattern": "^\\d\\d\\d\\d-\\d\\d-\\d\\d,\\s\\d\\d:\\d\\d:\\d\\d$"
},
{
"pattern": "none"
}
],
"description": "The start date of the variable(s) in the file asset"
},
"end_date": {
"type": "string",
"oneOf": [
{
"pattern": "^\\d\\d\\d\\d-\\d\\d-\\d\\d,\\s\\d\\d:\\d\\d:\\d\\d$"
},
{
"pattern": "none"
}
],
"description": "The end date of the variable(s) in the file asset"
},
"nominal_resolution": {
"type": "string",
"description": "The nominal resolution of the variable(s) in the file asset"
}
},
"required": [
"path",
"realm",
"variable",
"frequency",
"start_date",
"end_date"
]
}
4 changes: 1 addition & 3 deletions src/access_nri_intake/source/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
PATH_COLUMN = "path"
VARIABLE_COLUMN = "variable"

SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/e9055da95093ec2faa555c090fc5af17923d1566/au.org.access-nri/model/output/file-metadata/1-0-1.json"
SCHEMA_HASH = "8f2f069fa06d81ff086b91daa6503f75615aa90385ab61ee2d1a7956dc96f9a6"

_, ESM_JSONSCHEMA = get_jsonschema(
url=SCHEMA_URL, known_hash=SCHEMA_HASH, required=CORE_COLUMNS
metadata_file="data/metadata_schema_file.json", required=CORE_COLUMNS
)
Loading
Loading