require ensemble argument on cm2 and esm1.5 Builders

ACCESS-NRI · Jul 3, 2023 · 0e592a2 · 0e592a2
1 parent b156392
commit 0e592a2
Show file tree

Hide file tree

Showing 7 changed files with 99 additions and 94 deletions.
diff --git a/config/access-cm2.yaml b/config/access-cm2.yaml
@@ -4,46 +4,48 @@ translator: DefaultTranslator
 
 sources:
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/bx944/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/bx944
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/bx944a
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/bx944b
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/bx944c
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/bx944d
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/bx944/metadata.yaml
     ensemble: true
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/by647/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by647
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by647a
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by647b
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by647c
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by647d
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/by647/metadata.yaml
     ensemble: true
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/by473/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by473
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by473a
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by473b
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by473c
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by473d
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/by473/metadata.yaml
     ensemble: true
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/by578/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by578
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by578a
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by578b
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by578c
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/by578d
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/by578/metadata.yaml
     ensemble: true
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/bz687/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/bz687
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/bz687/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/cj877/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-CM2/cj877
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-CM2/cj877/metadata.yaml
+    ensemble: false
diff --git a/config/access-esm1-5.yaml b/config/access-esm1-5.yaml
@@ -4,43 +4,52 @@ translator: DefaultTranslator
 
 sources:
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2035/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2035
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2035/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2040/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2040
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2040/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2045/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2045
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2045/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2050/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2050
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2050/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2055/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2055
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2055/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2060/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2060
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/PI-GWL-B2060/metadata.yaml
+    ensemble: false
 
-    
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-C-05-r1/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-C-05-r1
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-C-05-r1/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-CN-05/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-CN-05
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-CN-05/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-nl-C-05-r1/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-nl-C-05-r1
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-nl-C-05-r1/metadata.yaml
+    ensemble: false
 
-  - path:
+  - metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-noluc-CN-05/metadata.yaml
+    path:
       - /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-noluc-CN-05
-    metadata_yaml: /g/data/p73/archive/non-CMIP/ACCESS-ESM1-5/HI-noluc-CN-05/metadata.yaml
+    ensemble: false
diff --git a/config/access-om2.yaml b/config/access-om2.yaml
@@ -4,93 +4,87 @@ translator: DefaultTranslator
 
 sources:
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_ryf9091_gadi/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2/1deg_jra55_ryf9091_gadi
-    metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_ryf9091_gadi/metadata.yaml
-
-
-  - path:
+
+  - metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle1/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2/1deg_jra55_iaf_omip2_cycle1
-    metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle1/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle2/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2/1deg_jra55_iaf_omip2_cycle2
-    metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle2/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle3/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2/1deg_jra55_iaf_omip2_cycle3
-    metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle3/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle4/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2/1deg_jra55_iaf_omip2_cycle4
-    metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle4/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle5/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2/1deg_jra55_iaf_omip2_cycle5
-    metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle5/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle6/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2/1deg_jra55_iaf_omip2_cycle6
-    metadata_yaml: /g/data/tm70/intake/metadata/1deg_jra55_iaf_omip2_cycle6/metadata.yaml
-
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_ryf9091_gadi/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-025/025deg_jra55_ryf9091_gadi
-    metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_ryf9091_gadi/metadata.yaml
-
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle1/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-025/025deg_jra55_iaf_omip2_cycle1
-    metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle1/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle2/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-025/025deg_jra55_iaf_omip2_cycle2
-    metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle2/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle3/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-025/025deg_jra55_iaf_omip2_cycle3
-    metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle3/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle4/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-025/025deg_jra55_iaf_omip2_cycle4
-    metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle4/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle5/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-025/025deg_jra55_iaf_omip2_cycle5
-    metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle5/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle6/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-025/025deg_jra55_iaf_omip2_cycle6
-    metadata_yaml: /g/data/tm70/intake/metadata/025deg_jra55_iaf_omip2_cycle6/metadata.yaml
-
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v13_ryf9091/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-01/01deg_jra55v13_ryf9091
-    metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v13_ryf9091/metadata.yaml
 
-    
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf/metadata.yaml
+    path:
       - /g/data/cj50/access-om2/raw-output/access-om2-01/01deg_jra55v140_iaf
-    metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle2/metadata.yaml
+    path:
       - /g/data/cj50/access-om2/raw-output/access-om2-01/01deg_jra55v140_iaf_cycle2
-    metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle2/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle3/metadata.yaml
+    path:
       - /g/data/cj50/access-om2/raw-output/access-om2-01/01deg_jra55v140_iaf_cycle3
-    metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle3/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle4/metadata.yaml
+    path:
       - /g/data/cj50/access-om2/raw-output/access-om2-01/01deg_jra55v140_iaf_cycle4
-    metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle4/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle4_jra55v150_extension/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-01/01deg_jra55v140_iaf_cycle4_jra55v150_extension
-    metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v140_iaf_cycle4_jra55v150_extension/metadata.yaml
-
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v150_iaf_cycle1/metadata.yaml
+    path:
       - /g/data/ik11/outputs/access-om2-01/01deg_jra55v150_iaf_cycle1
-    metadata_yaml: /g/data/tm70/intake/metadata/01deg_jra55v150_iaf_cycle1/metadata.yaml
 
diff --git a/config/cmip5.yaml b/config/cmip5.yaml
@@ -4,10 +4,10 @@ translator: Cmip5Translator
 
 sources:
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/cmip5_al33/metadata.yaml
+    path:
       - /g/data/dk92/catalog/v2/esm/cmip5-al33/catalog.json
-    metadata_yaml: /g/data/tm70/intake/metadata/cmip5_al33/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/cmip5_rr3/metadata.yaml
+    path:
       - /g/data/dk92/catalog/v2/esm/cmip5-rr3/catalog.json
-    metadata_yaml: /g/data/tm70/intake/metadata/cmip5_rr3/metadata.yaml
diff --git a/config/cmip6.yaml b/config/cmip6.yaml
@@ -4,10 +4,10 @@ translator: Cmip6Translator
 
 sources:
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/cmip6_fs38/metadata.yaml
+    path:
       - /g/data/dk92/catalog/v2/esm/cmip6-fs38/catalog.json
-    metadata_yaml: /g/data/tm70/intake/metadata/cmip6_fs38/metadata.yaml
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/cmip6_oi10/metadata.yaml
+    path:
       - /g/data/dk92/catalog/v2/esm/cmip6-oi10/catalog.json
-    metadata_yaml: /g/data/tm70/intake/metadata/cmip6_oi10/metadata.yaml
diff --git a/config/erai.yaml b/config/erai.yaml
@@ -4,6 +4,6 @@ translator: EraiTranslator
 
 sources:
 
-  - path:
+  - metadata_yaml: /g/data/tm70/intake/metadata/erai/metadata.yaml
+    path:
       - /g/data/hh5/public/apps/nci-intake-catalogue/erai/catalogue.json
-    metadata_yaml: /g/data/tm70/intake/metadata/erai/metadata.yaml
diff --git a/src/access_nri_intake/source/builders.py b/src/access_nri_intake/source/builders.py
@@ -267,15 +267,15 @@ def parser(file):
 class AccessEsm15Builder(BaseBuilder):
     """Intake-ESM datastore builder for ACCESS-ESM1.5 datasets"""
 
-    def __init__(self, path, ensemble=False):
+    def __init__(self, path, ensemble):
         """
         Initialise a AccessEsm15Builder
 
         Parameters
         ----------
         path: str or list of str
             Path or list of paths to crawl for assets/files.
-        ensemble: boolean, optional
+        ensemble: boolean
             Whether to treat each path as a separate member of an ensemble to join
             along a new member dimension
         """