From a7d52c397377f77758f24e9eaf6f35072fd899a2 Mon Sep 17 00:00:00 2001 From: Dougie Squire Date: Mon, 31 Jul 2023 14:01:31 +1000 Subject: [PATCH] update docs on environments --- docs/datastores/builders.rst | 2 +- docs/project_list.rst | 1 + docs/storage_flags.rst | 2 +- docs/usage/how.rst | 39 +++++++++++++------------ src/access_nri_intake/cli.py | 1 + src/access_nri_intake/data/catalog.yaml | 2 +- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/datastores/builders.rst b/docs/datastores/builders.rst index 4a103d60..cb2efac4 100644 --- a/docs/datastores/builders.rst +++ b/docs/datastores/builders.rst @@ -8,7 +8,7 @@ set of Intake-ESM datastore Builders for different ACCESS model outputs. In gene datastore for your ACCESS model output should be as simple as passing your output base directory to an appropriate Builder. -The access-nri-intake package is installed in the :code:`xp65` and :code:`hh5` analysis environments, or +The access-nri-intake package is installed in the :code:`hh5` and :code:`xp65` analysis environments, or users can install it into their own environment (see :ref:`installation` for details). The Builders can be imported from the :code:`access_nri_intake.source.builders` submodule. diff --git a/docs/project_list.rst b/docs/project_list.rst index 5b59336e..83e9fdba 100644 --- a/docs/project_list.rst +++ b/docs/project_list.rst @@ -6,3 +6,4 @@ * :code:`oi10` * :code:`p73` * :code:`rr3` +* :code:`xp65` diff --git a/docs/storage_flags.rst b/docs/storage_flags.rst index d01a52f3..165b38d8 100644 --- a/docs/storage_flags.rst +++ b/docs/storage_flags.rst @@ -1,3 +1,3 @@ .. code-block:: - gdata/al33+gdata/cj50+gdata/dk92+gdata/fs38+gdata/ik11+gdata/oi10+gdata/p73+gdata/rr3 \ No newline at end of file + gdata/al33+gdata/cj50+gdata/dk92+gdata/fs38+gdata/ik11+gdata/oi10+gdata/p73+gdata/rr3+gdata/xp65 \ No newline at end of file diff --git a/docs/usage/how.rst b/docs/usage/how.rst index 315f0dbb..28c28b9f 100644 --- a/docs/usage/how.rst +++ b/docs/usage/how.rst @@ -19,26 +19,30 @@ In order to use the catalog, you will need to have the following: Note you will need to join a project with a compute allocation. If you don't know what project is appropriate you will need to seek help from your local group or IT support. +#. **Access to the** :code:`xp65` **project**: this project houses the catalog table files. See the + `NCI documentation for how to join projects `_. + #. **Access to the projects that house the data you're interested in**: the catalog references data products across multiple projects on Gadi. Currently, data is included from the following projects: .. include:: ../project_list.rst If you wish to be able to access all the data in the catalog, you will need to be a member of all - these projects. See the `NCI documentation for how to join projects - `_. + these projects. .. attention:: Catalog users will only be able to load data from projects that they have access to. -#. **Access to the** :code:`xp65` **or** :code:`hh5` **projects**: these projects provide public - analysis environments in which the ACCESS-NRI catalog is installed (along with many other useful - packages). Alternatively, you can install the catalog into your own environment. +#. **An installation of the catalog**: the catalog is pre-installed in the + `CLEX CMS "analysis3" conda environment `_ and the + ACCESS-NRI "access-med" conda environment. Users are encouraged to use one of these environments to + use the catalog. Alternatively, you can install the catalog into your own environment as described + below. - .. warning:: - The ACCESS-NRI catalog is actually not yet installed in the :code:`hh5` environments, so for now - you'll have to use the :code:`xp65` environment. + .. attention:: + In order to use the CLEX CMS conda environments, you will need to also be a member of the + :code:`hh5` project .. _installation: @@ -46,7 +50,7 @@ Installing the catalog ^^^^^^^^^^^^^^^^^^^^^^ Most users will not need to install the catalog themselves and will instead use the catalog through one -of the public analysis environments provided in either :code:`xp65` or :code:`hh5` (see below). +of the public analysis environments provided in either :code:`hh5` or :code:`xp65` (see below). Advanced users that want to install the catalog into their own environment can do so in three ways: @@ -86,22 +90,21 @@ data it references are available from your session. In particular: * **Setting the storage flags**: in addition to being a member of the projects you want to access, you also have to explicity tell the JupyterLab app that you want to access them in your session. Specify - the project storage paths by entering them in the “Storage” dropdown. To allow access to all data - products in the catalog enter: + the project storage paths by entering them in the “Storage” dropdown. To allow access to the catalog + and all the data products it contains enter: .. include:: ../storage_flags.rst - If you want to use the :code:`xp65` or :code:`hh5` analysis environment, you'll also need to add - :code:`gdata/xp65` or :code:`gdata/hh5`, respectively. + If you want to use the :code:`hh5` analysis environment, you'll also need to add :code:`gdata/hh5`. .. attention:: You need to be a member of all projects you enter here. You can see what projects you are part of at `https://my.nci.org.au/mancini `_. * **Setting the environment**: you need to make sure that the catalog is installed in your JupyterLab - session. As mentioned above, the easiest way to do this is to use either the :code:`xp65` or - :code:`hh5` public analysis environments. You can activate the :code:`xp65` environment within your + session. As mentioned above, the easiest way to do this is to use either the :code:`hh5` or + :code:`xp65` public analysis environments. You can activate the :code:`hh5` environment within your JupyterLab session using the "Advanced options" to set the "Module directories" to - :code:`/g/data/xp65/public/modules` and "Modules" to :code:`conda/are`. Similarly, to use the - :code:`hh5` environment, set "Module directories" to :code:`/g/data/hh5/public/modules` and "Modules" - to :code:`conda/analysis3`. + :code:`/g/data/hh5/public/modules` and "Modules" to :code:`conda/analysis3`. Similarly, to use the + :code:`xp65` environment, set "Module directories" to :code:`/g/data/xp65/public/modules` and "Modules" + to :code:`conda/are`. diff --git a/src/access_nri_intake/cli.py b/src/access_nri_intake/cli.py index 0eca175a..7a405e4a 100644 --- a/src/access_nri_intake/cli.py +++ b/src/access_nri_intake/cli.py @@ -184,6 +184,7 @@ def _get_project(path): project |= set(esm_ds.df["path"].map(_get_project)) project |= {_get_project(path) for path in args["path"]} + project |= {_get_project(build_base_path)} storage_flags = "+".join(sorted([f"gdata/{proj}" for proj in project])) # Build the catalog diff --git a/src/access_nri_intake/data/catalog.yaml b/src/access_nri_intake/data/catalog.yaml index 5f035da9..0b6d6241 100644 --- a/src/access_nri_intake/data/catalog.yaml +++ b/src/access_nri_intake/data/catalog.yaml @@ -13,7 +13,7 @@ sources: description: ACCESS-NRI intake catalog driver: intake_dataframe_catalog.core.DfFileCatalog metadata: - storage: gdata/al33+gdata/cj50+gdata/dk92+gdata/fs38+gdata/ik11+gdata/oi10+gdata/p73+gdata/rr3 + storage: gdata/al33+gdata/cj50+gdata/dk92+gdata/fs38+gdata/ik11+gdata/oi10+gdata/p73+gdata/rr3+gdata/xp65 version: '{{version}}' parameters: version: