Merge pull request #38 from coecms/nri-intake

Add access-nri-intake. Add code that updates support scripts, configs…
coecms · Jul 28, 2023 · 1b89bff · 1b89bff
2 parents 44f5e70 + e2f334d
commit 1b89bff
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 20 deletions.
diff --git a/modules/are b/modules/are
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -75,6 +75,14 @@ function inner() {
     done
     popd
 
+    ### Update any supporting infrastructure
+    copy_if_changed "${SCRIPT_DIR}"/launcher.sh "${CONDA_SCRIPT_PATH}"/launcher.sh
+    for override in "${SCRIPT_DIR}"/overrides/*; do
+        copy_if_changed "${override}" "${CONDA_SCRIPT_PATH}"/overrides/"${override##*/}"
+    done
+    copy_and_replace_if_changed "${SCRIPT_DIR}"/../modules/common_v3 "${CONDA_MODULE_PATH}"/.common_v3       CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME SCRIPT_SUBDIR
+    copy_and_replace_if_changed "${SCRIPT_DIR}"/launcher_conf.sh     "${CONDA_SCRIPT_PATH}"/launcher_conf.sh CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME
+
     ### Create symlink tree
     mkdir -p "${CONDA_SCRIPT_PATH}"/"${FULLENV}".d/{bin,overrides}
     cp "${CONDA_SCRIPT_PATH}"/{launcher.sh,launcher_conf.sh} "${CONDA_SCRIPT_PATH}"/"${FULLENV}".d/bin

diff --git a/scripts/environment.yml b/scripts/environment.yml
@@ -30,7 +30,7 @@ dependencies:
 - seaborn
 - xarray
 - jupyter
-- notebook
+- notebook<7 ### Pin for now as jupyter_contrib_nbextensions not compatible with notebook 7
 - nb_conda
 - metpy
 - windspharm
@@ -180,7 +180,7 @@ dependencies:
 - geoplot
 - dask-ml>=2022.5.27
 - mamba
-- intake-esm==2023.1.6  #<2022.9.18  # downgrading until intake 0.6.6 issue solved
+- intake-esm==2023.07.07 
 - pykrige
 - nci_intake_catalogue>=1.3 # triggering update 
 - cmdline_provenance
@@ -276,6 +276,7 @@ dependencies:
 - cmor
 - bargeparse
 - accessnri::amami
+- accessnri::access-nri-intake
 - ipynbname==2023.1.0.0
 - pip:
   - railroad-diagrams ### Unlisted dependency of pip and pyparsing
diff --git a/scripts/functions.sh b/scripts/functions.sh
@@ -170,4 +170,31 @@ function copy_and_replace() {
         cp "${in}" "${out}"
     fi
 
+}
+
+function copy_if_changed() {
+    ### Copy "${1}" to "${2}", but only if "${1}" has changed
+    in="${1}"
+    out="${2}"
+
+    if ! diff -q "${in}" "${out}" > /dev/null; then
+        echo "Copying updated ${in##*/}"
+        cp "${in}" "${out}"
+    fi
+}
+
+function copy_and_replace_if_changed() {
+    ### copy_and_replace as above, but only if the end result would be
+    ### different
+    in="${1}"
+    final_out="${2}"
+    shift 2
+    out=$( mktemp -p /tmp )
+    trap 'rm -f "${out}"' EXIT
+    copy_and_replace "${in}" "${out}" "${@}"
+    if ! diff -q "${final_out}" "${out}" > /dev/null; then
+        echo "Copying updated ${in##*/}"
+        mv "${out}" "${final_out}"
+    fi
+
 }
diff --git a/scripts/initialise.sh b/scripts/initialise.sh
@@ -27,9 +27,8 @@ function inner() {
 
     mkdir -p "${CONDA_MODULE_PATH}"
     ### These files contain hard-coded paths to the conda installation - these paths are created with variables set by install_config.sh
-    copy_and_replace "${SCRIPT_DIR}"/../modules/common_v3 "${CONDA_MODULE_PATH}"/.common_v3 CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME SCRIPT_SUBDIR
-    copy_and_replace "${SCRIPT_DIR}"/../modules/are       "${CONDA_MODULE_PATH}"/are        CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME
-    copy_and_replace "${SCRIPT_DIR}"/launcher_conf.sh     "${CONDA_SCRIPT_PATH}"/launcher_conf.sh            CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME
+    copy_and_replace "${SCRIPT_DIR}"/../modules/common_v3 "${CONDA_MODULE_PATH}"/.common_v3       CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME SCRIPT_SUBDIR
+    copy_and_replace "${SCRIPT_DIR}"/launcher_conf.sh     "${CONDA_SCRIPT_PATH}"/launcher_conf.sh CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME
 
     conda clean -a -f -y
 

diff --git a/scripts/testconfig.yml b/scripts/testconfig.yml
@@ -34,6 +34,7 @@ skip:
 - pyparsing
 - tensorflow_estimator
 - acs_replica_intake  # can't load catalogue file
+- access_nri_intake  # can't load catalogue file
 - pip._vendor.chardet # encounters a null byte, doesn't seem to be an error
 - pykrige # AttributeError: module 'scipy.linalg' has no attribute 'pinv2' -- rk4417
 # for both hdbscan and cosima_cookbook see https://accessdev.nci.org.au/jenkins/job/conda/job/analysis3-unstable/1351/consoleFull