Add amami, update ipynbname, ants, mule, shumlib. Change order of ove…

…rlay args to launcher.sh to ensure overrides are allowed

scripts/environment.yml

@@ -8,7 +8,7 @@ variables:
 dependencies:
 - python>=3.9
 - pip
-- sqlalchemy==1.4.46 ### cosima cookbook does not import with sqlalchemy 2
+- sqlalchemy<2.0 ### cosima cookbook does not import with sqlalchemy 2
 - libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
 - coecms-nci
 - cdo>=1.9.5 # pinned for solver stability
@@ -38,7 +38,7 @@ dependencies:
 - arm_pyart
 - bokeh!=3.0.*,>=2.4.2 ### Dask needs bokeh!=3.0.*,>=2.4.2 for the dashboard.
 - datashader
-- f90nml
+- f90nml==1.4.3
 - wrf-python>=1.3.4 ### Force update
 - sympy
 - sh
@@ -85,7 +85,7 @@ dependencies:
 - hupper
 - kealib
 - mo_pack
-- mule
+- mule==2022.07.1
 - cf-units
 - python-stratify
 - nbdime
@@ -121,7 +121,7 @@ dependencies:
 - coecms-util
 - yamanifest
 - cmocean
-- payu>=1.0.27 ### force update
+- accessnri::payu
 - splitvar
 - marineHeatWaves
 - holoviews
@@ -148,7 +148,7 @@ dependencies:
 - pymunge
 - climtas>=0.3.3 # pin to force update
 - asyncssh
-- ants
+- ants==0.19.0
 - xhistogram
 - cartopy_userconfig=1.0.gadi
 - geopy
@@ -203,7 +203,7 @@ dependencies:
 - pymannkendall
 - param
 - tensorflow>=2.6 # pinned for solver stability
-- shap 
+- shap<0.42.0 # https://github.com/shap/shap/issues/3075
 - gh
 - pyqt
 - vtk
@@ -275,5 +275,7 @@ dependencies:
 - odc-geo
 - cmor
 - bargeparse
+- accessnri::amami
+- ipynbname==2023.1.0.0
 - pip:
   - railroad-diagrams ### Unlisted dependency of pip and pyparsing

scripts/launcher.sh

@@ -20,6 +20,11 @@ source "${conf_file}"
 declare -a PROG_ARGS=()
 while [[ $# -gt 0 ]]; do
     case "${1}" in 
+        "--cms_singularity_overlay_path_override")
+            ### Sometimes we do not want to use the 'correct' container
+            export CONTAINER_OVERLAY_PATH_OVERRIDE=1
+            shift
+            ;;
         "--cms_singularity_overlay_path")
             ### From time to time we need to manually specify an overlay filesystem, handle that here:
             export CONTAINER_OVERLAY_PATH="${2}"
@@ -65,9 +70,15 @@ fi
 ### Handle the case where we've been invoked directly. Make sure the container
 ### we need is on path, and that CONDA_BASE is set so that the right thing
 ### runs in the container. If we haven't been directly invoked, this does
-### nothing
+### nothing - Unless told otherwise
+###
+### Reminder: The --overlay argument that appears LAST takes priority, so put the
+### default container first, that way if we're intentionally trying to use it from
+### somewhere else (e.g. jobfs), the one on gdata will be mounted but not used.
 myenv=$( basename "${wrapper_bin%/*}" ".d" )
-[[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh: ]] || export CONTAINER_OVERLAY_PATH="${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh:${CONTAINER_OVERLAY_PATH}
+if ! [[ "${CONTAINER_OVERLAY_PATH_OVERRIDE}" ]]; then
+    [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh: ]] || export CONTAINER_OVERLAY_PATH="${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh:${CONTAINER_OVERLAY_PATH}
+fi
 export CONDA_BASE="${CONDA_BASE_ENV_PATH}/envs/${myenv}"
 
 if ! [[ -x "${SINGULARITY_BINARY_PATH}" ]]; then

scripts/overrides/jupyter.config.sh

@@ -1,5 +1,6 @@
 for i in "${CONDA_BASE_ENV_PATH}"/envs/*.sqsh; do
-    [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${i}": ]] || CONTAINER_OVERLAY_PATH="${CONTAINER_OVERLAY_PATH}":"${i}"
+    ### Prepend the other squashfs's - the overlay that appears last takes preference
+    [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${i}": ]] || CONTAINER_OVERLAY_PATH="${i}":"${CONTAINER_OVERLAY_PATH}"
 done
 ### Strip leading and/or trailing colons
 CONTAINER_OVERLAY_PATH="${CONTAINER_OVERLAY_PATH#:}"

scripts/testconfig.yml

@@ -55,6 +55,7 @@ preload:
 - sklearn.manifold
 - stratify
 - xarray
+- setuptools.command.build_ext ### Strange issue with numba.pycc
 
 # Allow loading, but pass exceptions. When exceptions no longer triggered
 # can remove