diff --git a/scripts/environment.yml b/scripts/environment.yml index f460dac..3cd12a9 100644 --- a/scripts/environment.yml +++ b/scripts/environment.yml @@ -8,7 +8,7 @@ variables: dependencies: - python>=3.9 - pip -- sqlalchemy==1.4.46 ### cosima cookbook does not import with sqlalchemy 2 +- sqlalchemy<2.0 ### cosima cookbook does not import with sqlalchemy 2 - libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability - coecms-nci - cdo>=1.9.5 # pinned for solver stability @@ -38,7 +38,7 @@ dependencies: - arm_pyart - bokeh!=3.0.*,>=2.4.2 ### Dask needs bokeh!=3.0.*,>=2.4.2 for the dashboard. - datashader -- f90nml +- f90nml==1.4.3 - wrf-python>=1.3.4 ### Force update - sympy - sh @@ -85,7 +85,7 @@ dependencies: - hupper - kealib - mo_pack -- mule +- mule==2022.07.1 - cf-units - python-stratify - nbdime @@ -121,7 +121,7 @@ dependencies: - coecms-util - yamanifest - cmocean -- payu>=1.0.27 ### force update +- accessnri::payu - splitvar - marineHeatWaves - holoviews @@ -148,7 +148,7 @@ dependencies: - pymunge - climtas>=0.3.3 # pin to force update - asyncssh -- ants +- ants==0.19.0 - xhistogram - cartopy_userconfig=1.0.gadi - geopy @@ -203,7 +203,7 @@ dependencies: - pymannkendall - param - tensorflow>=2.6 # pinned for solver stability -- shap +- shap<0.42.0 # https://github.com/shap/shap/issues/3075 - gh - pyqt - vtk @@ -275,5 +275,7 @@ dependencies: - odc-geo - cmor - bargeparse +- accessnri::amami +- ipynbname==2023.1.0.0 - pip: - railroad-diagrams ### Unlisted dependency of pip and pyparsing diff --git a/scripts/launcher.sh b/scripts/launcher.sh index 671faf0..63f8b42 100755 --- a/scripts/launcher.sh +++ b/scripts/launcher.sh @@ -20,6 +20,11 @@ source "${conf_file}" declare -a PROG_ARGS=() while [[ $# -gt 0 ]]; do case "${1}" in + "--cms_singularity_overlay_path_override") + ### Sometimes we do not want to use the 'correct' container + export CONTAINER_OVERLAY_PATH_OVERRIDE=1 + shift + ;; "--cms_singularity_overlay_path") ### From time to time we need to manually specify an overlay filesystem, handle that here: export CONTAINER_OVERLAY_PATH="${2}" @@ -65,9 +70,15 @@ fi ### Handle the case where we've been invoked directly. Make sure the container ### we need is on path, and that CONDA_BASE is set so that the right thing ### runs in the container. If we haven't been directly invoked, this does -### nothing +### nothing - Unless told otherwise +### +### Reminder: The --overlay argument that appears LAST takes priority, so put the +### default container first, that way if we're intentionally trying to use it from +### somewhere else (e.g. jobfs), the one on gdata will be mounted but not used. myenv=$( basename "${wrapper_bin%/*}" ".d" ) -[[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh: ]] || export CONTAINER_OVERLAY_PATH="${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh:${CONTAINER_OVERLAY_PATH} +if ! [[ "${CONTAINER_OVERLAY_PATH_OVERRIDE}" ]]; then + [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh: ]] || export CONTAINER_OVERLAY_PATH="${CONDA_BASE_ENV_PATH}"/envs/"${myenv}".sqsh:${CONTAINER_OVERLAY_PATH} +fi export CONDA_BASE="${CONDA_BASE_ENV_PATH}/envs/${myenv}" if ! [[ -x "${SINGULARITY_BINARY_PATH}" ]]; then diff --git a/scripts/overrides/jupyter.config.sh b/scripts/overrides/jupyter.config.sh index 866615b..1283021 100644 --- a/scripts/overrides/jupyter.config.sh +++ b/scripts/overrides/jupyter.config.sh @@ -1,5 +1,6 @@ for i in "${CONDA_BASE_ENV_PATH}"/envs/*.sqsh; do - [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${i}": ]] || CONTAINER_OVERLAY_PATH="${CONTAINER_OVERLAY_PATH}":"${i}" + ### Prepend the other squashfs's - the overlay that appears last takes preference + [[ :"${CONTAINER_OVERLAY_PATH}": =~ :"${i}": ]] || CONTAINER_OVERLAY_PATH="${i}":"${CONTAINER_OVERLAY_PATH}" done ### Strip leading and/or trailing colons CONTAINER_OVERLAY_PATH="${CONTAINER_OVERLAY_PATH#:}" diff --git a/scripts/testconfig.yml b/scripts/testconfig.yml index 92f4851..3b6149c 100644 --- a/scripts/testconfig.yml +++ b/scripts/testconfig.yml @@ -55,6 +55,7 @@ preload: - sklearn.manifold - stratify - xarray +- setuptools.command.build_ext ### Strange issue with numba.pycc # Allow loading, but pass exceptions. When exceptions no longer triggered # can remove