Merge remote-tracking branch 'origin/main' into grader

.github/workflows/lint.yml

@@ -1,6 +1,10 @@
 name: Ansible Lint
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main, grader]
 
 jobs:
   build:
@@ -13,4 +17,4 @@ jobs:
     - name: Deps
       run: pip install -r requirements.txt
     - name: Lint Ansible Playbook
-      run: ansible-lint --force-color research-cloud-plugin.yml shared-data-disk.yml
+      run: ansible-lint --profile min --force-color research-cloud-plugin.yml shared-data-disk.yml

README.md

@@ -82,7 +82,8 @@ Importantly:
 
 - Work on a folder on the windows file system.
 - Export VAGRANT_WSL_WINDOWS_ACCESS_USER_HOME_PATH="/mnt/c/.../infra"
-- Install [virtualbox_WSL2 vagrant plugin](https://github.com/Karandash8/virtualbox_WSL2)
+- `export PATH="$PATH:C:\Program Files\Oracle\VirtualBox"`
+- ` vagrant up --provider virtualbox`
 - Approve the firewall popup
 
 ## Catalog item registration

Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "generic/ubuntu2004"
+  config.vm.box = "bento/ubuntu-22.04"
   config.vm.synced_folder ".", "/vagrant"
 
   # Create a public network, which generally matched to bridged network.

roles/common/tasks/main.yml

@@ -1,9 +1,9 @@
 ---
 - name: Hostname
-  include_tasks: hostname.yml
+  ansible.builtin.include_tasks: hostname.yml
 - name: Firewall
-  include_tasks: firewall.yml
+  ansible.builtin.include_tasks: firewall.yml
 - name: Ppa
-  include_tasks: ppa.yml
+  ansible.builtin.include_tasks: ppa.yml
 - name: Pip3
-  include_tasks: pip3.yml
+  ansible.builtin.include_tasks: pip3.yml

roles/ewatercycle/defaults/main.yml

@@ -19,4 +19,4 @@ apptainer_image_root: '{{ data_root }}/singularity-images'
 home_root: /home
 # Version of ewatercycle Python package
 # /etc/ewatercycle.yaml will be slightly different depending on version.
-pyewatercycle_version: 2.2.0
+pyewatercycle_version: 2.3.0

roles/ewatercycle/tasks/main.yml

@@ -1,6 +1,20 @@
 ---
 # tasks file for ewatercycle
 
+- name: Install conda-lock in base conda env
+  ansible.builtin.pip:
+    name: conda-lock
+    executable: '{{ conda_root }}/bin/pip3'
+- name: eWaterCycle conda lock
+  ansible.builtin.get_url:
+    url: 'https://github.com/eWaterCycle/ewatercycle/raw/{{ pyewatercycle_version }}/conda-lock.yml'
+    dest: '{{ conda_root }}/conda-lock.yml'
+    mode: '0644'
+- name: Created conda environment with locked ewatercycle dependencies
+  command: '{{ conda_root }}/bin/conda-lock install --no-dev -p {{ conda_root }}/envs/{{ conda_environment }} {{ conda_root }}/conda-lock.yml'  # noqa no-changed-when
+  args:
+    creates:
+      '{{ conda_root }}/envs/{{ conda_environment }}'
 - name: Stage conda env file
   template:
     src: environment.yml.j2

roles/ewatercycle/templates/environment.yml.j2

@@ -1,87 +1,46 @@
 ---
+# pinned versions so future deployments have more chance to work
 name: ewatercycle
 channels:
   - conda-forge
   - defaults
 dependencies:
-  # TODO pin versions for reproducibility. Use `conda env export` and `pip list` to get versions
-  - python=3.10
-  - esmvaltool-python>=2.3.0
-  - ansible
   - pip:
       - ewatercycle=={{ pyewatercycle_version }}
-      - ewatercycle-hype
-      - ewatercycle-lisflood
-      - ewatercycle-marrmot
-      - ewatercycle-pcrglobwb
-      - ewatercycle-wflow
-      - ewatercycle-leakybucket
-      # TODO Once ewatercycle-wflowjl uses a containerized model, uncomment the following line:
-      # - ewatercycle-wflowjl
-      # TODO Once available uncomment the following line
-      # - git+https://github.com/eWaterCycle/ewatercycle-sfincs
-      - era5cli
-      # Latest published nbgrader does not work (validate button gives error) with latest jupyterlab,notebook
-      # So install bleeding edge version from github
-      - git+https://github.com/jupyter/nbgrader.git@9876829b6be4fccce621ffee907bd5f9ac400c44
-  - setuptools
-  - altair_saver
-  - altair
-  - dask-jobqueue
-  - distributed
-  - geoviews
-  - hydrostats
-  - nco
-  - ncview
-  - regionmask
-  - pyke  # required by iris, only available on conda or sourceforge
-  - iris-sample-data # for sample notebooks, see https://github.com/eWaterCycle/notebooks/issues/15
+      - ewatercycle-hype==0.0.1
+      - ewatercycle-lisflood==0.0.1
+      - ewatercycle-marrmot==0.0.2
+      - ewatercycle-pcrglobwb==0.0.2
+      - ewatercycle-wflow==0.0.2
+      - ewatercycle-leakybucket==0.0.2
+      - ewatercycle-HBV==1.8.4
+      # TODO ewatercycle-HBV should not depend on HBV, but currently it does
+      - HBV==1.5.2
+      # Conda version of jupyterhub-systemdspawner is not py3.12 compat, so use pip version
+      - jupyterhub-systemdspawner==1.0.1
+      # Conda version of papermill is not py3.12 compat, so use pip version
+      - papermill==2.6.0
   # Pangeo deps, see https://pangeo.io/setup_guides/hpc.html
-  - zarr
-  - hvplot
-  - numcodecs
-  # Lint / formatters
-  - autopep8
-  - black
-  - flake8
-  - isort
-  - pylint
-  - yapf
-  # Pymt
-  - pymt_cem
-  - pymt_ecsimplesnow
-  - pymt_gipl
-  - pymt_hydrotrend
-  - pymt_landlab
-  - pymt_permamodel
-  - pymt_rafem
-  - pymt_sedflux
-  - pymt
-  # - pymt_child  # Skipped as only a osx binary is available
-  # - pymt_topoflow  # Skipped not on conda-forge
+  # TODO add pangeo deps back?
+
   # Jupyter
-  - jupyter-server-proxy
-  - jupyterhub
-  - jupyterlab # Version pinned due to not all wanted extensions support v3
-  - notebook
-  - ipykernel
-  - ipympl
-  - ipynb-py-convert
-  - jupyterhub-idle-culler
-  - jupyterhub-systemdspawner
+  - jupyter-server-proxy=4.4.0
+  - jupyterhub=5.1.0
+  - jupyterlab=4.2.5
+  - notebook=7.2.2
+  - ipykernel=6.29.5
+  - ipympl=0.9.4
+  - ipynb-py-convert=0.4.6
+  - jupyterhub-idle-culler=1.3.1
+  - nodejs=22.7.0
   # JupyterLab extensions server dependencies, lab extension should be added to defaults of jupyter role
-  - ipyleaflet
-  - jupyterlab_code_formatter
-  - dask-labextension
-  - jupyterlab-geojson
-  - jupyter_bokeh
-  - pyviz_comms
-  - jupyterlab-git
-  - nbdime
-  - nbgitpuller
-  - papermill
-  # nbgrader deps
-  - rapidfuzz
-  - nbconvert
-  - nbconvert-pandoc
-  - pandoc
+  - jupyterlab_code_formatter=3.0.2
+  - black=24.8.0
+  - isort=5.13.2
+  - dask-labextension=7.0.0
+  - jupyterlab-geojson=3.4.0
+  - ipyleaflet=0.19.2
+  - jupyterlab-git=0.50.1
+  - nbdime=4.0.1
+  - nbgitpuller=1.2.1
+  - nbgrader=0.9.3

roles/jupyter/tasks/main.yml

@@ -38,7 +38,7 @@
     name: configurable-http-proxy
     executable: '{{ conda_environment_bin }}/npm'
   environment:
-        PATH: "{{ conda_environment_bin }}:{{ ansible_env.PATH }}"
+    PATH: "{{ conda_environment_bin }}:{{ ansible_env.PATH }}"
 - name: USGS data home directory
   file:
     path: '{{ jupyterhub_spawner_environment.USGS_DATA_HOME }}'

roles/labstart/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 tool_urls:
-  explore: /
+  # explore: /
   jupyter: /jupyter
-  analytics: https://analytics.ewatercycle.org
-  experiments: https://experiments.ewatercycle.org
-  lowresforecast: https://lowres-forecast.ewatercycle.org
+  # analytics: https://analytics.ewatercycle.org
+  # experiments: https://experiments.ewatercycle.org
+  # lowresforecast: https://lowres-forecast.ewatercycle.org

roles/labstart/templates/index.html.j2

@@ -15,6 +15,7 @@
     body {
         padding: 25px;
     }
+
     .card .card-img-top {
         max-height: 10rem;
         max-width: 10rem;
@@ -60,7 +61,7 @@
 
         </div>
         <div class="card-deck">
-        <!--
+            {%- if 'explore' in tool_urls -%}
             <div class="card" style="width: 18rem;">
                 <img src="images/explore.png" class="card-img-top mx-auto" alt="...">
                 <div class="card-body">
@@ -70,17 +71,32 @@
                     <a href="{{ tool_urls.explore }}" class="btn btn-primary btn-block">Explore</a>
                 </div>
             </div>
-            -->
+            {%- endif %}
             <div class="card" style="width: 18rem;">
                 <img src="images/jupyter.png" class="card-img-top mx-auto" alt="...">
                 <div class="card-body">
                     <p class="card-text">Experiment with models and datasets in a notebook.</p>
                 </div>
                 <div class="card-footer text-black bg-white border-white">
-                    <a href="{{ tool_urls.jupyter }}" class="btn btn-primary btn-block">Jupyter notebook</a>
+                    <a href="{{ tool_urls.jupyter }}" class="btn btn-primary btn-block">Jupyter</a>
+                </div>
+            </div>
+                        <div class="card" style="width: 18rem;">
+                <img src="images/analytics.png" class="card-img-top mx-auto" alt="...">
+                <div class="card-footer text-black bg-white border-white">
+                    <a href="/jupyter/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FeWaterCycle%2Fewatercycle.git&urlpath=lab%2Ftree%2Fewatercycle.git%2Fdocs%2Fuser_guide%2F00_intro.ipynb&branch=main" class="btn btn-primary btn-block">User guide notebook</a>
                 </div>
-                <!--
-            </div>{%- if 'analytics' in tool_urls -%}
+            </div>
+                        <div class="card" style="width: 18rem;">
+                <img src="images/cylc.png" class="card-img-top mx-auto" alt="...">
+                <div class="card-body">
+                    <p class="card-text">Tech paper notebooks.</p>
+                </div>
+                <div class="card-footer text-black bg-white border-white">
+                    <a href="/jupyter/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FeWaterCycle%2FtechnicalPaperExampleNotebooks&urlpath=lab%2Ftree%2FtechnicalPaperExampleNotebooks%2FCase1_Marrmot_Merrimack_ESMValTool.ipynb&branch=main" class="btn btn-primary btn-block">Jupyter notebook</a>
+                </div>
+            </div>
+            {%- if 'analytics' in tool_urls -%}
             <div class="card" style="width: 18rem;">
                 <img src="images/analytics.png" class="card-img-top mx-auto" alt="...">
                 <div class="card-body">
@@ -112,6 +128,21 @@
             </div>
             -->
         </div>
+            <div>
+                <h3>Configuration for teachbooks</h3>
+                <p>To open books on this machine, you can add the following configuration in the <b>_config.yml</b> file of your teachbook repository:</p>
+                <pre id="teachbook-config">
+                    # rest of configuration
+
+                    launch_buttons:
+                        jupyterhub_url: /jupyter/hub
+                </pre>
+                <script>
+                // Make url absolute
+                const config = document.getElementById('teachbook-config');
+                config.textContent = config.textContent.replace('/jupyter/hub', window.location.origin + '/jupyter/hub');
+                </script>
+             </div>
     </div>
 </body>
 

roles/prep_shared_data/tasks/main.yml

@@ -1,11 +1,11 @@
 ---
 - name: Download climate data
-  include_tasks: climate-data.yml
+  ansible.builtin.include_tasks: climate-data.yml
 - name: ESMValTool aux data
-  include_tasks: esmvaltool-aux-data.yml
+  ansible.builtin.include_tasks: esmvaltool-aux-data.yml
 - name: Build apptainer image files (sif) for each model
-  include_tasks: apptainer-images.yml
+  ansible.builtin.include_tasks: apptainer-images.yml
 - name: Download example parameter sets
-  include_tasks: example-parameter-sets.yml
+  ansible.builtin.include_tasks: example-parameter-sets.yml
 - name: Download example forcing
-  include_tasks: example-forcing.yml
+  ansible.builtin.include_tasks: example-forcing.yml

roles/rclone/tasks/mount.yml

@@ -1,6 +1,6 @@
 ---
 - name: Install
-  include_tasks: install.yml
+  ansible.builtin.include_tasks: install.yml
 - name: Configure rclone
   copy:
     dest: /etc/rclone.conf

roles/rclone/tasks/upload.yml

@@ -1,6 +1,6 @@
 ---
 - name: Install
-  include_tasks: install.yml
+  ansible.builtin.include_tasks: install.yml
 - name: Config directory
   file:
     path: ~/.config/rclone