Merge pull request #144 from iot-salzburg/v1.7_cuda-12.3_ubuntu-22.04

V1.7 cuda 12.3 ubuntu 22.04

CUDA 12.3, cuDNN 9 base image
Update PyTorch, Tensorflow
Update Jupyterlab to 4.1

.build/Dockerfile

@@ -7,7 +7,7 @@
 
 # Use NVIDIA CUDA as base image and run the same installation as in the other packages.
 # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs
-FROM nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04
+FROM nvidia/cuda:12.3.2-cudnn9-runtime-ubuntu22.04
 LABEL authors="Christoph Schranz <[email protected]>, Mathematical Michael <[email protected]>"
 # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary.
 RUN chmod 1777 /tmp && chmod 1777 /var/tmp
@@ -43,22 +43,23 @@ USER root
 # but lacks all features (e.g., download as all possible file formats)
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update --yes && \
-    # - `apt-get upgrade` is run to patch known vulnerabilities in apt-get packages as
-    #   the Ubuntu base image is rebuilt too seldom sometimes (less than once a month)
+    # - `apt-get upgrade` is run to patch known vulnerabilities in system packages
+    #   as the Ubuntu base image is rebuilt too seldom sometimes (less than once a month)
     apt-get upgrade --yes && \
     apt-get install --yes --no-install-recommends \
     # - bzip2 is necessary to extract the micromamba executable.
     bzip2 \
     ca-certificates \
     locales \
     sudo \
-    # - tini is installed as a helpful container entrypoint that reaps zombie
-    #   processes and such of the actual executable we want to start, see
-    #   https://github.com/krallin/tini#why-tini for details.
+    # - `tini` is installed as a helpful container entrypoint,
+    #   that reaps zombie processes and such of the actual executable we want to start
+    #   See https://github.com/krallin/tini#why-tini for details
     tini \
     wget && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
     echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    echo "C.UTF-8 UTF-8" >> /etc/locale.gen && \
     locale-gen
 
 # Configure environment
@@ -67,9 +68,9 @@ ENV CONDA_DIR=/opt/conda \
     NB_USER="${NB_USER}" \
     NB_UID=${NB_UID} \
     NB_GID=${NB_GID} \
-    LC_ALL=en_US.UTF-8 \
-    LANG=en_US.UTF-8 \
-    LANGUAGE=en_US.UTF-8
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    LANGUAGE=C.UTF-8
 ENV PATH="${CONDA_DIR}/bin:${PATH}" \
     HOME="/home/${NB_USER}"
 
@@ -123,34 +124,33 @@ RUN set -x && \
         # Should be simpler, see <https://github.com/mamba-org/mamba/issues/1437>
         arch="64"; \
     fi && \
-    wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \
-        "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \
-    tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \
-    rm /tmp/micromamba.tar.bz2 && \
+    # https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html#linux-and-macos
+    wget --progress=dot:giga -O - \
+        "https://micro.mamba.pm/api/micromamba/linux-${arch}/latest" | tar -xvj bin/micromamba && \
     PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \
     if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \
     # Install the packages
-    ./micromamba install \
+    ./bin/micromamba install \
         --root-prefix="${CONDA_DIR}" \
         --prefix="${CONDA_DIR}" \
         --yes \
         "${PYTHON_SPECIFIER}" \
         'mamba' \
         'jupyter_core' && \
-    rm micromamba && \
+    rm -rf /tmp/bin/ && \
     # Pin major.minor version of python
-    mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
+    # https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#preventing-packages-from-updating-pinning
+    mamba list --full-name 'python' | tail -1 | tr -s ' ' | cut -d ' ' -f 1,2 | sed 's/\.[^.]*$/.*/' >> "${CONDA_DIR}/conda-meta/pinned" && \
     mamba clean --all -f -y && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
 
-# Configure container startup
-ENTRYPOINT ["tini", "-g", "--"]
-CMD ["start.sh"]
-
 # Copy local files as late as possible to avoid cache busting
 COPY run-hooks.sh start.sh /usr/local/bin/
 
+# Configure container entrypoint
+ENTRYPOINT ["tini", "-g", "--", "start.sh"]
+
 USER root
 
 # Create dirs for startup hooks
@@ -181,16 +181,18 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 USER root
 
-# Install all OS dependencies for the Server that starts but lacks all
-# features (e.g., download as all possible file formats)
+# Install all OS dependencies for the Server that starts
+# but lacks all features (e.g., download as all possible file formats)
 RUN apt-get update --yes && \
     apt-get install --yes --no-install-recommends \
+    # - Add necessary fonts for matplotlib/seaborn
+    #   See https://github.com/jupyter/docker-stacks/pull/380 for details
     fonts-liberation \
-    # - pandoc is used to convert notebooks to html files
+    # - `pandoc` is used to convert notebooks to html files
     #   it's not present in the aarch64 Ubuntu image, so we install it here
     pandoc \
-    # - run-one - a wrapper script that runs no more
-    #   than one unique  instance  of  some  command with a unique set of arguments,
+    # - `run-one` - a wrapper script that runs no more
+    #   than one unique instance of some command with a unique set of arguments,
     #   we use `run-one-constantly` to support the `RESTARTABLE` option
     run-one && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
@@ -234,7 +236,7 @@ RUN fix-permissions /etc/jupyter/
 # HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck
 # This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands
 # https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799
-HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
+HEALTHCHECK --interval=3s --timeout=1s --start-period=3s --retries=3 \
     CMD /etc/jupyter/docker_healthcheck.py || exit 1
 
 # Switch back to jovyan to avoid accidental container runs as root
@@ -271,10 +273,10 @@ RUN apt-get update --yes && \
     vim-tiny \
     # git-over-ssh
     openssh-client \
-    # less is needed to run help in R
+    # `less` is needed to run help in R
     # see: https://github.com/jupyter/docker-stacks/issues/1588
     less \
-    # nbconvert dependencies
+    # `nbconvert` dependencies
     # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex
     texlive-xetex \
     texlive-fonts-recommended \
@@ -383,36 +385,36 @@ WORKDIR "${HOME}"
 
 LABEL maintainer="Christoph Schranz <[email protected]>, Mathematical Michael <[email protected]>"
 
+# Install dependencies for e.g. PyTorch
+RUN mamba install --quiet --yes \
+    pyyaml setuptools cmake cffi typing && \
+    mamba clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
 # Install Tensorflow, check compatibility here:
 # https://www.tensorflow.org/install/source#gpu
 # installation via conda leads to errors in version 4.8.2
 # Install CUDA-specific nvidia libraries and update libcudnn8 before that
 # using device_lib.list_local_devices() the cudNN version is shown, adapt version to tested compat
 USER ${NB_UID}
 RUN pip install --upgrade pip && \
-    pip install --no-cache-dir tensorflow==2.15.0 keras==2.15.0 && \
-    fix-permissions "${CONDA_DIR}" && \
-    fix-permissions "/home/${NB_USER}"
-
-# Install PyTorch with dependencies
-RUN mamba install --quiet --yes \
-    pyyaml setuptools cmake cffi typing && \
-    mamba clean --all -f -y && \
+    pip install --no-cache-dir tensorflow==2.16.1 keras==3.1.1 && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
 
 # Check compatibility here:
 # https://pytorch.org/get-started/locally/
 # Installation via conda leads to errors installing cudatoolkit=11.1
-# RUN pip install --no-cache-dir torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0  && \
-#     torchviz==0.0.2 --extra-index-url https://download.pytorch.org/whl/cu118
+# RUN pip install --no-cache-dir torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 \
+#  && torchviz==0.0.2 --extra-index-url https://download.pytorch.org/whl/cu121
 RUN set -ex \
  && buildDeps=' \
-    torch==2.1.2 \
-    torchvision==0.16.2 \
-    torchaudio==2.1.2 \
+    torch==2.2.2 \
+    torchvision==0.17.2 \
+    torchaudio==2.2.2 \
 ' \
- && pip install --no-cache-dir $buildDeps \
+ && pip install --no-cache-dir $buildDeps  --extra-index-url https://download.pytorch.org/whl/cu121 \
  && fix-permissions "${CONDA_DIR}" \
  && fix-permissions "/home/${NB_USER}"
 
@@ -441,7 +443,8 @@ RUN pip install --no-cache-dir nvidia-pyindex && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
 
-RUN mamba install -c nvidia cuda-nvcc=12.0.140 -y && \
+# Install cuda-nvcc with sepecific version, see here: https://anaconda.org/nvidia/cuda-nvcc/labels
+RUN mamba install -c nvidia cuda-nvcc=12.3.107 -y && \
     mamba clean --all -f -y && \
     fix-permissions $CONDA_DIR && \
     fix-permissions /home/$NB_USER
@@ -467,39 +470,42 @@ RUN apt-get update \
 USER $NB_UID
 RUN set -ex \
  && buildDeps=' \
-    graphviz==0.19.1 \
-    pytest==7.2.2 \
-' \
+    graphviz==0.20.3 \
+    pytest==8.1.1 \
+ ' \
  && pip install --no-cache-dir $buildDeps \
  && fix-permissions "${CONDA_DIR}" \
  && fix-permissions "/home/${NB_USER}"
 
 # upgrade jupyter-server for compatibility
-RUN pip install --no-cache-dir --upgrade \
-    distributed==2023.3.0 \
-    jupyter-server==2.4 \
-    # fix permissions of conda
+RUN set -ex \
+ && buildDeps=' \
+    distributed==2024.4.1 \
+    jupyter-server==2.13 \
+ ' \
+ && pip install --no-cache-dir $buildDeps \
  && fix-permissions "${CONDA_DIR}" \
  && fix-permissions "/home/${NB_USER}"
 
-RUN pip install --no-cache-dir \
+RUN set -ex \
+ && buildDeps=' \
     # install extension manager
     jupyter_contrib_nbextensions==0.7.0 \
-    jupyter_nbextensions_configurator==0.6.1 \
+    jupyter_nbextensions_configurator==0.6.3 \
     # install git extension
-    jupyterlab-git==0.41.0 \
+    jupyterlab-git==0.50.0 \
     # install plotly extension
-    plotly==5.13.1 \
-    # install drawio and graphical extensions
-    jupyterlab-drawio==0.9.0 \
-    rise==5.7.1 \
-    ipyleaflet==0.17.2 \
-    ipywidgets==8.0.4 \
+    plotly==5.20.0 \
+    # install drawio and graphical extensions, not compatible with Jupyterlab 4.X yet
+    # ipydrawio==1.3.0 \
+    ipyleaflet==0.18.2 \
+    ipywidgets==8.1.2 \
     # install spell checker
-    jupyterlab-spellchecker==0.7.3 && \
-    # fix permissions of conda
-    fix-permissions "${CONDA_DIR}" && \
-    fix-permissions "/home/${NB_USER}"
+    jupyterlab-spellchecker==0.8.4 \
+    ' \
+    && pip install --no-cache-dir $buildDeps \
+    && fix-permissions "${CONDA_DIR}" \
+    && fix-permissions "/home/${NB_USER}"
 
 # Switch back to jovyan to avoid accidental container runs as root
 USER $NB_UID

.build/docker_healthcheck.py

@@ -3,14 +3,27 @@
 # Distributed under the terms of the Modified BSD License.
 import json
 import os
+import subprocess
 from pathlib import Path
 
 import requests
 
 # Several operations below deliberately don't check for possible errors
 # As this is a healthcheck, it should succeed or raise an exception on error
 
-runtime_dir = Path("/home/") / os.environ["NB_USER"] / ".local/share/jupyter/runtime/"
+# Docker runs healtchecks using an exec
+# It uses the default user configured when running the image: root for the case of a custom NB_USER or jovyan for the case of the default image user.
+# We manually change HOME to make `jupyter --runtime-dir` report a correct path
+# More information: <https://github.com/jupyter/docker-stacks/pull/2074#issuecomment-1879778409>
+result = subprocess.run(
+    ["jupyter", "--runtime-dir"],
+    check=True,
+    capture_output=True,
+    text=True,
+    env=dict(os.environ) | {"HOME": "/home/" + os.environ["NB_USER"]},
+)
+runtime_dir = Path(result.stdout.rstrip())
+
 json_file = next(runtime_dir.glob("*server-*.json"))
 
 url = json.loads(json_file.read_bytes())["url"]

.build/setup-scripts/setup-julia-packages.bash

@@ -3,28 +3,28 @@ set -exuo pipefail
 # Requirements:
 # - Run as a non-root user
 # - The JULIA_PKGDIR environment variable is set
-# - Julia is already set up, with the setup-julia.bash command
+# - Julia is already set up, with the setup_julia.py command
 
 
 # If we don't specify what CPUs the precompilation should be done for, it's
 # *only* done for the target of the host doing the compilation.  When the
 # container runs on a host that's the same architecture, but a *different*
 # generation of CPU than what the build host was, the precompilation is useless
 # and Julia takes a long long time to start up. This specific multitarget comes
-# from https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L20-L76,
+# from https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L20-L82,
 # and may need to be updated as new CPU generations come out.
 # If the architecture the container runs on is different,
 # precompilation may still have to be re-done on first startup - but this
 # *should* catch most of the issues.  See
 # https://github.com/jupyter/docker-stacks/issues/2015 for more information
 if [ "$(uname -m)" == "x86_64" ]; then
-    # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L24
+    # See https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L23
     # for an explanation of these options
-    export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)"
+    export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1)"
 elif [ "$(uname -m)" == "aarch64" ]; then
-    # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L54
+    # See https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L56
     # for an explanation of these options
-    export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel"
+    export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel,clone_all;apple-m1,base(3);neoverse-512tvb,base(3)"
 fi
 
 # Install base Julia packages