Merge pull request #145 from kobotoolbox/upgrades-2021

Major upgrades 2021

.dockerignore

@@ -1 +1,3 @@
 data
+**/node_modules
+**/*.pyc

.github/workflows/docker-hub.yml

@@ -0,0 +1,22 @@
+name: Build and push *base* image to Docker Hub
+
+on:
+  push:
+    # Don't waste time building every push: consider only tags, which are
+    # usually releases
+    tags:
+      - '*'
+
+jobs:
+  docker-base-image-build-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: mr-smithers-excellent/docker-build-push@v5
+        with:
+          image: kobotoolbox/reports_base
+          tags: latest
+          registry: docker.io
+          dockerfile: Dockerfile.base
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}

.gitignore

@@ -9,3 +9,4 @@ koboreports/static/assets
 /media/
 /koboreports/static/login.css
 node_modules
+huey.db*

Dockerfile

@@ -4,18 +4,31 @@ FROM kobotoolbox/reports_base
 # koboreports #
 ###############
 
-COPY . /app/
+# Freshen dependencies in case they've changed since the base
+# image was built
+COPY environment.yml /app/
+COPY jsapp/package.json jsapp/package-lock.json /app/jsapp/
+RUN conda env update --prune
+RUN cd jsapp && npm install
 
-WORKDIR /app/demo
-RUN grunt build
-WORKDIR /app
+# Include all remaining source files except for jsapp/node_modules, which might
+# be peculiar to a developer's host environment
+COPY . /tmp/app/
+RUN (test \! -e /tmp/app/jsapp/node_modules || rm -r /tmp/app/jsapp/node_modules) && \
+    (shopt -s dotglob && cp -a /tmp/app/* /app/ && rm -r /tmp/app)
 
+# Build the front end
+RUN cd jsapp && npm run build
+
+# Run Python unit tests
 RUN source activate koboreports && \
-    python manage.py test --noinput
+    SECRET_KEY=bogus KPI_API_KEY=bogus ALLOWED_HOSTS=bogus python manage.py test --noinput
 
 # Persistent storage of uploaded XLSForms!
+# In production, you should also configure persistent storage in Dokku:
+# https://dokku.com/docs~v0.24.7/advanced-usage/persistent-storage/#persistent-storage
 VOLUME ["/app/media"]
 
 # As of Dokku 0.5.0, no ports should be `EXPOSE`d; see
 # http://dokku.viewdocs.io/dokku/deployment/methods/dockerfiles/#exposed-ports
-CMD ./run.sh # calls `manage.py migrate` and `collectstatic`
+CMD ./run.sh  # calls `manage.py migrate` and `collectstatic`

Dockerfile.base

@@ -1,66 +1,53 @@
-FROM ubuntu:xenial
-
-###########
-# apt-get #
-###########
-
-ADD https://deb.nodesource.com/setup_6.x /tmp/setup_6.x.bash
-
-RUN bash /tmp/setup_6.x.bash && \
-    apt-get install -y --no-install-recommends \
-        build-essential \
-        curl \
-        libgmp10 \
-        libpq-dev \
-        libxrender1 \
-        nodejs \
-        texlive-full \
-        wget
+FROM node:14
+
+# Docker default of `/bin/sh` doesn't support `source`
+SHELL ["/bin/bash", "-c"]
+
+# Add Conda repository
+# https://docs.conda.io/projects/conda/en/latest/user-guide/install/rpm-debian.html
+RUN curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
+    install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
+    echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list
+
+# Install Conda and other OS-level dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    conda \
+    pandoc \
+    libgmp10 \
+    libpq-dev \
+    libxrender1 \
+    texlive-full
 
 # Work around a font rendering problem; see
 # https://github.com/kobotoolbox/reports/issues/136
+# FIXME: figure out what subset of `texlive-full` is actually needed
+# Do not `apt-get autoremove` after this(!) since it would remove necessary
+# packages
 RUN apt-get remove -y tex-gyre
 
-##########
-# pandoc #
-##########
-
-# TODO: Remove --no-check-certificate
-RUN wget --no-check-certificate https://github.com/jgm/pandoc/releases/download/1.15.0.6/pandoc-1.15.0.6-1-amd64.deb -O pandoc.deb && \
-    dpkg -i pandoc.deb && \
-    rm pandoc.deb
-
-##############################
-# conda install Python and R #
-##############################
-
-RUN wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh && \
-    chmod +x miniconda.sh && \
-    ./miniconda.sh -b && \
-    rm miniconda.sh
-ENV PATH /root/miniconda2/bin:$PATH
+# `apt-get install conda` does not actually put Conda on the PATH
+ENV PATH /opt/conda/bin:$PATH
 RUN conda update --yes conda
 
 RUN mkdir /app
 WORKDIR /app
 
-# https://www.continuum.io/content/conda-data-science
+# Copy only the files that define dependencies, not all the source files, to
+# avoid unnecessarily invalidating the Docker layer cache
 COPY environment.yml /app/
-RUN conda env create
+COPY jsapp/package.json jsapp/package-lock.json /app/jsapp/
 
-# http://stackoverflow.com/a/25423366/3756632
-# need this for "source activate" commands
-RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+# Install Python and R dependencies
+RUN conda env create
 
-# R libraries not available through conda
+# Install R libraries not available through Conda
 RUN source activate koboreports && \
-    Rscript -e "install.packages('pander', repos='http://cran.rstudio.com/', type='source')" -e "library(pander)"
+    Rscript -e "install.packages('pander', repos='http://cran.rstudio.com/', type='source')" \
+        -e "library(pander)"
 
-#############################
-# install node dependencies #
-#############################
+# At this time, the `node:14` image includes npm 6, but npm pesters us about
+# upgrading to 7. Oblige it:
+RUN npm install -g npm@7
 
-RUN npm install -g grunt-cli
-COPY demo/package.json /tmp/package.json
-RUN cd /tmp && npm install && mkdir /app/demo && \
-    cp -a /tmp/node_modules /app/demo/
+# Install Node.js dependencies
+RUN cd jsapp && npm install