squashing docker int commits

.github/workflows/docker-publish.yaml

@@ -0,0 +1,55 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name:                 Publish Docker image
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registries:
+    name:             Push Docker image to multiple registries
+    runs-on:          ubuntu-latest
+    permissions:
+      packages:       write
+      contents:       read
+    steps:
+      - name:         Check out the repo
+        uses:         actions/checkout@v3
+
+      - name:         Log in to Docker Hub
+        uses:         docker/login-action@v3
+        with:
+          username:   ${{ secrets.DOCKER_USERNAME }}
+          password:   ${{ secrets.DOCKER_TOKEN }}
+
+      - name:         Log in to the Container registry
+        uses:         docker/login-action@v3
+        with:
+          registry:   ghcr.io
+          username:   ${{ github.actor }}
+          password:   ${{ secrets.GITHUB_TOKEN }}
+
+      - name:         Extract metadata (tags, labels) for Docker
+        id:           meta
+        uses:         docker/metadata-action@v5
+        with:
+          images:     |
+            ${{ secrets.DOCKER_USERNAME }}/${{ github.event.repository.name }}
+            ghcr.io/${{ github.repository }}
+
+      - name:         Build and push Docker images
+        uses:         docker/build-push-action@v5
+        with:
+          context:    .
+          push:       true
+          tags:       ${{ steps.meta.outputs.tags }}
+          labels:     ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -0,0 +1,65 @@
+# syntax=docker/dockerfile:1
+
+FROM nvcr.io/nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04
+
+ENV MAMBA_ROOT_PREFIX=/root/micromamba
+
+# install micromamba and other Linux tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  bc \
+  build-essential \
+  bzip2 \
+  git \
+  wget \
+  ca-certificates \
+  && wget -qO-  https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba \
+  && touch /root/.bashrc \
+  && /bin/micromamba shell init -s bash -p /opt/conda \
+  && grep -v '[ -z "\$PS1" ] && return' /root/.bashrc  > /opt/conda/bashrc \
+  && apt-get clean autoremove --yes \
+  && rm -rf /var/lib/{apt,dpkg,cache,log}
+
+SHELL ["bash", "-l" ,"-c"]
+
+# create SPHIRE-crYOLO Conda environment and download pre-trained model
+RUN micromamba create -n cryolo -c conda-forge pyqt=5 python=3.7 cudatoolkit=10.0.130 cudnn=7.6.5 numpy==1.18.5 libtiff wxPython=4.1.1 \
+  && eval "$(micromamba shell hook --shell bash)" \
+  && micromamba activate cryolo \
+  && pip install 'cryolo[gpu]' \
+  && micromamba deactivate \
+  && micromamba clean --all -y \
+  && wget ftp://ftp.gwdg.de/pub/misc/sphire/crYOLO-GENERAL-MODELS/gmodel_phosnet_202005_N63_c17.h5
+
+# create Topaz Conda environment
+RUN micromamba create -n topaz -c conda-forge -c tbepler -c pytorch python=3.6 pytorch=*=*cuda10.1* topaz=0.2.5 cudatoolkit=10.1.243 \
+  && micromamba clean --all -y
+
+# create REPIC Conda environment
+RUN micromamba create -n repic -c conda-forge python=3.8 \
+  && eval "$(micromamba shell hook --shell bash)" \
+  && micromamba activate repic \
+  && git clone https://github.com/ccameron/REPIC.git \
+  && cd REPIC \
+  && pip install . \
+  && cd .. \
+  && micromamba deactivate \
+  && micromamba clean --all -y \
+  && rm -rf REPIC/
+
+# download DeepPicker GitHub repo and set up Conda environment
+RUN git clone https://github.com/nejyeah/DeepPicker-python.git \
+  && eval "$(micromamba shell hook --shell bash)" \
+  && micromamba activate repic \
+  && cp $(pip show repic | grep -in "Location" | cut -f2 -d ' ')/../../../docs/patches/deeppicker/*.py DeepPicker-python/ \
+  && micromamba deactivate \
+  && micromamba create -n deep -c conda-forge -c anaconda -c pytorch python=3.6 cudatoolkit=10.1 matplotlib mrcfile  pytorch scikit-image scipy tensorflow-gpu=2.1 torchvision \
+  && micromamba clean --all -y
+
+# make RUN commands use the new environment
+SHELL ["micromamba", "run", "-n", "repic", "/bin/bash", "-c"]
+
+# code to run when container is started
+ENTRYPOINT ["micromamba", "run", "-n", "repic"]
+
+LABEL Author="Christopher JF Cameron"
+LABEL VERSION="v0.0.0"

README.md

@@ -29,9 +29,25 @@ Required:
 ** Required to reproduce manuscript results but if the Gurobi package is not found, REPIC will use the [SciPy ILP optimizer](https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.milp.html)
 
 ## Installation guide
-REPIC installation is expected to only take a few minutes:
+REPIC installation is expected to only take a few minutes.
 
-**<details><summary>Install using Conda (recommended)</summary><p>**
+:warning: **WARNING**: Only the Docker installation of REPIC includes pickers (SPHIRE-crYOLO, DeepPicker, and Topaz). If installing REPIC using either Conda or pip, pickers will need to be separately installed (see [docs/](https://github.com/ccameron/REPIC/tree/main/docs/) for installation instructions).
+
+**<details><summary>Install using Docker (recommended)</summary><p>**
+
+1. [Install Docker](https://docs.docker.com/engine/install/) if the ``` docker ``` command is unavailable
+2. [Install and set up NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for building and running GPU-accelerated containers
+3. Build CUDA-supported image from Dockerfile in REPIC GitHub repo:
+```
+docker build -t repic_img https://github.com/ccameron/REPIC.git#main
+```
+4. Run container with GPU acceleration (example [iter_pick.py](https://github.com/ccameron/REPIC/blob/main/repic/commands/iter_pick.py) command shown below):
+```
+docker run --gpus all -v <file_path>/REPIC/examples:/examples repic_img repic iter_pick /examples/10057/iter_config.json 4 100
+```
+</p></details>
+
+**<details><summary>Install using Conda</summary><p>**
 
 :warning: **WARNING**: if Python package conflicts are encountered during the Conda installation of REPIC, please ensure Conda channels are properly set for Bioconda. See [Bioconda Usage](https://bioconda.github.io/) for more information
 
@@ -55,6 +71,7 @@ conda clean --all
 </p></details>
 
 **<details><summary>Install from source using pip</summary><p>**
+
 1. Either download the package by clicking the "Clone or download" button, unzipping file in desired location, and renaming the directory "REPIC" OR using the following command line:
 ```
 git clone https://github.com/ccameron/REPIC
@@ -86,14 +103,38 @@ conda clean --all
 ```
 </p></details>
 
-To check if REPIC was correctly installed, run the following command (after activating the REPIC Conda environment):
+To check if REPIC was correctly installed, run the following command (after activating the REPIC Conda environment or using a REPIC container):
 ```
 repic -h
 ```
 A help menu should appear in the terminal.
 
 ## Integration
-REPIC is also available as a [Scipion](https://scipion.i2pc.es/)  plugin: [https://github.com/scipion-em/scipion-em-repic](https://github.com/scipion-em/scipion-em-repic)
+
+**<details><summary>Run using published Docker image (with Singularity/Apptainer)</summary><p>**
+
+A REPIC Docker image is published on both [DockerHub](https://hub.docker.com/r/cjfcameron/repic) and the [GitHub container registery](https://github.com/ccameron/REPIC/pkgs/container/repic). [Apptainer](https://apptainer.org/) (formerly Singularity) can be used to run this image:
+
+1. [Install Apptainer](https://apptainer.org/docs/admin/main/installation.html) if the ```apptainer``` command is unavailable
+2. [Install and set up NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for building and running GPU-accelerated containers
+3. Pull REPIC Docker image and convert to Singularity image format (SIF) (requires >8 Gb of memory and ~40 mins for conversion):
+```
+apptainer pull docker://cjfcameron/repic:main
+```
+If SIF file creation is taking a long time, increase the ```mksquashfs mem``` parameter in the Apptainer config file (apptainer.conf). See [here](https://apptainer.org/docs/admin/1.0/configfiles.html) for more information.
+
+4. Run container with GPU acceleration (example [iter_pick.py](https://github.com/ccameron/REPIC/blob/main/repic/commands/iter_pick.py) command shown below):
+```
+apptainer run --nv --bind <file_path>/REPIC/examples:/examples repic_main.sif repic iter_pick /examples/10057/iter_config.json 4 100
+```
+</p></details>
+
+**<details><summary>Run using Scipion plugin</summary><p>**
+
+REPIC is available as a [Scipion](https://scipion.i2pc.es/)  plugin: [https://github.com/scipion-em/scipion-em-repic](https://github.com/scipion-em/scipion-em-repic)
+
+See [here](https://scipion-em.github.io/docs/release-3.0.0/docs/scipion-modes/how-to-install.html#installing-other-plugins) for information about installing plugins for Scipion.
+</p></details>
 
 ## Example data
 Example [SPHIRE-crYOLO](https://cryolo.readthedocs.io/en/stable/), [DeepPicker](https://github.com/jianlin-cheng/DeepCryoEM), and [Topaz](https://github.com/tbepler/topaz) picked particle coordinate files for $\beta$-galactosidase ([EMPIAR-10017](https://www.ebi.ac.uk/empiar/EMPIAR-10017/)) micrographs are found in [examples/10017/](https://github.com/ccameron/REPIC/tree/main/examples/10017/). These files were generated by applying the pre-trained pickers to $\beta$-galactosidase micrographs, filtering false positive per author suggested thresholds, and then converting files to BOX format using [coord_converter.py](https://github.com/ccameron/REPIC/blob/main/repic/utils/coord_converter.py).
@@ -267,6 +308,10 @@ For other concerns, please email [Christopher JF Cameron](mailto:christopher.cam
 
 ## Releases
 
+### v0.2.1
+ - Scipion plugin created
+ - Docker/Singularity/Apptainer integrated
+
 ### v0.2.0
  - k-d tree algorithm integrated to reduce graph building runtime
  - Approval to include DeepPicker with REPIC install/distribution added: https://github.com/ccameron/REPIC/blob/main/imgs/deeppicker_approval.png  

repic/commands/iter_config.py

@@ -97,17 +97,13 @@ def main(args):
     del prefix, deep_files
 
     #   check that provided Conda environment names can be found
-    envs = []
     stdout = subprocess.check_output(
-        "conda info --envs", shell=True).decode(sys.stdout.encoding).strip()
-    for line in stdout.split('\n'):
-        if line.startswith(('#', ' ')):
-            continue
-        envs.append(line.split()[0])
+        "conda env list", shell=True).decode(sys.stdout.encoding).strip()
+    envs = [line.strip().split()[0] for line in stdout.split('\n')]
     envs = set([args.cryolo_env, args.deep_env, args.topaz_env]) - set(envs)
     assert (len(envs) ==
             0), f"Error - Conda environment(s) not found: {', '.join(envs)}"
-    del envs, stdout, line
+    del envs, stdout
 
     #   write JSON file of iter_pick parameters
     print(f"Writing config file to {args.out_file_path}")

repic/commands/iter_pick.py

@@ -77,7 +77,8 @@ def main(args):
     print(f"""Note - stderr and stdout are being written to: {out_file}
 Please review this file for iterative ensemble particle picking progress""")
     with open(out_file, 'wt') as o:
-        subprocess.run(cmd, text=True, stderr=subprocess.STDOUT, stdout=o)
+        subprocess.run(cmd, text=True,
+                       stderr=subprocess.STDOUT, stdout=o)
     del params_dict, script_dir, file_path, cmd, out_file
 
 

repic/iterative_particle_picking/fit_cryolo.sh

@@ -18,8 +18,11 @@ if [ -z "${REPIC_UTILS}" ]; then REPIC_UTILS=0; fi
 if [ -z "${CRYOLO_ENV}" ]; then CRYOLO_ENV="cryolo"; fi
 if [ -z "${CRYOLO_FILTERED_DIR}" ]; then CRYOLO_FILTERED_DIR=${REPIC_OUT_DIR}/filtered_tmp/; fi
 
-eval "$(conda shell.bash hook)"
-conda activate ${CRYOLO_ENV}
+if [ -x "$(command -v micromamba)" ]; then
+  eval "$(micromamba shell hook -s bash)" && micromamba activate ${CRYOLO_ENV}
+elif [ -x "$(command -v conda)" ]; then
+  eval "$(conda shell.bash hook)" && conda activate ${CRYOLO_ENV}
+fi
 
 #	create SPHIRE-crYOLO config file
 #	batch_size set to 2 for consistency between development machines
@@ -43,8 +46,6 @@ cryolo_train.py -c ${REPIC_OUT_DIR}/config_cryolo.json \
     -e 32 \
     --seed 1 \
 
-conda deactivate
-
 #	save runtime to storage
 END=$(date +%s.%N)
 DIFF=$( echo "${END} - ${START}" | bc -l )

repic/iterative_particle_picking/fit_deep.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 #
 #	fit_deep.sh - fits DeepPicker model to micrographs
-# author: Christopher JF Cameron
+#	author: Christopher JF Cameron
 #
 
 #	track runtime
@@ -31,15 +31,16 @@ for BOX_FILE in ${REPIC_VAL_COORD}/*.box; do
 done
 REPIC_VAL_COORD=${REPIC_VAL_COORD}/STAR
 
-eval "$(conda shell.bash hook)"
-conda deactivate
+if [ -x "$(command -v micromamba)" ]; then
+  eval "$(micromamba shell hook -s bash)" && micromamba activate ${DEEP_ENV}
+elif [ -x "$(command -v conda)" ]; then
+  eval "$(conda shell.bash hook)" && conda activate ${DEEP_ENV}
+fi
 
 #	softlink micrographs to coordinate directory
 cp -s ${REPIC_TRAIN_MRC}/*.mrc ${REPIC_TRAIN_COORD}/
 cp -s ${REPIC_VAL_MRC}/*.mrc ${REPIC_VAL_COORD}/
 
-conda activate ${DEEP_ENV}
-
 python ${DEEP_DIR}/train.py --train_type 1 \
     --train_inputDir ${REPIC_TRAIN_COORD} \
     --validation_inputDir ${REPIC_VAL_COORD} \
@@ -55,8 +56,6 @@ python ${DEEP_DIR}/train.py --train_type 1 \
 rm -rf ${REPIC_TRAIN_COORD}
 rm -rf ${REPIC_VAL_COORD}
 
-conda deactivate
-
 #	save runtime to storage
 END=$(date +%s.%N)
 DIFF=$( echo "${END} - ${START}" | bc -l )

repic/iterative_particle_picking/fit_topaz.sh

@@ -19,8 +19,11 @@ if [ -z "${TOPAZ_SCALE}" ]; then TOPAZ_SCALE=0; fi
 if [ -z "${TOPAZ_PARTICLE_RAD}" ]; then TOPAZ_PARTICLE_RAD=0; fi
 if [ -z "${TOPAZ_BALANCE}" ]; then TOPAZ_BALANCE=0.0625; fi
 
-eval "$(conda shell.bash hook)"
-conda activate ${TOPAZ_ENV}
+if [ -x "$(command -v micromamba)" ]; then
+  eval "$(micromamba shell hook -s bash)" && micromamba activate ${TOPAZ_ENV}
+elif [ -x "$(command -v conda)" ]; then
+  eval "$(conda shell.bash hook)" && conda activate ${TOPAZ_ENV}
+fi
 
 topaz convert -s ${TOPAZ_SCALE} \
     -o ${REPIC_OUT_DIR}/particles_train_downsampled.txt \
@@ -33,10 +36,13 @@ topaz convert -s ${TOPAZ_SCALE} \
 TOPAZ_NUM_PARTICLES=$(echo "$((${REPIC_NUM_PARTICLES} * 125/100))")
 
 #	train Topaz model
-topaz train -n ${TOPAZ_NUM_PARTICLES} \
+#	RESNET16 with 64 units suggested by A. Noble during NYSBC visit
+topaz train --num-particles ${TOPAZ_NUM_PARTICLES} \
     --num-workers 8 \
     --minibatch-size 64 \
     --minibatch-balance ${TOPAZ_BALANCE} \
+    --model resnet16 \
+    --units 64 \
     --train-images ${REPIC_TRAIN_MRC}/downsampled_mrc/ \
     --train-targets ${REPIC_OUT_DIR}/particles_train_downsampled.txt \
     --test-images ${REPIC_VAL_MRC}/downsampled_mrc/ \
@@ -48,8 +54,6 @@ topaz train -n ${TOPAZ_NUM_PARTICLES} \
 rm -f ${REPIC_OUT_DIR}/particles_train_downsampled.txt
 rm -f ${REPIC_OUT_DIR}/particles_val_downsampled.txt
 
-conda deactivate
-
 #	save runtime to storage
 END=$(date +%s.%N)
 DIFF=$( echo "${END} - ${START}" | bc -l )

repic/iterative_particle_picking/preprocess_topaz.sh

@@ -15,8 +15,11 @@ if [ -z "${REPIC_OUT_DIR}" ]; then REPIC_OUT_DIR=0; fi
 if [ -z "${TOPAZ_ENV}" ]; then TOPAZ_ENV="topaz"; fi
 if [ -z "${TOPAZ_SCALE}" ]; then TOPAZ_SCALE=0; fi
 
-eval "$(conda shell.bash hook)"
-conda activate ${TOPAZ_ENV}
+if [ -x "$(command -v micromamba)" ]; then
+  eval "$(micromamba shell hook -s bash)" && micromamba activate ${TOPAZ_ENV}
+elif [ -x "$(command -v conda)" ]; then
+  eval "$(conda shell.bash hook)" && conda activate ${TOPAZ_ENV}
+fi
 
 #	downsample micrographs
 topaz preprocess -s ${TOPAZ_SCALE} \
@@ -32,8 +35,6 @@ topaz preprocess -s ${TOPAZ_SCALE} \
     -o ${REPIC_TEST_MRC}/downsampled_mrc/ \
     ${REPIC_TEST_MRC}/*.mrc
 
-conda deactivate
-
 #	save runtime to storage
 END=$(date +%s.%N)
 DIFF=$( echo "${END} - ${START}" | bc -l )

repic/iterative_particle_picking/run.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 #
-# run.sh - perform particle picking using iterative ensemble learning
-# author: Christopher JF Cameron
+#	run.sh - perform particle picking using iterative ensemble learning
+#	author: Christopher JF Cameron
 #
 
 #  set filepath to REPIC install src/

repic/iterative_particle_picking/run_cryolo.sh

@@ -15,8 +15,11 @@ if [ -z "${REPIC_UTILS}" ]; then REPIC_UTILS=0; fi
 if [ -z "${CRYOLO_ENV}" ]; then CRYOLO_ENV="cryolo"; fi
 if [ -z "${CRYOLO_FILTERED_DIR}" ]; then CRYOLO_FILTERED_DIR=${REPIC_OUT_DIR}/filtered_tmp/; fi
 
-eval "$(conda shell.bash hook)"
-conda activate ${CRYOLO_ENV}
+if [ -x "$(command -v micromamba)" ]; then
+  eval "$(micromamba shell hook -s bash)" && micromamba activate ${CRYOLO_ENV}
+elif [ -x "$(command -v conda)" ]; then
+  eval "$(conda shell.bash hook)" && conda activate ${CRYOLO_ENV}
+fi
 
 # create SPHIRE-crYOLO config file
 cryolo_gui.py config ${REPIC_OUT_DIR}/config_cryolo.json \
@@ -35,8 +38,6 @@ cryolo_predict.py -c ${REPIC_OUT_DIR}/config_cryolo.json \
     -t 0.0 \
     --write_empty
 
-conda deactivate
-
 #	save runtime to storage
 END=$(date +%s.%N)
 DIFF=$( echo "${END} - ${START}" | bc -l )