Initial commit

.github/workflows/test_workflow.yml

@@ -0,0 +1,44 @@
+name: Test Nextflow Workflow
+
+on: 
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    # - name: Install dependencies
+    #   run: |
+    #     # Installing Nextflow
+    #     curl -s https://get.nextflow.io | bash
+    #     mv nextflow /usr/local/bin
+
+    #     # Installing Conda
+    #     wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    #     chmod +x Miniconda3-latest-Linux-x86_64.sh
+    #     bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda
+    #     export PATH="$HOME/miniconda/bin:$PATH"
+    #     conda init bash
+    #     conda config --set always_yes yes --set changeps1 no
+
+    #     # Install mamba
+    #     conda install -c conda-forge mamba
+    - name: Set up Conda
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: nextflow
+        environment-file: test/test_environment.yml
+        auto-activate-base: true
+    - name: Test Workflow
+      shell: bash -el {0}
+      run: |
+        conda activate nextflow && make run

.gitignore

@@ -0,0 +1,3 @@
+*.ipynb_checkpoints*
+*.env
+*credentials

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "GNPS2_DeploymentTooling"]
+	path = GNPS2_DeploymentTooling
+	url = https://github.com/Wang-Bioinformatics-Lab/GNPS2_DeploymentTooling.git

Makefile

@@ -0,0 +1,8 @@
+run:
+	nextflow run ./nf_workflow.nf -resume -c nextflow.config
+
+run_hpcc:
+	nextflow run ./nf_workflow.nf -resume -c nextflow_hpcc.config
+
+run_docker:
+	nextflow run ./nf_workflow.nf -resume -with-docker <CONTAINER NAME>

README.md

@@ -0,0 +1,85 @@
+# Nextflow Template
+
+To run the workflow to test simply do
+
+```
+make run
+```
+
+To learn NextFlow checkout this documentation:
+
+https://www.nextflow.io/docs/latest/index.html
+
+## Installation
+
+You will need to have conda, mamba, and nextflow installed to run things locally. 
+
+## GNPS2 Workflow Input information
+
+Check the definition for the workflow input and display parameters:
+https://wang-bioinformatics-lab.github.io/GNPS2_Documentation/workflowdev/
+
+
+## Deployment to GNPS2
+
+In order to deploy, we have a set of deployment tools that will enable deployment to the various gnps2 systems. To run the deployment, you will need the following setup steps completed:
+
+1. Checked out of the deployment submodules
+1. Conda environment and dependencies
+1. SSH configuration updated
+
+### Checking out the deployment submodules
+
+use the following commands from the deploy_gnps2 folder. 
+
+You might need to checkout the module, do this by running
+
+```
+git submodule init
+git submodule update
+```
+
+You will also need to specify the user on the server that you've been given that your public key has been associated with. If you want to not enter this every time you do a deployment, you can create a Makefile.credentials file in the deploy_gnps2 folder with the following contents
+
+```
+USERNAME=<enter the username>
+```
+
+### Deployment Dependencies
+
+You will need to install the dependencies in GNPS2_DeploymentTooling/requirements.txt on your own local machine. 
+
+You can find this [here](https://github.com/Wang-Bioinformatics-Lab/GNPS2_DeploymentTooling).
+
+One way to do this is to use conda to create an environment, for example:
+
+```
+conda create -n deploy python=3.8
+pip install -r GNPS2_DeploymentTooling/requirements.txt
+```
+
+### SSH Configuration
+
+Also update your ssh config file to include the following ssh target:
+
+```
+Host ucr-gnps2-dev
+    Hostname ucr-lemon.duckdns.org
+```
+
+### Deploying to Dev Server
+
+To deploy to development, use the following command, if you don't have your ssh public key installed onto the server, you will not be able to deploy.
+
+```
+make deploy-dev
+```
+
+### Deploying to Production Server
+
+To deploy to production, use the following command, if you don't have your ssh public key installed onto the server, you will not be able to deploy.
+
+```
+make deploy-prod
+```
+

bin/conda_env.yml

@@ -0,0 +1,10 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.8.8
+  - pandas
+  - pip:
+    - xmltodict
+    - requests
+    - pyteomics

bin/python_script.py

@@ -0,0 +1,20 @@
+import sys
+import argparse
+import pandas as pd
+
+def main():
+    parser = argparse.ArgumentParser(description='Test write out a file.')
+    parser.add_argument('input_filename')
+    parser.add_argument('output_filename')
+
+    args = parser.parse_args()
+
+    df = pd.DataFrame()
+    df["OUTPUT"] = "1"
+    df["OUTPUT2"] = "2"
+
+    # saving file
+    df.to_csv(args.output_filename, sep="\t", index=False)
+
+if __name__ == "__main__":
+    main()

deploy_gnps2/Makefile

@@ -0,0 +1,2 @@
+-include ./Makefile.credentials
+include ./Makefile.deploytemplate

deploy_gnps2/Makefile.deploytemplate

@@ -0,0 +1 @@
+../GNPS2_DeploymentTooling/Makefile.deploytemplate

deploy_gnps2/fabfile.py

@@ -0,0 +1 @@
+../GNPS2_DeploymentTooling/fabfile.py

deploy_gnps2/fabric.yml

@@ -0,0 +1 @@
+../GNPS2_DeploymentTooling/fabric.yml

jupyter/.dockerignore

@@ -0,0 +1 @@
+data

jupyter/Dockerfile

@@ -0,0 +1,14 @@
+from jupyter/datascience-notebook:lab-3.4.4
+
+COPY requirements.txt /
+RUN pip install -r /requirements.txt
+RUN pip install jupyterlab-git
+
+# Nextfow
+RUN conda install -y mamba -c conda-forge
+RUN mamba install -y -c bioconda nextflow
+
+# Installing system dependencies
+USER root
+RUN apt-get update && apt-get install build-essential -y
+USER jovyan

jupyter/Dockerfile.coder

@@ -0,0 +1,17 @@
+from linuxserver/code-server:4.7.0
+
+RUN apt-get update && apt-get install -y \
+    git-core
+RUN apt-get install build-essential wget -y
+
+ENV CONDA_DIR /opt/conda
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p $CONDA_DIR
+ENV PATH=$CONDA_DIR/bin:$PATH
+
+COPY requirements.txt /
+RUN pip install -r /requirements.txt
+
+# Nextfow
+RUN conda install -y mamba -c conda-forge
+RUN mamba install -y -c bioconda nextflow

jupyter/Makefile

@@ -0,0 +1,22 @@
+
+jupyter-compose:
+	docker-compose build
+	docker-compose -f docker-compose.yml --compatibility up
+
+jupyter-compose-background:
+	docker-compose build
+	docker-compose -f docker-compose.yml --compatibility up -d
+
+jupyter-stop:
+	docker-compose -f docker-compose.yml down
+
+coder-compose:
+	docker-compose -f docker-compose-coder.yml build
+	docker-compose -f docker-compose-coder.yml --compatibility up
+
+coder-compose-background:
+	docker-compose -f docker-compose-coder.yml build
+	docker-compose -f docker-compose-coder.yml --compatibility up -d
+
+coder-stop:
+	docker-compose -f docker-compose-coder.yml down

jupyter/README

@@ -0,0 +1,30 @@
+## Wang Bioinformatics Lab Nextflow In Browser Dev Template
+
+Here we are able to launch a notebook inside of docker with all the dependencies you would want. 
+
+### Customizing Docker for you
+
+1. Update dependencies - change the requirements.txt
+1. Update your password - create an .env file with LOGINPASSWORD="YOUR_PASSWORD" or else it will not be available
+1. Update the port for the server - change port 9000 to something around 9000, but not 9000 in docker-compose.yml or docker-compose-coder.yml
+1. Change the name of the container - change wanglab-jupyter in docker-compose.yml or docker-compose-coder.yml
+
+### Launching the Jupyter Notebook
+
+make jupyter-compose
+
+### Launching Coder
+
+make coder-compose
+
+### Layout
+
+All your code for notebooks will go in src. 
+
+All the data you'll want to work with will go into data.
+
+Additional documentation will go into docs. 
+
+### Best Practices
+
+Commit and save changes often and push to github. 

jupyter/docker-compose-coder.yml

@@ -0,0 +1,23 @@
+version: '3'
+services:
+  wanglab-coder-workflowtemplate:
+    build:
+      context: .
+      dockerfile: Dockerfile.coder
+    container_name: wanglab-coder-workflowtemplate
+    volumes:
+      - ..:/config/workspace:rw
+    ports:
+    - "8443:8443"
+    restart: unless-stopped
+    deploy:
+     resources:
+       limits:
+         memory: 16000M
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=US/Pacific
+      - PASSWORD=${LOGINPASSWORD:-PASSWORD}
+    labels:
+      - traefik.http.routers.backend.rule=Host(`test.wanglab.science`)

jupyter/docker-compose.yml

@@ -0,0 +1,20 @@
+
+version: '3'
+services:
+  wanglab-jupyter-workflowtemplate:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: wanglab-jupyter-workflowtemplate
+    volumes:
+      - ../:/home/jovyan/work:rw
+    ports:
+    - "9000:8888"
+    restart: unless-stopped
+    command: jupyter lab --ip='*' --port=8888 --no-browser --NotebookApp.token='${LOGINPASSWORD:-PASSWORD}' --allow-root
+    deploy:
+     resources:
+       limits:
+         memory: 16000M
+    labels:
+      - traefik.http.routers.backend.rule=Host(`test.wanglab.science`)

jupyter/requirements.txt

@@ -0,0 +1 @@
+pandas

nextflow.config

@@ -0,0 +1,2 @@
+conda.enabled = true
+conda.useMamba = true

nextflow_slurm.config

@@ -0,0 +1,6 @@
+conda.useMamba = true
+executor{
+    name='slurm'
+    queueSize=32 //The maximum number of tasks to queue at one time
+    jobName = { "<Your_workflow_name>-$task.name".replaceAll("\\s", "_") }
+}

nf_workflow.nf

@@ -0,0 +1,31 @@
+#!/usr/bin/env nextflow
+nextflow.enable.dsl=2
+
+params.input_spectra = "README.md"
+
+TOOL_FOLDER = "$baseDir/bin"
+
+process processDataPython {
+    publishDir "./nf_output", mode: 'copy'
+
+    conda "$TOOL_FOLDER/conda_env.yml"
+
+    input:
+    file input 
+
+    output:
+    file 'python_output.tsv'
+
+    """
+    python $TOOL_FOLDER/python_script.py $input python_output.tsv
+    """
+}
+
+
+workflow {
+    data_ch = Channel.fromPath(params.input_spectra)
+
+    // Outputting Python
+    processDataPython(data_ch)
+
+}

requirements.txt

@@ -0,0 +1 @@
+jupyter/requirements.txt

test/test_environment.yml

@@ -0,0 +1,7 @@
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - nextflow
+  - mamba