nf-core · bentsherman · Apr 27, 2024 · Apr 27, 2024 · Apr 27, 2024 · Apr 27, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -59,7 +59,7 @@ jobs:
       fail-fast: false
       matrix:
         NXF_VER:
-          - "24.04.2"
+          - "24.10.2"
           - "latest-everything"
         profile:
           - "conda"

diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@
 [![GitHub Actions CI Status](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml)
 [![GitHub Actions Linting Status](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524)[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.10.2-23aa62.svg)](https://www.nextflow.io/)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)

diff --git a/conf/base.config b/conf/base.config
@@ -15,12 +15,6 @@ process {
     memory = { 6.GB   * task.attempt }
     time   = { 4.h    * task.attempt }
 
-    publishDir = [
-        path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
-        mode: params.publish_dir_mode,
-        saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-    ]
-
     errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
     maxRetries    = 1
     maxErrors     = '-1'

diff --git a/main.nf b/main.nf
@@ -9,6 +9,9 @@
 ----------------------------------------------------------------------------------------
 */
 
+nextflow.preview.output = true
+nextflow.preview.topic = true
+
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS
@@ -18,6 +21,7 @@
 include { SRA                     } from './workflows/sra'
 include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline'
 include { PIPELINE_COMPLETION     } from './subworkflows/local/utils_nfcore_fetchngs_pipeline'
+include { softwareVersionsToYAML  } from './subworkflows/nf-core/utils_nfcore_pipeline'
 
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -38,8 +42,16 @@ workflow NFCORE_FETCHNGS {
     //
     // WORKFLOW: Download FastQ files for SRA / ENA / GEO / DDBJ ids
     //
-    SRA ( ids )
+    SRA (
+        ids,
+        params.ena_metadata_fields ?: '',
+        params.download_method,
+        params.skip_fastq_download,
+        params.dbgap_key
+    )
 
+    emit:
+    samples = SRA.out.samples
 }
 
 /*
@@ -81,6 +93,33 @@ workflow {
         params.monochrome_logs,
         params.hook_url
     )
+
+    publish:
+    NFCORE_FETCHNGS.out.samples >> 'samples'
+    softwareVersionsToYAML() >> 'versions'
+}
+
+
+output {
+    samples {
+        path { _sample ->
+            def dirs = [
+                'fastq': 'fastq',
+                'md5': 'fastq/md5'
+            ]
+            return { file -> "${dirs[file.extension]}/${file.name}" }
+        }
+        index {
+            path 'samplesheet/samplesheet.json'
+        }
+    }
+
+    versions {
+        path '.'
+        index {
+            path 'nf_core_fetchngs_software_mqc_versions.yml'
+        }
+    }
 }
 
 /*

diff --git a/modules/local/aspera_cli/main.nf b/modules/local/aspera_cli/main.nf
@@ -14,7 +14,7 @@ process ASPERA_CLI {
     output:
     tuple val(meta), path("*fastq.gz"), emit: fastq
     tuple val(meta), path("*md5")     , emit: md5
-    path "versions.yml"               , emit: versions
+    tuple val("${task.process}"), val('aspera_cli'), eval('ascli --version'), topic: versions
 
     script:
     def args = task.ext.args ?: ''
@@ -31,11 +31,6 @@ process ASPERA_CLI {
 
         echo "${meta.md5_1}  ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5
         md5sum -c ${meta.id}.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            aspera_cli: \$(ascli --version)
-        END_VERSIONS
         """
     } else {
         """
@@ -58,11 +53,22 @@ process ASPERA_CLI {
 
         echo "${meta.md5_2}  ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5
         md5sum -c ${meta.id}_2.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            aspera_cli: \$(ascli --version)
-        END_VERSIONS
         """
     }
 }
+
+workflow {
+    def input = [
+        [ id:'SRX9626017_SRR13191702', single_end:false, md5_1: '89c5be920021a035084d8aeb74f32df7', md5_2: '56271be38a80db78ef3bdfc5d9909b98' ],
+        [
+            'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_1.fastq.gz',
+            'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_2.fastq.gz'
+        ]
+    ]
+    def user = 'era-fasp'
+
+    ASPERA_CLI (
+        input,
+        user
+    )
+}
diff --git a/modules/local/aspera_cli/nextflow.config b/modules/local/aspera_cli/nextflow.config
@@ -1,17 +1,5 @@
 process {
     withName: 'ASPERA_CLI' {
         ext.args = '-QT -l 300m -P33001'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }
diff --git a/modules/local/multiqc_mappings_config/main.nf b/modules/local/multiqc_mappings_config/main.nf
@@ -11,17 +11,12 @@ process MULTIQC_MAPPINGS_CONFIG {
 
     output:
     path "*config.yml" , emit: yml
-    path "versions.yml", emit: versions
+    tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions
 
     script:
     """
     multiqc_mappings_config.py \\
         $csv \\
         multiqc_config.yml
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        python: \$(python --version | sed 's/Python //g')
-    END_VERSIONS
     """
 }
diff --git a/modules/local/multiqc_mappings_config/nextflow.config b/modules/local/multiqc_mappings_config/nextflow.config
diff --git a/modules/local/sra_fastq_ftp/main.nf b/modules/local/sra_fastq_ftp/main.nf
@@ -15,7 +15,7 @@ process SRA_FASTQ_FTP {
     output:
     tuple val(meta), path("*fastq.gz"), emit: fastq
     tuple val(meta), path("*md5")     , emit: md5
-    path "versions.yml"               , emit: versions
+    tuple val("${task.process}"), val('wget'), eval("echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')"), topic: versions
 
     script:
     def args = task.ext.args ?: ''
@@ -28,11 +28,6 @@ process SRA_FASTQ_FTP {
 
         echo "${meta.md5_1}  ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5
         md5sum -c ${meta.id}.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//'))
-        END_VERSIONS
         """
     } else {
         """
@@ -51,11 +46,6 @@ process SRA_FASTQ_FTP {
 
         echo "${meta.md5_2}  ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5
         md5sum -c ${meta.id}_2.fastq.gz.md5
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//'))
-        END_VERSIONS
         """
     }
 }
diff --git a/modules/local/sra_fastq_ftp/nextflow.config b/modules/local/sra_fastq_ftp/nextflow.config
@@ -1,17 +1,5 @@
 process {
     withName: 'SRA_FASTQ_FTP' {
         ext.args = '-t 5 -nv -c -T 60'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }
diff --git a/modules/local/sra_ids_to_runinfo/main.nf b/modules/local/sra_ids_to_runinfo/main.nf
@@ -14,7 +14,7 @@ process SRA_IDS_TO_RUNINFO {
 
     output:
     path "*.tsv"       , emit: tsv
-    path "versions.yml", emit: versions
+    tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions
 
     script:
     def metadata_fields = fields ? "--ena_metadata_fields ${fields}" : ''
@@ -24,10 +24,5 @@ process SRA_IDS_TO_RUNINFO {
         id.txt \\
         ${id}.runinfo.tsv \\
         $metadata_fields
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        python: \$(python --version | sed 's/Python //g')
-    END_VERSIONS
     """
 }
diff --git a/modules/local/sra_ids_to_runinfo/nextflow.config b/modules/local/sra_ids_to_runinfo/nextflow.config
diff --git a/modules/local/sra_runinfo_to_ftp/main.nf b/modules/local/sra_runinfo_to_ftp/main.nf
@@ -11,17 +11,12 @@ process SRA_RUNINFO_TO_FTP {
 
     output:
     path "*.tsv"       , emit: tsv
-    path "versions.yml", emit: versions
+    tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions
 
     script:
     """
     sra_runinfo_to_ftp.py \\
         ${runinfo.join(',')} \\
         ${runinfo.toString().tokenize(".")[0]}.runinfo_ftp.tsv
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        python: \$(python --version | sed 's/Python //g')
-    END_VERSIONS
     """
 }
diff --git a/modules/local/sra_runinfo_to_ftp/nextflow.config b/modules/local/sra_runinfo_to_ftp/nextflow.config
diff --git a/modules/local/sra_to_samplesheet/nextflow.config b/modules/local/sra_to_samplesheet/nextflow.config
diff --git a/modules/nf-core/custom/sratoolsncbisettings/main.nf b/modules/nf-core/custom/sratoolsncbisettings/main.nf
diff --git a/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh b/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh
diff --git a/modules/nf-core/sratools/fasterqdump/main.nf b/modules/nf-core/sratools/fasterqdump/main.nf
diff --git a/modules/nf-core/sratools/fasterqdump/nextflow.config b/modules/nf-core/sratools/fasterqdump/nextflow.config
diff --git a/modules/nf-core/sratools/prefetch/main.nf b/modules/nf-core/sratools/prefetch/main.nf
diff --git a/modules/nf-core/sratools/prefetch/nextflow.config b/modules/nf-core/sratools/prefetch/nextflow.config
diff --git a/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh b/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh