Workflow output definition

assets/schema_mappings.yml

@@ -0,0 +1,28 @@
+$schema: 'http://json-schema.org/draft-07/schema'
+$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/assets/schema_mappings.yml'
+title: 'nf-core/fetchngs pipeline - id_mappings.csv schema'
+description: 'Schema for the mappings file produced by fetchngs'
+type: array
+items:
+  type: object
+  properties:
+    sample:
+      type: string
+    experiment_accession:
+      type: string
+    run_accession:
+      type: string
+    sample_accession:
+      type: string
+    experiment_alias:
+      type: string
+    run_alias:
+      type: string
+    sample_alias:
+      type: string
+    experiment_title:
+      type: string
+    sample_title:
+      type: string
+    sample_description:
+      type: string

assets/schema_samplesheet.yml

@@ -0,0 +1,81 @@
+$schema: 'http://json-schema.org/draft-07/schema'
+$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/assets/schema_mappings.yml'
+title: 'nf-core/fetchngs pipeline - samplesheet.csv schema'
+description: 'Schema for the samplesheet file produced by fetchngs'
+type: array
+items:
+  type: object
+  properties:
+    sample:
+      type: string
+    fastq_1:
+      type: string
+      format: file-path
+      pattern: '^\\S+\\.f(ast)?q\\.gz$'
+    fastq_2:
+      type: string
+      format: file-path
+      pattern: '^\\S+\\.f(ast)?q\\.gz$'
+    run_accession:
+      type: string
+    experiment_accession:
+      type: string
+    sample_accession:
+      type: string
+    secondary_sample_accession:
+      type: string
+    study_accession:
+      type: string
+    secondary_study_accession:
+      type: string
+    submission_accession:
+      type: string
+    run_alias:
+      type: string
+    experiment_alias:
+      type: string
+    sample_alias:
+      type: string
+    study_alias:
+      type: string
+    library_layout:
+      type: string
+    library_selection:
+      type: string
+    library_source:
+      type: string
+    library_strategy:
+      type: string
+    library_name:
+      type: string
+    instrument_model:
+      type: string
+    instrument_platform:
+      type: string
+    base_count:
+      type: integer
+    read_count:
+      type: integer
+    tax_id:
+      type: string
+    scientific_name:
+      type: string
+    sample_title:
+      type: string
+    experiment_title:
+      type: string
+    study_title:
+      type: string
+    sample_description:
+      type: string
+    fastq_md5:
+      type: string
+      pattern: '^[0-9a-f]{32}$'
+    fastq_bytes:
+      type: integer
+    fastq_ftp:
+      type: string
+    fastq_galaxy:
+      type: string
+    fastq_aspera:
+      type: string

conf/base.config

@@ -14,12 +14,6 @@ process {
     memory = { check_max( 6.GB * task.attempt, 'memory' ) }
     time   = { check_max( 4.h  * task.attempt, 'time'   ) }
 
-    publishDir = [
-        path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
-        mode: params.publish_dir_mode,
-        saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-    ]
-
     errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
     maxRetries    = 1
     maxErrors     = '-1'

main.nf

@@ -10,6 +10,7 @@
 */
 
 nextflow.enable.dsl = 2
+nextflow.preview.topic = true
 
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -34,6 +35,13 @@ workflow NFCORE_FETCHNGS {
     //
     SRA ( ids )
 
+    emit:
+    samplesheet     = SRA.out.samplesheet
+    mappings        = SRA.out.mappings
+    sample_mappings = SRA.out.sample_mappings
+    sra_metadata    = SRA.out.sra_metadata
+    versions        = SRA.out.versions
+
 }
 
 /*
@@ -83,6 +91,32 @@ workflow {
     )
 }
 
+output {
+    directory params.outdir, mode: params.publish_dir_mode
+
+    'fastq' {
+        topic 'fastq'
+    }
+
+    'fastq/md5' {
+        topic 'md5'
+    }
+
+    'metadata' {
+        topic 'runinfo-tsv'
+    }
+
+    'pipeline_info' {
+        topic 'versions-yml'
+    }
+
+    'samplesheet' {
+        select NFCORE_FETCHNGS.out.samplesheet, schema: 'assets/schema_samplesheet.yml'
+        select NFCORE_FETCHNGS.out.mappings, schema: 'assets/schema_mappings.yml'
+        select NFCORE_FETCHNGS.out.sample_mappings
+    }
+}
+
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     THE END

modules/local/aspera_cli/nextflow.config

@@ -1,17 +1,5 @@
 process {
     withName: 'ASPERA_CLI' {
         ext.args = '-QT -l 300m -P33001'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }

modules/local/sra_fastq_ftp/nextflow.config

@@ -1,17 +1,5 @@
 process {
     withName: 'SRA_FASTQ_FTP' {
         ext.args = '-t 5 -nv -c -T 60'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }

output.yml

@@ -0,0 +1,16 @@
+$schema: 'http://json-schema.org/draft-07/schema'
+$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/output.yml'
+title: 'nf-core/fetchngs pipeline outputs'
+description: ''
+type: object
+properties:
+  id_mappings:
+    type: string
+    format: file-path
+    mimetype: text/csv
+    schema: assets/schema_mappings.yml
+  samplesheet:
+    type: string
+    format: file-path
+    mimetype: text/csv
+    schema: assets/schema_samplesheet.yml

workflows/sra/main.nf

@@ -54,6 +54,7 @@ workflow SRA {
     SRA_RUNINFO_TO_FTP
         .out
         .tsv
+        // .topic('runinfo-tsv')
         .splitCsv(header:true, sep:'\t')
         .map {
             meta ->
@@ -123,6 +124,7 @@ workflow SRA {
             .fastq
             .mix(SRA_FASTQ_FTP.out.fastq)
             .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads)
+            // .topic('fastq')
             .map {
                 meta, fastq ->
                     def reads = fastq instanceof List ? fastq.flatten() : [ fastq ]
@@ -134,6 +136,9 @@ workflow SRA {
                     return meta_clone
             }
             .set { ch_sra_metadata }
+
+        ASPERA_CLI.out.md5.topic('md5')
+        SRA_FASTQ_FTP.out.md5.topic('md5')
     }
 
     //
@@ -153,7 +158,7 @@ workflow SRA {
         .map { it[1] }
         .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName })
         .map { it.text.tokenize('\n').join('\n') }
-        .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet")
+        .collectFile(name:'samplesheet.csv')
         .set { ch_samplesheet }
 
     SRA_TO_SAMPLESHEET
@@ -162,7 +167,7 @@ workflow SRA {
         .map { it[1] }
         .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName })
         .map { it.text.tokenize('\n').join('\n') }
-        .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet")
+        .collectFile(name:'id_mappings.csv')
         .set { ch_mappings }
 
     //
@@ -181,7 +186,8 @@ workflow SRA {
     // Collate and save software versions
     //
     softwareVersionsToYAML(ch_versions)
-        .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true)
+        .collectFile(name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true)
+        .topic('versions-yml')
 
     emit:
     samplesheet     = ch_samplesheet

workflows/sra/nextflow.config

@@ -1,8 +1,3 @@
-includeConfig "../../modules/local/multiqc_mappings_config/nextflow.config"
 includeConfig "../../modules/local/aspera_cli/nextflow.config"
 includeConfig "../../modules/local/sra_fastq_ftp/nextflow.config"
-includeConfig "../../modules/local/sra_ids_to_runinfo/nextflow.config"
-includeConfig "../../modules/local/sra_runinfo_to_ftp/nextflow.config"
-includeConfig "../../modules/local/sra_to_samplesheet/nextflow.config"
-includeConfig "../../modules/nf-core/sratools/prefetch/nextflow.config"
 includeConfig "../../subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config"