PR: Wire report notebooks into pipeline

.gitignore

@@ -7,6 +7,7 @@ runs/*
 results*
 work
 out-*
+out/*
 
 ## nf-test
 .nf-test/
@@ -34,4 +35,6 @@ notebooks/*
 tmp
 
 ## vscode
-.vscode/*
+.vscode/*
+
+.e2e_test_tmp/

modules/local/compare/giana.nf

@@ -9,8 +9,8 @@ process GIANA_CALC {
     val threshold_vgene
 
     output:
-    path "${patient}_VgeneScores.txt"
-    path "${patient}_giana.txt"
+    path "${patient}_VgeneScores.txt", emit: 'vgene_scores'
+    path "${patient}_giana.txt", emit: 'giana_output'
     // path "giana_EncodingMatrix.txt"
 
     script:   

modules/local/compare/gliph2.nf

@@ -8,16 +8,22 @@ process GLIPH2_TURBOGLIPH {
     tuple val(patient), path(concat_cdr3)
 
     output:
-    path "${patient}/all_motifs.txt", emit: 'all_motifs'
-    path "${patient}/clone_network.txt", emit: 'clone_network'
-    path "${patient}/cluster_member_details.txt", emit: 'cluster_member_details'
+    // all_motifs/clone_network/cluster_member_details/global_similarities are
+    // copied to patient-prefixed top-level names because multiple patients'
+    // outputs share the same basename otherwise (e.g. "all_motifs.txt"), which
+    // collides when several patients' files are staged together downstream.
+    tuple val(patient), path("${patient}_all_motifs.txt"), emit: 'all_motifs'
+    tuple val(patient), path("${patient}_clone_network.txt"), emit: 'clone_network'
+    tuple val(patient), path("${patient}_cluster_member_details.txt"), emit: 'cluster_member_details'
     path "${patient}/convergence_groups.txt", emit: 'convergence_groups'
-    path "${patient}/global_similarities.txt", emit: 'global_similarities'
+    tuple val(patient), path("${patient}_global_similarities.txt"), emit: 'global_similarities'
     path "${patient}/local_similarities.txt", emit: 'local_similarities'
     path "${patient}/parameter.txt", emit: 'gliph2_parameters'
 
     script:
     """
+    mkdir -p ${patient}
+
     Rscript - <<EOF
     #!/usr/bin/env Rscript
 
@@ -30,27 +36,80 @@ process GLIPH2_TURBOGLIPH {
     colnames(df)[colnames(df) == "duplicate_count"] <- "counts"
     df[,'patient'] <- df[,'sample']
 
-    result <- turboGliph::gliph2(
-        cdr3_sequences = df,
-        result_folder = "./${patient}",
-        lcminp = ${params.local_min_pvalue},
-        sim_depth = ${params.simulation_depth},
-        kmer_mindepth = ${params.kmer_min_depth},
-        lcminove = ${params.local_min_OVE},
-        all_aa_interchangeable = FALSE,
-        n_cores = ${task.cpus}
-    )
-
-    df3 <- read.csv('${patient}/cluster_member_details.txt', sep = '\t', stringsAsFactors = FALSE, check.names = FALSE)
-    df3[,'sample'] <- df3[,'patient']
-    df3 <- merge(df3, df[, c("CDR3b", "TRBV", "sample", 'counts')], by = c("CDR3b", "TRBV", "sample", 'counts'), all.x = TRUE)
-    df3 <- df3[, c('CDR3b', 'TRBV', 'TRBJ', 'counts', 'sample', 'tag', 'seq_ID', 'ultCDR3b')]
+    # gliph2() can throw an uncaught internal error (e.g. a dplyr::left_join()
+    # type mismatch) rather than just reporting "no results found" - catch it
+    # and fall back to empty results for the whole patient.
+    gliph2_ok <- tryCatch({
+        turboGliph::gliph2(
+            cdr3_sequences = df,
+            result_folder = "./${patient}",
+            lcminp = ${params.local_min_pvalue},
+            sim_depth = ${params.simulation_depth},
+            kmer_mindepth = ${params.kmer_min_depth},
+            lcminove = ${params.local_min_OVE},
+            all_aa_interchangeable = FALSE,
+            n_cores = ${task.cpus}
+        )
+        TRUE
+    }, error = function(e) {
+        message("turboGliph::gliph2() failed for ${patient}, treating as no clusters found: ", conditionMessage(e))
+        FALSE
+    })
+
+    # Also covers the narrower "no significant clusters" case, where gliph2()
+    # writes cluster_member_details.txt as a single blank line with no header,
+    # which read.csv rejects ("no lines available in input").
+    df3 <- tryCatch({
+        tmp <- read.csv('${patient}/cluster_member_details.txt', sep = '\t', stringsAsFactors = FALSE, check.names = FALSE)
+        tmp[,'sample'] <- tmp[,'patient']
+        tmp <- merge(tmp, df[, c("CDR3b", "TRBV", "sample", 'counts')], by = c("CDR3b", "TRBV", "sample", 'counts'), all.x = TRUE)
+        tmp[, c('CDR3b', 'TRBV', 'TRBJ', 'counts', 'sample', 'tag', 'seq_ID', 'ultCDR3b')]
+    }, error = function(e) {
+        data.frame(CDR3b=character(), TRBV=character(), TRBJ=character(), counts=integer(),
+                   sample=character(), tag=character(), seq_ID=character(), ultCDR3b=character())
+    })
     write.table(df3, "${patient}/cluster_member_details.txt", sep = "\t", row.names = FALSE, quote = FALSE)
+
+    # Backfill placeholders for any other declared output file gliph2() left
+    # missing, so Nextflow's output declarations don't fail the task.
+    if (!file.exists('${patient}/all_motifs.txt')) {
+        write.table(data.frame(motif=character(), num_in_sample=integer(), num_in_ref=integer(),
+                                fisher.score=double(), num_fold=double()),
+                    '${patient}/all_motifs.txt', sep = "\t", row.names = FALSE, quote = FALSE)
+    }
+    if (!file.exists('${patient}/clone_network.txt')) {
+        file.create('${patient}/clone_network.txt')
+    }
+    if (!file.exists('${patient}/global_similarities.txt')) {
+        write.table(data.frame(cluster_tag=character(), cluster_size=integer(), unique_CDR3b=integer(),
+                                num_in_ref=integer(), fisher.score=double(), aa_at_position=character(),
+                                TRBV=character(), CDR3b=character()),
+                    '${patient}/global_similarities.txt', sep = "\t", row.names = FALSE, quote = FALSE)
+    }
+    if (!file.exists('${patient}/convergence_groups.txt')) {
+        file.create('${patient}/convergence_groups.txt')
+    }
+    if (length(Sys.glob('${patient}/local_similarities_*.txt')) == 0) {
+        write.table(data.frame(motif=character(), num_in_sample=integer(), num_in_ref=integer(),
+                                fisher.score=double(), num_fold=double(), start=integer(), stop=integer(),
+                                members=character()),
+                    '${patient}/local_similarities_none.txt', sep = "\t", row.names = FALSE, quote = FALSE)
+    }
+    if (!file.exists('${patient}/parameter.txt')) {
+        file.create('${patient}/parameter.txt')
+    }
     EOF
 
     # Rename local_similarities file to standardize output name
     input_file="${patient}/local_similarities_*.txt"
     cat \$input_file > ${patient}/local_similarities.txt
+
+    # Copy to patient-prefixed top-level names to avoid basename collisions
+    # when multiple patients' outputs are staged together downstream.
+    cp ${patient}/all_motifs.txt ${patient}_all_motifs.txt
+    cp ${patient}/clone_network.txt ${patient}_clone_network.txt
+    cp ${patient}/cluster_member_details.txt ${patient}_cluster_member_details.txt
+    cp ${patient}/global_similarities.txt ${patient}_global_similarities.txt
     """
 }
 

modules/local/report/render_notebook.nf

@@ -4,20 +4,39 @@ process RENDER_NOTEBOOK {
     label 'process_single'
 
     input:
-    tuple path(notebook), path(files) // path(files) just stages files in root dir
+    // path(files) stages files flat in the root dir; staged_layout optionally
+    // symlinks them into a project_dir-style subdirectory tree for notebooks that
+    // read from a nested project_dir/subdir/file layout instead of bare filenames.
+    // It's a list of [dest_path, source_basename] pairs (e.g.
+    // ["sample/sample_stats.csv", "sample_stats.csv"]) rather than just a dest
+    // path, because source and dest basenames can legitimately differ - e.g.
+    // per-patient files staged under a shared basename (like GLIPH2's
+    // "patientA_all_motifs.txt") get symlinked to their unprefixed, per-patient
+    // subdirectory destination ("gliph2/patientA/all_motifs.txt").
+    tuple path(notebook), path(files), val(staged_layout)
     val project_name
     val workflow_cmd
 
     output:
     path "${notebook.getBaseName()}.html", emit: report_html
 
     script:
+    def stage_cmds = staged_layout.collect { dest, src ->
+        "mkdir -p \"\$(dirname '${dest}')\"; ln -sf \"\$PWD/${src}\" '${dest}'"
+    }.join('\n    ')
+    def project_dir_arg = staged_layout ? "-P project_dir:'.'" : ''
     """
+    ${stage_cmds}
     ## render qmd report to html
     quarto render ${notebook} \\
         -P project_name:${project_name} \\
         -P workflow_cmd:'${workflow_cmd}' \\
         -P sample_table:${file(params.samplesheet)} \\
+        -P subject_col:'${params.subject_col}' \\
+        -P timepoint_col:'${params.timepoint_col}' \\
+        -P timepoint_order_col:'${params.timepoint_order_col}' \\
+        -P alias_col:'${params.alias_col}' \\
+        ${project_dir_arg} \\
         --to html
     """
 

modules/local/sample/tcrspecificity.nf

@@ -20,9 +20,9 @@ process VDJDB_VDJMATCH {
     path(ref_db)
 
     output:
-    path("${sample_meta.sample}.vdjmatch.txt")
-    path("${sample_meta.sample}.annot.summary.txt")
-    path "logs/${sample_meta.sample}.vdjmatch.log"
+    path("${sample_meta.sample}.vdjmatch.txt"), emit: 'vdjmatch_txt'
+    path("${sample_meta.sample}.annot.summary.txt"), emit: 'annot_summary'
+    path "logs/${sample_meta.sample}.vdjmatch.log", emit: 'log'
 
     script:
     def memGb = (task.memory.toMega() * 0.8 / 1024).intValue()

nextflow.config

@@ -75,6 +75,13 @@ params {
 
     //reports
     template_qc = "${projectDir}/notebooks/template_qc.qmd"
+    template_discovery_brief = "${projectDir}/notebooks/template_discovery_brief.qmd"
+    template_details_part1 = "${projectDir}/notebooks/template_details_part1.qmd"
+    template_details_part2 = "${projectDir}/notebooks/template_details_part2.qmd"
+    template_pheno_sc = "${projectDir}/notebooks/template_pheno_sc.qmd"
+    template_pheno_bulk = "${projectDir}/notebooks/template_pheno_bulk.qmd"
+    template_patient_clustering_on = "${projectDir}/notebooks/template_patient_clustering_on.qmd"
+    template_patient_clustering_off = "${projectDir}/notebooks/template_patient_clustering_off.qmd"
 }
 
 includeConfig 'conf/base.config'

nextflow_schema.json

@@ -120,6 +120,34 @@
         "template_qc": {
           "type": "string",
           "description": "Path to QC notebook template."
+        },
+        "template_discovery_brief": {
+          "type": "string",
+          "description": "Path to discovery brief notebook template."
+        },
+        "template_details_part1": {
+          "type": "string",
+          "description": "Path to details (part 1) notebook template."
+        },
+        "template_details_part2": {
+          "type": "string",
+          "description": "Path to details (part 2) notebook template."
+        },
+        "template_pheno_sc": {
+          "type": "string",
+          "description": "Path to single-cell phenotype notebook template."
+        },
+        "template_pheno_bulk": {
+          "type": "string",
+          "description": "Path to bulk (TCRpheno) phenotype notebook template."
+        },
+        "template_patient_clustering_on": {
+          "type": "string",
+          "description": "Path to patient-level clustering (GIANA/GLIPH2) notebook template, used when patient workflow_level is run."
+        },
+        "template_patient_clustering_off": {
+          "type": "string",
+          "description": "Path to placeholder notebook template used when patient workflow_level is not run."
         }
       }
     },

notebooks/template_details_part1.qmd

@@ -6,7 +6,7 @@ format:
     toc: true
     toc_depth: 3
     code-fold: true
-    embed-resources: false
+    embed-resources: true
     number-sections: true
     smooth-scroll: true
     grid:

notebooks/template_details_part2.qmd

@@ -6,7 +6,7 @@ format:
     toc: true
     toc_depth: 3
     code-fold: true
-    embed-resources: false
+    embed-resources: true
     number-sections: true
     smooth-scroll: true
     grid:
@@ -103,6 +103,4 @@ This pipeline can be used to analyze both **single-cell and bulk TCR data**. Ple
 
 {{< include ./template_sharing.qmd >}}
 
-{{< include ./template_giana.qmd >}}
-
-{{< include ./template_gliph.qmd >}}
+{{< include ./template_patient_clustering.qmd >}}