From b46edc1c2a731591754be429dc5b559755ef4510 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 10 Feb 2026 13:41:38 -0500 Subject: [PATCH 01/33] init test workflow --- .github/workflows/test.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..75c372f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +#inspired by https://docs.cirro.bio/pipelines/development/#automated-testing +name: test + +on: + pull_request: + branches: + - 'main' + workflow_dispatch: +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + lfs: true + + - uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '17' + + - name: install nextflow + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: install nf-test + run: | + wget -qO- https://get.nf-test.com | bash + sudo mv nf-test /usr/local/bin + + - name: run tests + run: nf-test test From 81d4539da6b5e8078fa9412b36506860aca37534 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 10 Feb 2026 14:29:11 -0500 Subject: [PATCH 02/33] init azimuth unit test --- .gitignore | 1 + nf-test.config | 8 ++++ tests/data/create_data.nf | 14 +++++++ tests/data/templates/make-test-seurat.r | 52 ++++++++++++++++++++++++ tests/modules/local/azimuth/main.nf.test | 38 +++++++++++++++++ tests/nextflow.config | 5 +++ 6 files changed, 118 insertions(+) create mode 100644 nf-test.config create mode 100644 tests/data/create_data.nf create mode 100644 tests/data/templates/make-test-seurat.r create mode 100644 tests/modules/local/azimuth/main.nf.test create mode 100644 tests/nextflow.config diff --git a/.gitignore b/.gitignore index 146ec77..e55f15e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ figures/ *.zip *.tar.gz *.sif +.nf-test* diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 0000000..2fa82ad --- /dev/null +++ b/nf-test.config @@ -0,0 +1,8 @@ +config { + + testsDir "tests" + workDir ".nf-test" + configFile "tests/nextflow.config" + profile "docker" + +} diff --git a/tests/data/create_data.nf b/tests/data/create_data.nf new file mode 100644 index 0000000..eaaa7dc --- /dev/null +++ b/tests/data/create_data.nf @@ -0,0 +1,14 @@ +process CREATE_DATA { + + tag "Creating test data for annotation" + label 'process_medium' + + container 'syedsazaidi/scratch-annotation:latest' + + output: + path("sr_tiny.rds") , emit: sr_tiny + path("sr_ref_tiny.rds") , emit: sr_ref_tiny + + script: + template 'make-test-seurat.r' +} \ No newline at end of file diff --git a/tests/data/templates/make-test-seurat.r b/tests/data/templates/make-test-seurat.r new file mode 100644 index 0000000..0a95afd --- /dev/null +++ b/tests/data/templates/make-test-seurat.r @@ -0,0 +1,52 @@ +#!/usr/bin/env Rscript + +library(Seurat) + +set.seed(42) + +#create a tiny Seurat object for testing purposes +sr <- CreateSeuratObject( + counts = matrix( + data = rpois(n = 20000, lambda = 10), + nrow = 200, + ncol = 100, + dimnames = list( + paste0("Gene", 1:200), + paste0("Cell", 1:100) + ) + ) +) + +sr <- NormalizeData(sr) +sr <- FindVariableFeatures(sr) +sr <- ScaleData(sr, features = VariableFeatures(sr)) +sr <- RunPCA(sr, features = VariableFeatures(sr)) + +saveRDS(sr, file = "sr_tiny.rds") + +#create a tiny reference Seurat object with seurat_annotation column for +sr_ref <- CreateSeuratObject( + counts = matrix( + data = rpois(n = 40000, lambda = 10), + nrow = 200, + ncol = 200, + dimnames = list( + paste0("Gene", 1:200), + paste0("Cell", 1:200) + ) + ) +) + + +sr_ref[["seurat_annotations"]] <- sample( + x = c("TypeA", "TypeB"), + size = 200, + replace = TRUE +) + +sr_ref <- NormalizeData(sr_ref) +sr_ref <- FindVariableFeatures(sr_ref) +sr_ref <- ScaleData(sr_ref, features = VariableFeatures(sr_ref)) +sr_ref <- RunPCA(sr_ref, features = VariableFeatures(sr_ref)) + +saveRDS(sr_ref, file = "sr_ref_tiny.rds") diff --git a/tests/modules/local/azimuth/main.nf.test b/tests/modules/local/azimuth/main.nf.test new file mode 100644 index 0000000..26e8f43 --- /dev/null +++ b/tests/modules/local/azimuth/main.nf.test @@ -0,0 +1,38 @@ +nextflow_process{ + name 'azimuth_test' + script 'modules/local/azimuth/main.nf' + process 'AZIMUTH_ANNOTATION' + + setup { + run("CREATE_DATA"){ + script "tests/data/create_data.nf" + process { + + } + } + } + + test("Run Azimuth") { + + when { + process { + // """ + // input[0] = file("${projectDir}/modules/local/azimuth/notebook_azimuth.qmd") + // input[1] = file("${projectDir}/tests/data/sr_tiny.rds") + // input[2] = file("${projectDir}/tests/data/sr_ref_tiny.rds") + // input[3] = file(params.page_config) + // """ + """ + input[0] = file("${projectDir}/modules/local/azimuth/notebook_azimuth.qmd") + input[1] = CREATE_DATA.out.sr_tiny + input[2] = CREATE_DATA.out.sr_ref_tiny + input[3] = file(params.page_config) + """ + } + } + + then { + assert process.success + } + } +} \ No newline at end of file diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 0000000..c19b1ad --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,5 @@ +/* +======================================================================================== + Nextflow config file for running tests +======================================================================================== +*/ From 9b8b4731e343889d5948642776a503faf7bca3ff Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 10 Feb 2026 14:40:14 -0500 Subject: [PATCH 03/33] limit resources for github --- tests/nextflow.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/nextflow.config b/tests/nextflow.config index c19b1ad..5aac9d8 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -3,3 +3,7 @@ Nextflow config file for running tests ======================================================================================== */ +params { + max_cpus = 2 + max_memory = '4 GB' +} \ No newline at end of file From 0643ed355436c09aa0d7e3e109830c598de2c522 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 10 Feb 2026 14:48:32 -0500 Subject: [PATCH 04/33] replace resource limits --- nf-test.config | 5 +++++ tests/nextflow.config | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nf-test.config b/nf-test.config index 2fa82ad..545a55e 100644 --- a/nf-test.config +++ b/nf-test.config @@ -5,4 +5,9 @@ config { configFile "tests/nextflow.config" profile "docker" + params { + max_cpus = 2 + max_memory = '4 GB' + } + } diff --git a/tests/nextflow.config b/tests/nextflow.config index 5aac9d8..c19b1ad 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -3,7 +3,3 @@ Nextflow config file for running tests ======================================================================================== */ -params { - max_cpus = 2 - max_memory = '4 GB' -} \ No newline at end of file From 1e50a7cad1bcd69f8838ebc5e24d0ed890ac09b1 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 10 Feb 2026 14:50:05 -0500 Subject: [PATCH 05/33] replace resource limits --- nf-test.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf-test.config b/nf-test.config index 545a55e..7a8a706 100644 --- a/nf-test.config +++ b/nf-test.config @@ -5,9 +5,9 @@ config { configFile "tests/nextflow.config" profile "docker" - params { - max_cpus = 2 - max_memory = '4 GB' - } +} +params { + max_cpus = 2 + max_memory = '4 GB' } From 44519cb03c95089ddd3e198cc0fbc10a7dc3635d Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 10 Feb 2026 14:57:43 -0500 Subject: [PATCH 06/33] replace resource limits --- nf-test.config | 6 ------ tests/modules/local/azimuth/main.nf.test | 11 +++++------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/nf-test.config b/nf-test.config index 7a8a706..14fc5d4 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,13 +1,7 @@ config { - testsDir "tests" workDir ".nf-test" configFile "tests/nextflow.config" profile "docker" - } -params { - max_cpus = 2 - max_memory = '4 GB' -} diff --git a/tests/modules/local/azimuth/main.nf.test b/tests/modules/local/azimuth/main.nf.test index 26e8f43..756aab9 100644 --- a/tests/modules/local/azimuth/main.nf.test +++ b/tests/modules/local/azimuth/main.nf.test @@ -3,9 +3,14 @@ nextflow_process{ script 'modules/local/azimuth/main.nf' process 'AZIMUTH_ANNOTATION' + setup { run("CREATE_DATA"){ script "tests/data/create_data.nf" + params { + max_cpus = 2 + max_memory = '4 GB' + } process { } @@ -16,12 +21,6 @@ nextflow_process{ when { process { - // """ - // input[0] = file("${projectDir}/modules/local/azimuth/notebook_azimuth.qmd") - // input[1] = file("${projectDir}/tests/data/sr_tiny.rds") - // input[2] = file("${projectDir}/tests/data/sr_ref_tiny.rds") - // input[3] = file(params.page_config) - // """ """ input[0] = file("${projectDir}/modules/local/azimuth/notebook_azimuth.qmd") input[1] = CREATE_DATA.out.sr_tiny From 929f87d75466a3586918a35c7951161f899cfe66 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 10 Feb 2026 19:48:28 -0500 Subject: [PATCH 07/33] init sctype tests --- tests/data/create_data.nf | 3 ++ tests/data/templates/make-test-seurat.r | 45 +++++++++++++------ tests/modules/local/sctype/major/main.nf.test | 41 +++++++++++++++++ 3 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 tests/modules/local/sctype/major/main.nf.test diff --git a/tests/data/create_data.nf b/tests/data/create_data.nf index eaaa7dc..c24a030 100644 --- a/tests/data/create_data.nf +++ b/tests/data/create_data.nf @@ -5,6 +5,9 @@ process CREATE_DATA { container 'syedsazaidi/scratch-annotation:latest' + input: + path(cell_markers_database) + output: path("sr_tiny.rds") , emit: sr_tiny path("sr_ref_tiny.rds") , emit: sr_ref_tiny diff --git a/tests/data/templates/make-test-seurat.r b/tests/data/templates/make-test-seurat.r index 0a95afd..ad15d62 100644 --- a/tests/data/templates/make-test-seurat.r +++ b/tests/data/templates/make-test-seurat.r @@ -4,15 +4,23 @@ library(Seurat) set.seed(42) + +#get the cell_markers_database (normally from assets) +cmdb <- read.csv("${cell_markers_database}", stringsAsFactors = FALSE) + + #create a tiny Seurat object for testing purposes +n_genes <- 200 +n_cells <- 1000 +genes <- sample(unique(cmdb[["markers"]]), n_genes, replace = FALSE) sr <- CreateSeuratObject( counts = matrix( - data = rpois(n = 20000, lambda = 10), - nrow = 200, - ncol = 100, + data = rpois(n = n_genes * n_cells, lambda = 10), + nrow = n_genes, + ncol = n_cells, dimnames = list( - paste0("Gene", 1:200), - paste0("Cell", 1:100) + genes, + paste0("Cell", 1:n_cells) ) ) ) @@ -22,28 +30,37 @@ sr <- FindVariableFeatures(sr) sr <- ScaleData(sr, features = VariableFeatures(sr)) sr <- RunPCA(sr, features = VariableFeatures(sr)) +#sctype wants seurat_clusters in query +sr[["seurat_clusters"]] <- sample( + x = c("TypeA", "TypeB", "TypeC"), + size = n_cells, + replace = TRUE +) + saveRDS(sr, file = "sr_tiny.rds") -#create a tiny reference Seurat object with seurat_annotation column for +#create a tiny reference Seurat object with seurat_annotation column for azimuth anotation +n_cells_ref <- 1000 sr_ref <- CreateSeuratObject( counts = matrix( - data = rpois(n = 40000, lambda = 10), - nrow = 200, - ncol = 200, + data = rpois(n = n_genes * n_cells_ref, lambda = 10), + nrow = n_genes, + ncol = n_cells_ref, dimnames = list( - paste0("Gene", 1:200), - paste0("Cell", 1:200) + genes, + paste0("Cell", 1:n_cells_ref) ) ) ) - +#azimuth wants seurat annotations in ref sr_ref[["seurat_annotations"]] <- sample( - x = c("TypeA", "TypeB"), - size = 200, + x = c("TypeA", "TypeB", "TypeC"), + size = n_cells_ref, replace = TRUE ) + sr_ref <- NormalizeData(sr_ref) sr_ref <- FindVariableFeatures(sr_ref) sr_ref <- ScaleData(sr_ref, features = VariableFeatures(sr_ref)) diff --git a/tests/modules/local/sctype/major/main.nf.test b/tests/modules/local/sctype/major/main.nf.test new file mode 100644 index 0000000..425d86a --- /dev/null +++ b/tests/modules/local/sctype/major/main.nf.test @@ -0,0 +1,41 @@ +nextflow_process{ + name 'sctype_test' + script 'modules/local/sctype/major/main.nf' + process 'SCYTPE_MAJOR_ANNOTATION' + + + setup { + run("CREATE_DATA"){ + script "tests/data/create_data.nf" + params { + max_cpus = 2 + max_memory = '4 GB' + } + process { + """ + input[0] = file("${projectDir}/assets/cell_markers_database.csv") + """ + } + } + } + + test("Run Sctype") { + + tag "sctype" + + when { + process { + """ + input[0] = file("${projectDir}/modules/local/sctype/major/notebook_sctype_major.qmd") + input[1] = CREATE_DATA.out.sr_tiny + input[2] = file("${projectDir}/assets/cell_markers_database.csv") + input[3] = file(params.page_config) + """ + } + } + + then { + assert process.success + } + } +} \ No newline at end of file From 5a59fabe395254cd32c5935ed1276693c129c481 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Wed, 11 Feb 2026 11:27:22 -0500 Subject: [PATCH 08/33] increase genes to 2500 to avoid AddModuleScore() nbin error --- tests/data/templates/make-test-seurat.r | 26 ++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/data/templates/make-test-seurat.r b/tests/data/templates/make-test-seurat.r index ad15d62..61214b2 100644 --- a/tests/data/templates/make-test-seurat.r +++ b/tests/data/templates/make-test-seurat.r @@ -9,13 +9,20 @@ set.seed(42) cmdb <- read.csv("${cell_markers_database}", stringsAsFactors = FALSE) -#create a tiny Seurat object for testing purposes -n_genes <- 200 +# create a tiny Seurat object for testing purposes +# below 2500 Seurat AddModuleScore fails with error described below +# https://github.com/satijalab/seurat/issues/4819#issuecomment-2825615354 +n_markers <- 200 +n_genes <- 2500 n_cells <- 1000 -genes <- sample(unique(cmdb[["markers"]]), n_genes, replace = FALSE) +genes <- sample(unique(cmdb[["markers"]]), n_markers, replace = FALSE) + +# add some dummy genes to please Seurat's AddModuleScore and pipeline +genes <- c(genes, paste0("TEST", 1:(n_genes - length(genes)))) + sr <- CreateSeuratObject( counts = matrix( - data = rpois(n = n_genes * n_cells, lambda = 10), + data = rnbinom(n_genes * n_cells, size = 1, mu = 100), nrow = n_genes, ncol = n_cells, dimnames = list( @@ -30,7 +37,7 @@ sr <- FindVariableFeatures(sr) sr <- ScaleData(sr, features = VariableFeatures(sr)) sr <- RunPCA(sr, features = VariableFeatures(sr)) -#sctype wants seurat_clusters in query +# sctype module wants seurat_clusters in query sr[["seurat_clusters"]] <- sample( x = c("TypeA", "TypeB", "TypeC"), size = n_cells, @@ -39,11 +46,11 @@ sr[["seurat_clusters"]] <- sample( saveRDS(sr, file = "sr_tiny.rds") -#create a tiny reference Seurat object with seurat_annotation column for azimuth anotation +# a tiny reference Seurat object with seurat_annotation column for azimuth n_cells_ref <- 1000 sr_ref <- CreateSeuratObject( counts = matrix( - data = rpois(n = n_genes * n_cells_ref, lambda = 10), + data = rnbinom(n_genes * n_cells_ref, size = 1, mu = 100), nrow = n_genes, ncol = n_cells_ref, dimnames = list( @@ -53,17 +60,18 @@ sr_ref <- CreateSeuratObject( ) ) -#azimuth wants seurat annotations in ref +# azimuth wants seurat annotations in ref sr_ref[["seurat_annotations"]] <- sample( x = c("TypeA", "TypeB", "TypeC"), size = n_cells_ref, replace = TRUE ) - sr_ref <- NormalizeData(sr_ref) sr_ref <- FindVariableFeatures(sr_ref) sr_ref <- ScaleData(sr_ref, features = VariableFeatures(sr_ref)) + +# azimuth also wants PCA in ref sr_ref <- RunPCA(sr_ref, features = VariableFeatures(sr_ref)) saveRDS(sr_ref, file = "sr_ref_tiny.rds") From c13f1953528c98a4f910e8cb2cdf6fe14c846288 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Wed, 11 Feb 2026 12:42:44 -0500 Subject: [PATCH 09/33] add idents so sr object --- tests/data/templates/make-test-seurat.r | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data/templates/make-test-seurat.r b/tests/data/templates/make-test-seurat.r index 61214b2..2cec996 100644 --- a/tests/data/templates/make-test-seurat.r +++ b/tests/data/templates/make-test-seurat.r @@ -43,6 +43,7 @@ sr[["seurat_clusters"]] <- sample( size = n_cells, replace = TRUE ) +Idents(sr) <- "seurat_clusters" saveRDS(sr, file = "sr_tiny.rds") From 545921a4157b57974098f8d926d22a7b4d7f308c Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Wed, 11 Feb 2026 15:17:19 -0500 Subject: [PATCH 10/33] fix azimuth test input --- tests/modules/local/azimuth/main.nf.test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/modules/local/azimuth/main.nf.test b/tests/modules/local/azimuth/main.nf.test index 756aab9..623167c 100644 --- a/tests/modules/local/azimuth/main.nf.test +++ b/tests/modules/local/azimuth/main.nf.test @@ -12,7 +12,9 @@ nextflow_process{ max_memory = '4 GB' } process { - + """ + input[0] = file("${projectDir}/assets/cell_markers_database.csv") + """ } } } From 3d909ce3044882b2c7b9115ce972867a284b7586 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Thu, 12 Feb 2026 11:31:56 -0500 Subject: [PATCH 11/33] rm dot from nf-test out dir to fix quarto --- .gitignore | 4 +++- nf-test.config | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e55f15e..a773372 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ figures/ *.zip *.tar.gz *.sif -.nf-test* +_nf-test* # temporary files for nf-test - quarto does not produce _freeze and _report if testdir starts with dot + +/.quarto/ diff --git a/nf-test.config b/nf-test.config index 14fc5d4..44be616 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,6 +1,6 @@ config { testsDir "tests" - workDir ".nf-test" + workDir "_nf-test" configFile "tests/nextflow.config" profile "docker" } From ade8ff51b0ef94624ac9a5d7c6bee5b33d3ceccd Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Tue, 17 Feb 2026 13:34:18 -0500 Subject: [PATCH 12/33] save intermediate version (azimuth not being installed) --- docker/scratch-annotation.Dockerfile | 149 ++++++--------------------- 1 file changed, 32 insertions(+), 117 deletions(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index 02793e7..c6bae12 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -19,21 +19,17 @@ RUN apt-get update && apt-get install -y \ wget \ libcurl4-gnutls-dev \ libssl-dev \ - libxml2-dev + libxml2-dev \ + libgsl-dev \ + python3 \ + python3-pip \ + python3-venv # Updating quarto to Quarto v1.4.553 -RUN wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.553/quarto-1.4.553-linux-amd64.deb -O quarto-1.4.553-linux-amd64.deb -RUN dpkg -i quarto-1.4.553-linux-amd64.deb - -# # Install remotes package -# RUN R -e "install.packages('remotes')" - -# Install Python3 -# RUN apt-get install -y \ -# python3 \ -# python3-pip -RUN apt-get update && apt-get install -y python3 python3-pip python3-venv +RUN wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.553/quarto-1.4.553-linux-amd64.deb -O quarto-1.4.553-linux-amd64.deb \ + && dpkg -i quarto-1.4.553-linux-amd64.deb \ + && rm quarto-1.4.553-linux-amd64.deb # Install fundamental R packages ARG R_DEPS="c(\ @@ -53,7 +49,11 @@ ARG R_DEPS="c(\ ARG DEV_DEPS="c(\ 'bnprks/BPCells', \ 'cellgeni/sceasy', \ - 'immunogenomics/presto' \ + 'immunogenomics/presto', \ + 'PaulingLiu/ROGUE', \ + 'zhanghao-njmu/SCP', \ + 'immunogenomics/presto', \ + 'satijalab/azimuth' \ )" ARG WEB_DEPS="c(\ @@ -91,120 +91,35 @@ ARG R_BIOC_DEPS="c(\ 'scDblFinder' \ )" + +# Install Seurat Wrappers +RUN wget https://github.com/satijalab/seurat/archive/refs/heads/seurat5.zip -O /tmp/seurat-v5.zip \ + && wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /tmp/seurat-data.zip \ + && wget https://github.com/satijalab/seurat-wrappers/archive/refs/heads/seurat5.zip -O /tmp/seurat-wrappers.zip \ + && Rscript -e "devtools::install_local('/tmp/seurat-v5.zip')" \ + && Rscript -e "devtools::install_local('/tmp/seurat-data.zip')" \ + && Rscript -e "devtools::install_local('/tmp/seurat-wrappers.zip')" \ + && rm -rf /tmp/*.zip + # Setting repository URL ARG R_REPO="http://cran.us.r-project.org" # Caching R-lib on the building process -RUN Rscript -e "install.packages(${R_DEPS}, Ncpus = 8, repos = '${R_REPO}', clean = TRUE)" -RUN Rscript -e "install.packages(${WEB_DEPS}, Ncpus = 8, repos = '${R_REPO}', clean = TRUE)" - -# Install BiocManager -RUN Rscript -e "BiocManager::install(${R_BIOC_DEPS})" -RUN Rscript -e 'remotes::install_github("ctlab/fgsea")' - -# RUN Rscript -e 'BiocManager::install("readr", dependencies = TRUE)' -# RUN Rscript -e 'BiocManager::install("dplyr", dependencies = TRUE)' -# RUN Rscript -e 'BiocManager::install("ggplot2", dependencies = TRUE)' -# RUN Rscript -e 'BiocManager::install("Seurat", dependencies = TRUE)' -# RUN Rscript -e 'BiocManager::install("DT", dependencies = TRUE)' -# RUN Rscript -e 'BiocManager::install("SingleCellExperiment", dependencies = TRUE)' -# RUN Rscript -e 'BiocManager::install("scDblFinder", dependencies = TRUE, force = TRUE)' -# RUN Rscript -e 'BiocManager::install("lpsymphony", dependencies = TRUE, force = TRUE)' -# RUN Rscript -e 'BiocManager::install("IHW", dependencies = TRUE, force = TRUE)' -# RUN Rscript -e 'BiocManager::install("scp", dependencies = TRUE, force = TRUE)' -# RUN Rscript -e 'BiocManager::install(c("DOSE", "enrichplot", "clusterProfiler"), force = TRUE)' - -# Install Seurat Wrappers -RUN wget https://github.com/satijalab/seurat/archive/refs/heads/seurat5.zip -O /opt/seurat-v5.zip -RUN wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /opt/seurat-data.zip -RUN wget https://github.com/satijalab/seurat-wrappers/archive/refs/heads/seurat5.zip -O /opt/seurat-wrappers.zip - -RUN Rscript -e "devtools::install_local('/opt/seurat-v5.zip')" -RUN Rscript -e "devtools::install_local('/opt/seurat-data.zip')" -RUN Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')" - - -# Install SCP package from GitHub -RUN Rscript -e "remotes::install_github('bnprks/BPCells/r')" -RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, force = TRUE)" -# RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, force = TRUE)" -RUN Rscript -e "remotes::install_github('zhanghao-njmu/SCP', upgrade = 'always', dependencies = TRUE, force = TRUE)" - - -# # Download the Miniconda installer -# RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh && \ -# chmod +x /tmp/miniconda.sh && \ -# /tmp/miniconda.sh -b -p /opt/miniconda && \ -# rm /tmp/miniconda.sh - -# # Update PATH environment variable -# ENV PATH=/opt/miniconda/bin:$PATH - +RUN Rscript -e "install.packages(${R_DEPS}, Ncpus = 8, repos = '${R_REPO}', clean = TRUE)" \ + && Rscript -e "install.packages(${WEB_DEPS}, Ncpus = 8, repos = '${R_REPO}', clean = TRUE)" \ + && Rscript -e "BiocManager::install(${R_BIOC_DEPS})" \ + && Rscript -e "devtools::install_github(${DEV_DEPS})" -# # Install R packages -# RUN Rscript -e 'install.packages("remotes")' && \ -# Rscript -e 'remotes::install_github("zhanghao-njmu/SCP", upgrade = "always", force = TRUE, quiet = TRUE)' \ -# Rscript -e 'SCP::PrepareEnv( \ -# miniconda_repo = "https://mirrors.bfsu.edu.cn/anaconda/miniconda", \ -# pip_options = "-i https://pypi.tuna.tsinghua.edu.cn/simple")' -# Set the conda binary path and prepare the SCP environment -# RUN Rscript -e 'options(reticulate.conda_binary = "/opt/miniconda/bin/conda"); SCP::PrepareEnv(force = TRUE)' - - -# RUN Rscript -e 'renv::activate()' -# RUN wget https://github.com/zhanghao-njmu/SCP/archive/refs/heads/main.zip -O /opt/SCP.zip -# RUN unzip -o /opt/SCP.zip -d /opt/SCP -# RUN Rscript -e "devtools::install_local('/opt/SCP/SCP-main')" - - -# RUN Rscript -e "devtools::install_local('/opt/SCP.zip')" -# RUN Rscript -e 'devtools::install_github("zhanghao-njmu/SCP")' -# RUN Rscript -e 'remotes::install_github("zhanghao-njmu/SCP", dependencies = TRUE, force = TRUE)' - - -# Install packages on Github -RUN Rscript -e "devtools::install_github(${DEV_DEPS})" - - -# Create and activate virtual environment -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -# # Install Python packages related to cell annotation -# RUN python3 -m pip install --no-cache-dir scSpectra -# RUN python3 -m pip install --no-cache-dir celltypist -# RUN python3 -m pip install --no-cache-dir metatime -# RUN python3 -m pip install --no-cache-dir session_info -# # Set up venv and install Python packages -# RUN python3 -m venv /opt/venv \ -# && /opt/venv/bin/pip install --no-cache-dir scSpectra celltypist metatime session_info \ -# && ln -s /opt/venv/bin/python /usr/local/bin/python \ -# && ln -s /opt/venv/bin/pip /usr/local/bin/pip - - # Install Python packages for data science +# Install Python packages for data science RUN python3 -m venv /opt/venv \ - && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn matplotlib seaborn jupyter jupyter-cache papermill scSpectra celltypist metatime session_info \ + && /opt/venv/bin/pip install --no-cache-dir --upgrade pip \ + && /opt/venv/bin/pip install --no-cache-dir \ + numpy pandas scikit-learn matplotlib seaborn jupyter jupyter-cache \ + papermill scSpectra celltypist metatime session_info \ && ln -s /opt/venv/bin/python /usr/local/bin/python \ && ln -s /opt/venv/bin/pip /usr/local/bin/pip -# # RUN Rscript -e "install.packages(${R_ANNOT_DEPS}, Ncpus = 8, repos = '${R_REPO}', clean = TRUE)" -# RUN python3 -m venv /opt/venv -# ENV PATH="/opt/venv/bin:$PATH" -# Setting celltypist variable -ENV CELLTYPIST_FOLDER=/opt/celltypist - -# Installing celltypist models -COPY docker/setup.py /opt/ -RUN python3 /opt/setup.py - -# Install presto -RUN Rscript -e "install.packages('devtools')" -RUN Rscript -e "devtools::install_github('immunogenomics/presto')" - -# Install Azimuth -# RUN Rscript -e "devtools::install_github('satijalab/azimuth')" - # Set the working directory WORKDIR /data From 65614a3cbb228430be256eaea90b217faa73115e Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Wed, 18 Feb 2026 18:52:43 -0500 Subject: [PATCH 13/33] temp switch to local docker --- tests/modules/local/azimuth/main.nf.test | 5 +++++ tests/modules/local/sctype/major/main.nf.test | 4 ++-- tests/nextflow.config | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/modules/local/azimuth/main.nf.test b/tests/modules/local/azimuth/main.nf.test index 623167c..d7f87e9 100644 --- a/tests/modules/local/azimuth/main.nf.test +++ b/tests/modules/local/azimuth/main.nf.test @@ -3,6 +3,8 @@ nextflow_process{ script 'modules/local/azimuth/main.nf' process 'AZIMUTH_ANNOTATION' + config 'nextflow.config' + setup { run("CREATE_DATA"){ @@ -20,6 +22,9 @@ nextflow_process{ } test("Run Azimuth") { + + tag 'azimuth' + config 'tests/nextflow.config' when { process { diff --git a/tests/modules/local/sctype/major/main.nf.test b/tests/modules/local/sctype/major/main.nf.test index 425d86a..ad41906 100644 --- a/tests/modules/local/sctype/major/main.nf.test +++ b/tests/modules/local/sctype/major/main.nf.test @@ -3,7 +3,6 @@ nextflow_process{ script 'modules/local/sctype/major/main.nf' process 'SCYTPE_MAJOR_ANNOTATION' - setup { run("CREATE_DATA"){ script "tests/data/create_data.nf" @@ -21,7 +20,8 @@ nextflow_process{ test("Run Sctype") { - tag "sctype" + tag 'sctype' + config 'tests/nextflow.config' when { process { diff --git a/tests/nextflow.config b/tests/nextflow.config index c19b1ad..051cd55 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -3,3 +3,9 @@ Nextflow config file for running tests ======================================================================================== */ + +process { + withName: 'SCYTPE_MAJOR_ANNOTATION' { + container = 'szaidi:latest' + } +} \ No newline at end of file From cedd14c548e0aaf349dd80dca504458ec4f9e63a Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Thu, 19 Feb 2026 15:56:23 -0500 Subject: [PATCH 14/33] change cran repo --- docker/scratch-annotation.Dockerfile | 78 ++++++++++++++++++---------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index a58da26..d0c3531 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -12,6 +12,8 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ ARG GITHUB_PAT ENV GITHUB_PAT=${GITHUB_PAT} +ENV R_REPOS="https://packagemanager.posit.co/cran/latest" + # Install system dependencies RUN apt-get update && apt-get install -y \ software-properties-common \ @@ -45,19 +47,52 @@ RUN dpkg -i quarto-1.4.553-linux-amd64.deb RUN apt-get update && apt-get install -y python3 python3-pip python3-venv # Install core R packages -RUN Rscript -e "install.packages(c('R.utils','rmarkdown','devtools','tidyverse','readr', 'dplyr', 'ggplot2', 'cowplot', 'remotes', 'BiocManager','reticulate', 'HGNChelper', 'optparse'), repos='http://cran.us.r-project.org')" +RUN Rscript -e "install.packages(c( \ + 'R.utils', \ + 'rmarkdown', \ + 'devtools', \ + 'tidyverse', \ + 'readr', \ + 'dplyr', \ + 'ggplot2', \ + 'cowplot', \ + 'remotes', \ + 'BiocManager',\ + 'reticulate', \ + 'HGNChelper', \ + 'optparse'), repos='${R_REPOS}')" + +RUN Rscript -e "BiocManager::install(c( \ + 'S4Vectors', \ + 'DelayedMatrixStats', \ + 'BiocGenerics',\ + 'Biobase', \ + 'SummarizedExperiment', \ + 'AnnotationDbi', \ + 'org.Hs.eg.db'), ask=FALSE, update=TRUE)" -RUN Rscript -e "BiocManager::install(c('S4Vectors','DelayedMatrixStats','BiocGenerics','Biobase', 'SummarizedExperiment', 'AnnotationDbi', 'org.Hs.eg.db'), ask=FALSE, update=TRUE)" RUN Rscript -e "BiocManager::install(c( \ - 'HDF5Array','rhdf5','rhdf5lib', \ + 'HDF5Array', \ + 'rhdf5', \ + 'rhdf5lib', \ 'SingleCellExperiment', \ - 'GOSemSim','MatrixGenerics','treeio','DOSE','ggtree','enrichplot', \ - 'clusterProfiler','DirichletMultinomial','rtracklayer','GenomicFeatures', \ - 'BSgenome','ensembldb','TFBSTools', \ - 'BSgenome.Hsapiens.UCSC.hg38','EnsDb.Hsapiens.v86' \ - ), \ - ask=FALSE, update=FALSE )" + 'GOSemSim', \ + 'MatrixGenerics', \ + 'treeio', \ + 'DOSE',\ + 'ggtree',\ + 'enrichplot', \ + 'clusterProfiler',\ + 'DirichletMultinomial',\ + 'rtracklayer',\ + 'GenomicFeatures', \ + 'BSgenome',\ + 'ensembldb',\ + 'TFBSTools', \ + 'BSgenome.Hsapiens.UCSC.hg38'\ + ,'EnsDb.Hsapiens.v86' \ + ), ask=FALSE, update=FALSE )" # Setting repository URL ARG R_REPO="http://cran.us.r-project.org" @@ -82,10 +117,9 @@ RUN Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')" # Install SCP package from GitHub # RUN Rscript -e "remotes::install_github('bnprks/BPCells/r')" -RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, force = TRUE)" -# RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, force = TRUE)" -RUN Rscript -e "remotes::install_github('zhanghao-njmu/SCP', upgrade = 'always', dependencies = TRUE, force = TRUE)" -RUN Rscript -e "remotes::install_github('cellgeni/sceasy', upgrade = 'always', dependencies = TRUE, force = TRUE)" +RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'always',force = TRUE)" +RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'always', force = TRUE)" +RUN Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'always', force = TRUE)" @@ -111,8 +145,8 @@ ENV PATH="/opt/venv/bin:$PATH" ENV CELLTYPIST_FOLDER=/opt/celltypist # Installing celltypist models -COPY setup.py /opt/ -RUN python3 /opt/setup.py +# COPY setup.py /opt/ +# RUN python3 /opt/setup.py # Additional packages RUN apt-get install -y libhdf5-dev @@ -131,14 +165,6 @@ RUN Rscript -e "devtools::install_github('stephenturner/annotables')" # RUN Rscript -e "install.packages('rJava', repos = 'http://cran.us.r-project.org')" RUN Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')" -# RUN Rscript -e "devtools::install_github('AntonioDeFalco/SCEVAN', dependencies = TRUE, upgrade = 'never')" -# install SCP -RUN Rscript -e "remotes::install_github( \ - 'zhanghao-njmu/SCP', \ - dependencies=TRUE, \ - upgrade='always', \ - auth_token = Sys.getenv('GITHUB_PAT'))" - RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -149,11 +175,7 @@ RUN apt-get update && \ RUN Rscript -e "install.packages(c('DirichletMultinomial','TFBSTools'), dependencies = TRUE, repos = BiocManager::repositories())" # install Azimuth -RUN Rscript -e "remotes::install_github( \ - 'satijalab/azimuth', ref = 'master', \ - dependencies=TRUE, \ - upgrade='always', \ - auth_token = Sys.getenv('GITHUB_PAT'))" +RUN Rscript -e "devtools::install_github('satijalab/azimuth', ref = 'master', dependencies=TRUE, upgrade='never')" # Cleaning apt-get cache From 2b30766e975f648de721539a861c4ed39b4a6444 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Thu, 19 Feb 2026 15:58:42 -0500 Subject: [PATCH 15/33] rm obsolete code --- docker/scratch-annotation.Dockerfile | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index d0c3531..63a6453 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -94,16 +94,9 @@ RUN Rscript -e "BiocManager::install(c( \ ,'EnsDb.Hsapiens.v86' \ ), ask=FALSE, update=FALSE )" -# Setting repository URL -ARG R_REPO="http://cran.us.r-project.org" - -# # Install BiocManager -# RUN Rscript -e "BiocManager::install(${R_BIOC_DEPS})" RUN Rscript -e 'remotes::install_github("ctlab/fgsea")' - - # Install Seurat Wrappers RUN wget https://github.com/satijalab/seurat/archive/refs/heads/seurat5.zip -O /opt/seurat-v5.zip RUN wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /opt/seurat-data.zip @@ -113,25 +106,15 @@ RUN Rscript -e "devtools::install_local('/opt/seurat-v5.zip')" RUN Rscript -e "devtools::install_local('/opt/seurat-data.zip')" RUN Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')" - - # Install SCP package from GitHub -# RUN Rscript -e "remotes::install_github('bnprks/BPCells/r')" RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'always',force = TRUE)" RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'always', force = TRUE)" RUN Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'always', force = TRUE)" - - - # Create and activate virtual environment RUN python3 -m venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" -# # Upgrade pip and install Python packages in venv -# RUN pip install --upgrade pip && \ -# pip install numpy pandas scikit-learn matplotlib seaborn jupyter jupyter-cache papermill - # Create and activate a virtual environment before installing Python packages RUN python3 -m venv /opt/venv \ && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn matplotlib seaborn jupyter jupyter-cache papermill anndata scanpy scipy session_info scSpectra metatime celltypist \ @@ -162,7 +145,6 @@ RUN apt-get install -y jags # Install annotables RUN Rscript -e "devtools::install_github('stephenturner/annotables')" -# RUN Rscript -e "install.packages('rJava', repos = 'http://cran.us.r-project.org')" RUN Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')" @@ -170,8 +152,8 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ libgsl-dev \ && rm -rf /var/lib/apt/lists/* - # Install DirichletMultinomial + TFBSTools (and all of their R & Bioc deps) +# Install DirichletMultinomial + TFBSTools (and all of their R & Bioc deps) RUN Rscript -e "install.packages(c('DirichletMultinomial','TFBSTools'), dependencies = TRUE, repos = BiocManager::repositories())" # install Azimuth From 945b7adb5715d8323ef3a7f45e75caccbceb5d06 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Thu, 19 Feb 2026 20:32:30 -0500 Subject: [PATCH 16/33] move apt tasks to one run --- docker/scratch-annotation.Dockerfile | 39 ++++++++-------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index 63a6453..310ff67 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -33,18 +33,23 @@ RUN apt-get update && apt-get install -y \ libpng-dev \ libtiff5-dev \ zlib1g-dev \ - libxt-dev + libxt-dev \ + default-jre \ + libgfortran5 \ + jags \ + python3 \ + python3-pip \ + python3-venv \ + libhdf5-dev \ + libgsl-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # Updating quarto to Quarto v1.4.553 RUN wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.553/quarto-1.4.553-linux-amd64.deb -O quarto-1.4.553-linux-amd64.deb RUN dpkg -i quarto-1.4.553-linux-amd64.deb -# # Install remotes package -# RUN R -e "install.packages('remotes')" - - -RUN apt-get update && apt-get install -y python3 python3-pip python3-venv # Install core R packages RUN Rscript -e "install.packages(c( \ @@ -131,40 +136,18 @@ ENV CELLTYPIST_FOLDER=/opt/celltypist # COPY setup.py /opt/ # RUN python3 /opt/setup.py -# Additional packages -RUN apt-get install -y libhdf5-dev RUN Rscript -e "install.packages('hdf5r')" - -# Java + Fortran -RUN apt-get update && apt-get install -y default-jre libgfortran5 - -# JAGS -RUN apt-get install -y jags - - # Install annotables RUN Rscript -e "devtools::install_github('stephenturner/annotables')" RUN Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - libgsl-dev \ - && rm -rf /var/lib/apt/lists/* - # Install DirichletMultinomial + TFBSTools (and all of their R & Bioc deps) RUN Rscript -e "install.packages(c('DirichletMultinomial','TFBSTools'), dependencies = TRUE, repos = BiocManager::repositories())" # install Azimuth RUN Rscript -e "devtools::install_github('satijalab/azimuth', ref = 'master', dependencies=TRUE, upgrade='never')" - -# Cleaning apt-get cache -RUN apt-get clean -RUN rm -rf /var/lib/apt/lists/* - - # Command to run on container start CMD ["bash"] From 5a678912d2074ee9783601a86769cd1ad767007c Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Fri, 20 Feb 2026 07:12:35 -0500 Subject: [PATCH 17/33] bump and fix seurat to pass tests --- docker/scratch-annotation.Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index 310ff67..b2ee437 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -103,7 +103,7 @@ RUN Rscript -e "BiocManager::install(c( \ RUN Rscript -e 'remotes::install_github("ctlab/fgsea")' # Install Seurat Wrappers -RUN wget https://github.com/satijalab/seurat/archive/refs/heads/seurat5.zip -O /opt/seurat-v5.zip +RUN wget https://github.com/satijalab/seurat/archive/refs/tags/v5.3.1.zip -O /opt/seurat-v5.zip RUN wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /opt/seurat-data.zip RUN wget https://github.com/satijalab/seurat-wrappers/archive/refs/heads/seurat5.zip -O /opt/seurat-wrappers.zip @@ -112,9 +112,9 @@ RUN Rscript -e "devtools::install_local('/opt/seurat-data.zip')" RUN Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')" # Install SCP package from GitHub -RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'always',force = TRUE)" -RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'always', force = TRUE)" -RUN Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'always', force = TRUE)" +RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'never',force = TRUE)" +RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'never', force = TRUE)" +RUN Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'never', force = TRUE)" # Create and activate virtual environment RUN python3 -m venv /opt/venv From 9e635b0cc16f3edbf8917276fc128b32bd789301 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 10:42:02 -0500 Subject: [PATCH 18/33] cleanup and rearrange --- docker/scratch-annotation.Dockerfile | 75 +++++++++++----------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index b2ee437..06915dd 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -6,11 +6,9 @@ WORKDIR /opt # Timezone settings ENV TZ=US/Central -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo $TZ > /etc/timezone -# pass your PAT at build time so remotes::install_github can auth -ARG GITHUB_PAT -ENV GITHUB_PAT=${GITHUB_PAT} +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone + ENV R_REPOS="https://packagemanager.posit.co/cran/latest" @@ -65,9 +63,11 @@ RUN Rscript -e "install.packages(c( \ 'BiocManager',\ 'reticulate', \ 'HGNChelper', \ - 'optparse'), repos='${R_REPOS}')" - + 'hdf5r', \ + 'optparse' \ + ), repos='${R_REPOS}')" +# Bioconductor packages RUN Rscript -e "BiocManager::install(c( \ 'S4Vectors', \ 'DelayedMatrixStats', \ @@ -75,9 +75,9 @@ RUN Rscript -e "BiocManager::install(c( \ 'Biobase', \ 'SummarizedExperiment', \ 'AnnotationDbi', \ - 'org.Hs.eg.db'), ask=FALSE, update=TRUE)" - -RUN Rscript -e "BiocManager::install(c( \ + 'org.Hs.eg.db' \ + ), ask=FALSE, update=TRUE)" \ + && Rscript -e "BiocManager::install(c( \ 'HDF5Array', \ 'rhdf5', \ 'rhdf5lib', \ @@ -102,52 +102,37 @@ RUN Rscript -e "BiocManager::install(c( \ RUN Rscript -e 'remotes::install_github("ctlab/fgsea")' -# Install Seurat Wrappers -RUN wget https://github.com/satijalab/seurat/archive/refs/tags/v5.3.1.zip -O /opt/seurat-v5.zip -RUN wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /opt/seurat-data.zip -RUN wget https://github.com/satijalab/seurat-wrappers/archive/refs/heads/seurat5.zip -O /opt/seurat-wrappers.zip - -RUN Rscript -e "devtools::install_local('/opt/seurat-v5.zip')" -RUN Rscript -e "devtools::install_local('/opt/seurat-data.zip')" -RUN Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')" +# Seurat and wrappers +RUN wget https://github.com/satijalab/seurat/archive/refs/tags/v5.3.1.zip -O /opt/seurat-v5.zip \ + && wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /opt/seurat-data.zip \ + && wget https://github.com/satijalab/seurat-wrappers/archive/refs/heads/seurat5.zip -O /opt/seurat-wrappers.zip \ + && Rscript -e "devtools::install_local('/opt/seurat-v5.zip')" \ + && Rscript -e "devtools::install_local('/opt/seurat-data.zip')" \ + && Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')" \ + && rm /opt/seurat-v5.zip /opt/seurat-data.zip /opt/seurat-wrappers.zip \ + && Rscript -e "devtools::install_github('satijalab/azimuth', ref = 'master', dependencies=TRUE, upgrade='never')" # Install SCP package from GitHub -RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'never',force = TRUE)" -RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'never', force = TRUE)" -RUN Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'never', force = TRUE)" +RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'never',force = TRUE)" \ + && Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'never', force = TRUE)" \ + && Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'never', force = TRUE)" -# Create and activate virtual environment -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" +# Install annotables +RUN Rscript -e "devtools::install_github('stephenturner/annotables')" \ + && Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')" + # Create and activate a virtual environment before installing Python packages RUN python3 -m venv /opt/venv \ - && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn matplotlib seaborn jupyter jupyter-cache papermill anndata scanpy scipy session_info scSpectra metatime celltypist \ + && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn \ + matplotlib seaborn jupyter jupyter-cache papermill anndata scanpy scipy \ + session_info scSpectra metatime celltypist \ && ln -s /opt/venv/bin/python /usr/local/bin/python \ && ln -s /opt/venv/bin/pip /usr/local/bin/pip -# Create and activate virtual environment -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" -# Setting celltypist variable +ENV PATH="/opt/venv/bin:$PATH" ENV CELLTYPIST_FOLDER=/opt/celltypist -# Installing celltypist models -# COPY setup.py /opt/ -# RUN python3 /opt/setup.py - -RUN Rscript -e "install.packages('hdf5r')" - -# Install annotables -RUN Rscript -e "devtools::install_github('stephenturner/annotables')" -RUN Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')" - -# Install DirichletMultinomial + TFBSTools (and all of their R & Bioc deps) -RUN Rscript -e "install.packages(c('DirichletMultinomial','TFBSTools'), dependencies = TRUE, repos = BiocManager::repositories())" - -# install Azimuth -RUN Rscript -e "devtools::install_github('satijalab/azimuth', ref = 'master', dependencies=TRUE, upgrade='never')" - # Command to run on container start CMD ["bash"] From 1262cd7d755e5a44036e656901bbd42e1375c741 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 11:09:02 -0500 Subject: [PATCH 19/33] rm gpu support due to image size --- docker/scratch-annotation.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index 06915dd..b8e00ba 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -124,6 +124,9 @@ RUN Rscript -e "devtools::install_github('stephenturner/annotables')" \ # Create and activate a virtual environment before installing Python packages RUN python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir --upgrade pip \ + #install torch with cpu support only to avoid huge CUDA + && /opt/venv/bin/pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu \ && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn \ matplotlib seaborn jupyter jupyter-cache papermill anndata scanpy scipy \ session_info scSpectra metatime celltypist \ From 7bb2bbb1395520ad1c57e81df4b28367c0307929 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 13:51:05 -0500 Subject: [PATCH 20/33] rm reactome.db due to size --- docker/scratch-annotation.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index b8e00ba..13cd3fd 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -115,7 +115,9 @@ RUN wget https://github.com/satijalab/seurat/archive/refs/tags/v5.3.1.zip -O /op # Install SCP package from GitHub RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'never',force = TRUE)" \ && Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'never', force = TRUE)" \ - && Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'never', force = TRUE)" + && Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'never', force = TRUE)" \ + # uninstall reactome.db that SCP pulls (1.7Gb) + && Rscript -e "remove.packages('reactome.db')" # Install annotables RUN Rscript -e "devtools::install_github('stephenturner/annotables')" \ From 656906c0cce7df1dc91a968df507e6f94c0dec13 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 13:51:49 -0500 Subject: [PATCH 21/33] add docker build action --- .github/workflows/build-push-container.yml | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/build-push-container.yml diff --git a/.github/workflows/build-push-container.yml b/.github/workflows/build-push-container.yml new file mode 100644 index 0000000..d4975b2 --- /dev/null +++ b/.github/workflows/build-push-container.yml @@ -0,0 +1,59 @@ +# +name: build-push-container + +# Configures this workflow to run every time a change is pushed to the branch called `release`. +on: + push: + paths: + - '**/env.yml' + - '**/Dockerfile' + - '**/*.dockerfile' + - '.github/workflows/build-push-container.yml' + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,format=short,prefix= # Generates a tag like '860c190' + main + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From ac6c79933c07af6eb9ea6c0abfb744f3a69f9faa Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 13:59:07 -0500 Subject: [PATCH 22/33] add sctype state test --- tests/modules/local/sctype/state/main.nf.test | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/modules/local/sctype/state/main.nf.test diff --git a/tests/modules/local/sctype/state/main.nf.test b/tests/modules/local/sctype/state/main.nf.test new file mode 100644 index 0000000..b783497 --- /dev/null +++ b/tests/modules/local/sctype/state/main.nf.test @@ -0,0 +1,56 @@ +nextflow_process{ + name 'sctype_test' + script 'modules/local/sctype/state/main.nf' + process 'SCYTPE_STATE_ANNOTATION' + + setup { + run("CREATE_DATA"){ + script "tests/data/create_data.nf" + params { + max_cpus = 2 + max_memory = '4 GB' + } + process { + """ + input[0] = file("${projectDir}/assets/cell_markers_database.csv") + """ + } + } + run("SCYTPE_MAJOR_ANNOTATION"){ + script "modules/local/sctype/major/main.nf" + process { + """ + input[0] = file("${projectDir}/modules/local/sctype/major/notebook_sctype_major.qmd") + input[1] = CREATE_DATA.out.sr_tiny + input[2] = file("${projectDir}/assets/cell_markers_database.csv") + input[3] = file(params.page_config) + """ + } + } + } + + test("Run Sctype States") { + + tag 'sctype' + tag 'state' + config 'tests/nextflow.config' + + when { + process { + """ + input[0] = file("${projectDir}/modules/local/sctype/state/notebook_sctype_states.qmd") + input[1] = SCYTPE_MAJOR_ANNOTATION.out.seurat_rds + input[2] = file("${projectDir}/assets/cell_markers_database.csv") + input[3] = SCYTPE_MAJOR_ANNOTATION.out.major_list.splitText().map{ iter -> iter.split(":") } + .filter{ !(it[0] =~ "Unknown|Epithelial|Fibroblast|NK_Cells") } + .map{ it[0].trim() } + input[4] = file(params.page_config) + """ + } + } + + then { + assert process.success + } + } +} \ No newline at end of file From 15bc66cda79a3a8b1d281f80e3c3223c1579e439 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 14:08:26 -0500 Subject: [PATCH 23/33] temp disable tests action --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75c372f..59f5a14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,9 +2,9 @@ name: test on: - pull_request: - branches: - - 'main' + # pull_request: + # branches: + # - 'main' workflow_dispatch: jobs: test: From 945c4060df4a5a65dda09cc51480ca1c5c7ae13a Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 14:23:45 -0500 Subject: [PATCH 24/33] update dockerfile path --- .github/workflows/build-push-container.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-container.yml b/.github/workflows/build-push-container.yml index d4975b2..4a929bf 100644 --- a/.github/workflows/build-push-container.yml +++ b/.github/workflows/build-push-container.yml @@ -5,9 +5,7 @@ name: build-push-container on: push: paths: - - '**/env.yml' - - '**/Dockerfile' - - '**/*.dockerfile' + - 'docker/scratch-annotation.Dockerfile' - '.github/workflows/build-push-container.yml' workflow_dispatch: @@ -53,6 +51,7 @@ jobs: id: push uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: + file: docker/scratch-annotation.Dockerfile context: . push: true tags: ${{ steps.meta.outputs.tags }} From 3faf4fe3578c0a91db3e7a7cc91d0b257f4d0e97 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 15:40:37 -0500 Subject: [PATCH 25/33] re-enable test workflow, use new image --- .github/workflows/test.yml | 6 +++--- tests/nextflow.config | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59f5a14..75c372f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,9 +2,9 @@ name: test on: - # pull_request: - # branches: - # - 'main' + pull_request: + branches: + - 'main' workflow_dispatch: jobs: test: diff --git a/tests/nextflow.config b/tests/nextflow.config index 051cd55..f24da73 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -6,6 +6,12 @@ process { withName: 'SCYTPE_MAJOR_ANNOTATION' { - container = 'szaidi:latest' + container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' + } + withName: 'SCYTPE_STATE_ANNOTATION' { + container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' + } + withName: 'AZIMUTH_ANNOTATION' { + container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' } } \ No newline at end of file From 1e8583408a337e2429163fde35d5bacc2e1285bd Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 16:05:31 -0500 Subject: [PATCH 26/33] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/workflows/test.yml | 4 ++-- docker/scratch-annotation.Dockerfile | 10 ++++------ tests/modules/local/azimuth/main.nf.test | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75c372f..6dbd57b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,12 +10,12 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive lfs: true - - uses: actions/setup-java@v2 + - uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '17' diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index 13cd3fd..c21dfd8 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -32,8 +32,7 @@ RUN apt-get update && apt-get install -y \ libtiff5-dev \ zlib1g-dev \ libxt-dev \ - default-jre \ - libgfortran5 \ + jags \ python3 \ python3-pip \ @@ -95,8 +94,8 @@ RUN Rscript -e "BiocManager::install(c( \ 'BSgenome',\ 'ensembldb',\ 'TFBSTools', \ - 'BSgenome.Hsapiens.UCSC.hg38'\ - ,'EnsDb.Hsapiens.v86' \ + 'BSgenome.Hsapiens.UCSC.hg38', \ + 'EnsDb.Hsapiens.v86' \ ), ask=FALSE, update=FALSE )" @@ -113,7 +112,7 @@ RUN wget https://github.com/satijalab/seurat/archive/refs/tags/v5.3.1.zip -O /op && Rscript -e "devtools::install_github('satijalab/azimuth', ref = 'master', dependencies=TRUE, upgrade='never')" # Install SCP package from GitHub -RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'never',force = TRUE)" \ +RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'never', force = TRUE)" \ && Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'never', force = TRUE)" \ && Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'never', force = TRUE)" \ # uninstall reactome.db that SCP pulls (1.7Gb) @@ -122,7 +121,6 @@ RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE # Install annotables RUN Rscript -e "devtools::install_github('stephenturner/annotables')" \ && Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')" - # Create and activate a virtual environment before installing Python packages RUN python3 -m venv /opt/venv \ diff --git a/tests/modules/local/azimuth/main.nf.test b/tests/modules/local/azimuth/main.nf.test index d7f87e9..262cd89 100644 --- a/tests/modules/local/azimuth/main.nf.test +++ b/tests/modules/local/azimuth/main.nf.test @@ -3,7 +3,7 @@ nextflow_process{ script 'modules/local/azimuth/main.nf' process 'AZIMUTH_ANNOTATION' - config 'nextflow.config' + setup { From 957a08f5f6d522507c8df1b756aba7562e902b1a Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Sat, 21 Feb 2026 16:10:52 -0500 Subject: [PATCH 27/33] implement review suggestions --- docker/scratch-annotation.Dockerfile | 4 ++-- tests/data/create_data.nf | 10 ++++------ tests/nextflow.config | 3 +++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docker/scratch-annotation.Dockerfile b/docker/scratch-annotation.Dockerfile index c21dfd8..f26dfab 100644 --- a/docker/scratch-annotation.Dockerfile +++ b/docker/scratch-annotation.Dockerfile @@ -122,10 +122,10 @@ RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE RUN Rscript -e "devtools::install_github('stephenturner/annotables')" \ && Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')" -# Create and activate a virtual environment before installing Python packages +# Create and activate a virtual environment before installing Python packages +# Note the no-gpu version of torch to avoid ~5Gb dependencies and keep image size smaller RUN python3 -m venv /opt/venv \ && /opt/venv/bin/pip install --no-cache-dir --upgrade pip \ - #install torch with cpu support only to avoid huge CUDA && /opt/venv/bin/pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu \ && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn \ matplotlib seaborn jupyter jupyter-cache papermill anndata scanpy scipy \ diff --git a/tests/data/create_data.nf b/tests/data/create_data.nf index c24a030..19f544b 100644 --- a/tests/data/create_data.nf +++ b/tests/data/create_data.nf @@ -1,16 +1,14 @@ process CREATE_DATA { - tag "Creating test data for annotation" - label 'process_medium' - - container 'syedsazaidi/scratch-annotation:latest' + tag 'Creating test data for annotation' + label 'process_medium' input: path(cell_markers_database) output: - path("sr_tiny.rds") , emit: sr_tiny - path("sr_ref_tiny.rds") , emit: sr_ref_tiny + path('sr_tiny.rds') , emit: sr_tiny + path('sr_ref_tiny.rds') , emit: sr_ref_tiny script: template 'make-test-seurat.r' diff --git a/tests/nextflow.config b/tests/nextflow.config index f24da73..28db466 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -14,4 +14,7 @@ process { withName: 'AZIMUTH_ANNOTATION' { container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' } + withName: 'CREATE_DATA' { + container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' + } } \ No newline at end of file From 19c1b10cb1b865be546b71c3e2699b6e864c5642 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Wed, 4 Mar 2026 19:01:54 -0500 Subject: [PATCH 28/33] add tests, improve test data creation --- tests/data/templates/make-test-seurat.r | 154 ++++++++++-------- .../local/sctype/aggregate/main.nf.test | 67 ++++++++ tests/modules/local/sctype/major/main.nf.test | 1 + tests/modules/local/sctype/state/main.nf.test | 2 +- tests/nextflow.config | 3 + 5 files changed, 161 insertions(+), 66 deletions(-) create mode 100644 tests/modules/local/sctype/aggregate/main.nf.test diff --git a/tests/data/templates/make-test-seurat.r b/tests/data/templates/make-test-seurat.r index 2cec996..d7fd523 100644 --- a/tests/data/templates/make-test-seurat.r +++ b/tests/data/templates/make-test-seurat.r @@ -4,75 +4,99 @@ library(Seurat) set.seed(42) - -#get the cell_markers_database (normally from assets) +# get the cell_markers_database (normally from assets) cmdb <- read.csv("${cell_markers_database}", stringsAsFactors = FALSE) - # create a tiny Seurat object for testing purposes -# below 2500 Seurat AddModuleScore fails with error described below -# https://github.com/satijalab/seurat/issues/4819#issuecomment-2825615354 -n_markers <- 200 -n_genes <- 2500 -n_cells <- 1000 -genes <- sample(unique(cmdb[["markers"]]), n_markers, replace = FALSE) - -# add some dummy genes to please Seurat's AddModuleScore and pipeline -genes <- c(genes, paste0("TEST", 1:(n_genes - length(genes)))) - -sr <- CreateSeuratObject( - counts = matrix( - data = rnbinom(n_genes * n_cells, size = 1, mu = 100), - nrow = n_genes, - ncol = n_cells, - dimnames = list( - genes, - paste0("Cell", 1:n_cells) - ) - ) -) - -sr <- NormalizeData(sr) -sr <- FindVariableFeatures(sr) -sr <- ScaleData(sr, features = VariableFeatures(sr)) -sr <- RunPCA(sr, features = VariableFeatures(sr)) - -# sctype module wants seurat_clusters in query -sr[["seurat_clusters"]] <- sample( - x = c("TypeA", "TypeB", "TypeC"), - size = n_cells, - replace = TRUE -) -Idents(sr) <- "seurat_clusters" - +make_small_seurat <- function(cmdb, ncell_each = 100) { + # below 2500 genes Seurat AddModuleScore fails with error described below + # https://github.com/satijalab/seurat/issues/4819#issuecomment-2825615354 + + lineage <- cmdb[cmdb["parent_level"] == "Lineage_markers", + c("parent_level", "cell_annotation", "markers")] + detailed <- cmdb[cmdb["parent_level"] != "Lineage_markers", + c("parent_level", "cell_annotation", "markers")] + cell_types <- unique(detailed[["cell_annotation"]]) + + # add cells that are in the lineage but not in the detailed list + missing <- setdiff(unique(lineage[["cell_annotation"]]), + unique(detailed[["parent_level"]])) + cell_types <- c(cell_types, missing) + + # create counts from negative binomial based on the cell types in the cmdb + counts_list <- lapply(cell_types, function(cell_type) { + cell_type_genes <- detailed[detailed[["cell_annotation"]] == cell_type, + "markers"] + + parent <- detailed[["parent_level"]][detailed[["cell_annotation"]]==cell_type] + parent <- unique(parent) + + #check that the detailed cell type has only one parent in the cmdb + if(length(parent) > 1) { + stop(paste("Cell type", cell_type, "has multiple parents in the cmdb")) + } else if (length(parent)==0) { + parent <- cell_type # if the cell type is not in the lineage, it is its own parent + } + + lineage_genes <- lineage[lineage[["cell_annotation"]] == parent, + "markers"] + cell_type_genes <- unique(c(lineage_genes, cell_type_genes)) + + ct <- matrix( + data = rnbinom(length(cell_type_genes) * ncell_each, size = 1, mu = 100), + nrow = length(cell_type_genes), + ncol = ncell_each, + dimnames = list( + cell_type_genes, + paste0("Cell", 1:ncell_each) + ) + ) |> t() |> as.data.frame() + ct[["major_type"]] <- parent # preserve cell names for after rbind.fill + ct + }) + + counts <- do.call(plyr::rbind.fill, counts_list) + counts[is.na(counts)] <- 0 + rownames(counts) <- paste0(counts[["major_type"]], 1:nrow(counts)) + counts["major_type"] <- NULL + + # if resulting counts has less than 2500 genes, add random genes to reach 2500 + if (ncol(counts) < 2500) { + n_genes_to_add <- 2500 - ncol(counts) + random_genes <- paste0("DUMMY", seq_len(n_genes_to_add)) + random_counts <- matrix( + data = rnbinom(nrow(counts) * n_genes_to_add, size = 1, mu = 100), + nrow = nrow(counts), + ncol = n_genes_to_add, + dimnames = list(rownames(counts), random_genes) + ) |> as.data.frame() + #set 80% of the random counts to 0 to make them more realistic + zeroes <- sample(length(random_counts), size = length(random_counts) * 0.8) + random_counts[zeroes] <- 0 + counts <- cbind(counts, random_counts) + } + + sr <- CreateSeuratObject(counts = t(as.matrix(counts))) + sr <- NormalizeData(sr) + sr <- FindVariableFeatures(sr) + sr <- ScaleData(sr, features = VariableFeatures(sr)) + sr <- RunPCA(sr, features = VariableFeatures(sr)) + sr <- FindNeighbors(sr, dims = 1:10) + sr <- FindClusters(sr, resolution = 0.5) #create seurat_clusters sctype needs + + sr +} + +# a small input Seurat object for testing +sr <- make_small_seurat(cmdb, ncell_each = 100) +# sctype module wants patient_id in the metadata +sr[["patient_id"]] <- "Patient1" saveRDS(sr, file = "sr_tiny.rds") -# a tiny reference Seurat object with seurat_annotation column for azimuth -n_cells_ref <- 1000 -sr_ref <- CreateSeuratObject( - counts = matrix( - data = rnbinom(n_genes * n_cells_ref, size = 1, mu = 100), - nrow = n_genes, - ncol = n_cells_ref, - dimnames = list( - genes, - paste0("Cell", 1:n_cells_ref) - ) - ) -) - -# azimuth wants seurat annotations in ref -sr_ref[["seurat_annotations"]] <- sample( - x = c("TypeA", "TypeB", "TypeC"), - size = n_cells_ref, - replace = TRUE -) - -sr_ref <- NormalizeData(sr_ref) -sr_ref <- FindVariableFeatures(sr_ref) -sr_ref <- ScaleData(sr_ref, features = VariableFeatures(sr_ref)) - -# azimuth also wants PCA in ref -sr_ref <- RunPCA(sr_ref, features = VariableFeatures(sr_ref)) +# a small ref Seurat object for testing +sr_ref <- make_small_seurat(cmdb, ncell_each = 100) +# azimuth needs seurat_annotations, make one from original cell names +sr_ref[["seurat_annotations"]] <- gsub("[0-9]+", "", rownames(sr_ref@meta.data)) saveRDS(sr_ref, file = "sr_ref_tiny.rds") + diff --git a/tests/modules/local/sctype/aggregate/main.nf.test b/tests/modules/local/sctype/aggregate/main.nf.test new file mode 100644 index 0000000..081b469 --- /dev/null +++ b/tests/modules/local/sctype/aggregate/main.nf.test @@ -0,0 +1,67 @@ +nextflow_process{ + name 'sctype_test' + script 'modules/local/sctype/aggregate/main.nf' + process 'SCYTPE_AGGREGATE_ANNOTATION' + + setup { + run("CREATE_DATA"){ + script "tests/data/create_data.nf" + params { + max_cpus = 2 + max_memory = '4 GB' + } + process { + """ + input[0] = file("${projectDir}/assets/cell_markers_database.csv") + """ + } + } + run("SCYTPE_MAJOR_ANNOTATION"){ + script "modules/local/sctype/major/main.nf" + process { + """ + input[0] = file("${projectDir}/modules/local/sctype/major/notebook_sctype_major.qmd") + input[1] = CREATE_DATA.out.sr_tiny + input[2] = file("${projectDir}/assets/cell_markers_database.csv") + input[3] = file(params.page_config) + """ + } + } + run("SCYTPE_STATE_ANNOTATION"){ + script "modules/local/sctype/state/main.nf" + process { + """ + input[0] = file("${projectDir}/modules/local/sctype/state/notebook_sctype_states.qmd") + input[1] = SCYTPE_MAJOR_ANNOTATION.out.seurat_rds + input[2] = file("${projectDir}/assets/cell_markers_database.csv") + input[3] = SCYTPE_MAJOR_ANNOTATION.out.major_list.splitText().map{ iter -> iter.split(":") } + .filter{ !(it[0] =~ "Unknown|Epithelial|Fibroblast|NK_Cells") } + .map{ it[0].trim() } + input[4] = file(params.page_config) + """ + } + } + } + + test("Run Sctype Aggregate") { + + tag 'sctype' + tag 'sctype-aggregate' + config 'tests/nextflow.config' + + when { + process { + """ + input[0] = file("${projectDir}/modules/local/sctype/aggregate/notebook_sctype_aggregate.qmd") + input[1] = SCYTPE_MAJOR_ANNOTATION.out.seurat_rds + input[2] = SCYTPE_STATE_ANNOTATION.out.annotation.collect() + input[3] = file(params.page_config) + """ + } + } + + then { + assert process.success + } + } +} \ No newline at end of file diff --git a/tests/modules/local/sctype/major/main.nf.test b/tests/modules/local/sctype/major/main.nf.test index ad41906..73ecd73 100644 --- a/tests/modules/local/sctype/major/main.nf.test +++ b/tests/modules/local/sctype/major/main.nf.test @@ -21,6 +21,7 @@ nextflow_process{ test("Run Sctype") { tag 'sctype' + tag 'sctype-major' config 'tests/nextflow.config' when { diff --git a/tests/modules/local/sctype/state/main.nf.test b/tests/modules/local/sctype/state/main.nf.test index b783497..96dfaa1 100644 --- a/tests/modules/local/sctype/state/main.nf.test +++ b/tests/modules/local/sctype/state/main.nf.test @@ -32,7 +32,7 @@ nextflow_process{ test("Run Sctype States") { tag 'sctype' - tag 'state' + tag 'sctype-state' config 'tests/nextflow.config' when { diff --git a/tests/nextflow.config b/tests/nextflow.config index 28db466..6b8e9ca 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -17,4 +17,7 @@ process { withName: 'CREATE_DATA' { container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' } + withName: 'SCYTPE_AGGREGATE_ANNOTATION' { + container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' + } } \ No newline at end of file From 3ed7903a6b91fb58512a73282501a1287ba85298 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Thu, 5 Mar 2026 13:43:30 -0500 Subject: [PATCH 29/33] add helpers tests --- tests/data/create_data.nf | 1 + tests/data/templates/make-test-seurat.r | 7 +++ .../local/helpers/convert/main.nf.test | 42 ++++++++++++++++++ .../modules/local/helpers/subset/main.nf.test | 43 +++++++++++++++++++ tests/nextflow.config | 6 +++ 5 files changed, 99 insertions(+) create mode 100644 tests/modules/local/helpers/convert/main.nf.test create mode 100644 tests/modules/local/helpers/subset/main.nf.test diff --git a/tests/data/create_data.nf b/tests/data/create_data.nf index 19f544b..3a09969 100644 --- a/tests/data/create_data.nf +++ b/tests/data/create_data.nf @@ -9,6 +9,7 @@ process CREATE_DATA { output: path('sr_tiny.rds') , emit: sr_tiny path('sr_ref_tiny.rds') , emit: sr_ref_tiny + path('cell_status.csv') , emit: cell_status script: template 'make-test-seurat.r' diff --git a/tests/data/templates/make-test-seurat.r b/tests/data/templates/make-test-seurat.r index d7fd523..bee9182 100644 --- a/tests/data/templates/make-test-seurat.r +++ b/tests/data/templates/make-test-seurat.r @@ -93,6 +93,13 @@ sr <- make_small_seurat(cmdb, ncell_each = 100) sr[["patient_id"]] <- "Patient1" saveRDS(sr, file = "sr_tiny.rds") +# simulate cell malignancy status metadata for testing +cell_status <- data.frame( + barcode = colnames(sr), + cell_status = sample(c("TME", "Malignant"), size = ncol(sr), replace = TRUE) +) +write.csv(cell_status, file = "cell_status.csv", row.names = FALSE) + # a small ref Seurat object for testing sr_ref <- make_small_seurat(cmdb, ncell_each = 100) diff --git a/tests/modules/local/helpers/convert/main.nf.test b/tests/modules/local/helpers/convert/main.nf.test new file mode 100644 index 0000000..be98cf9 --- /dev/null +++ b/tests/modules/local/helpers/convert/main.nf.test @@ -0,0 +1,42 @@ +nextflow_process{ + name 'helpers_convert_test' + script 'modules/local/helpers/convert/main.nf' + process 'HELPER_SCEASY_CONVERTER' + + + + + setup { + run("CREATE_DATA"){ + script "tests/data/create_data.nf" + params { + max_cpus = 2 + max_memory = '4 GB' + } + process { + """ + input[0] = file("${projectDir}/assets/cell_markers_database.csv") + """ + } + } + } + + test("Run Converter") { + tag 'helpers' + tag 'sceasy' + tag 'convert' + config 'tests/nextflow.config' + + when { + process { + """ + input[0] = CREATE_DATA.out.sr_tiny + """ + } + } + + then { + assert process.success + } + } +} \ No newline at end of file diff --git a/tests/modules/local/helpers/subset/main.nf.test b/tests/modules/local/helpers/subset/main.nf.test new file mode 100644 index 0000000..09991a4 --- /dev/null +++ b/tests/modules/local/helpers/subset/main.nf.test @@ -0,0 +1,43 @@ +nextflow_process{ + name 'helpers_subset_test' + script 'modules/local/helpers/subset/main.nf' + process 'HELPER_SEURAT_SUBSET' + + + + + setup { + run("CREATE_DATA"){ + script "tests/data/create_data.nf" + params { + max_cpus = 2 + max_memory = '4 GB' + } + process { + """ + input[0] = file("${projectDir}/assets/cell_markers_database.csv") + """ + } + } + } + + test("Run Subset") { + + tag 'helpers' + tag 'subset' + config 'tests/nextflow.config' + + when { + process { + """ + input[0] = CREATE_DATA.out.sr_tiny + input[1] = CREATE_DATA.out.cell_status + """ + } + } + + then { + assert process.success + } + } +} \ No newline at end of file diff --git a/tests/nextflow.config b/tests/nextflow.config index 6b8e9ca..efb34a9 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -20,4 +20,10 @@ process { withName: 'SCYTPE_AGGREGATE_ANNOTATION' { container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' } + withName: 'HELPER_SEURAT_SUBSET' { + container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' + } + withName: 'HELPER_SCEASY_CONVERTER' { + container = 'ghcr.io/break-through-cancer/scratch-annotation:945c406' + } } \ No newline at end of file From 3bebfabfedd4128f5b220ab0681e174bc3533450 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Thu, 5 Mar 2026 15:13:02 -0500 Subject: [PATCH 30/33] make runnable --- .../local/helpers/convert/resources/usr/bin/sceasy_converter.R | 0 modules/local/helpers/subset/resources/usr/bin/seurat_subset.R | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R mode change 100644 => 100755 modules/local/helpers/subset/resources/usr/bin/seurat_subset.R diff --git a/modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R b/modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R old mode 100644 new mode 100755 diff --git a/modules/local/helpers/subset/resources/usr/bin/seurat_subset.R b/modules/local/helpers/subset/resources/usr/bin/seurat_subset.R old mode 100644 new mode 100755 From 6065223c4275949a07b041c96ba78bcc85fa1f8d Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Fri, 6 Mar 2026 07:49:13 -0500 Subject: [PATCH 31/33] bugfix: ignore case when matching --- bin/sceasy_converter.R | 0 bin/seurat_subset.R | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/sceasy_converter.R mode change 100644 => 100755 bin/seurat_subset.R diff --git a/bin/sceasy_converter.R b/bin/sceasy_converter.R old mode 100644 new mode 100755 diff --git a/bin/seurat_subset.R b/bin/seurat_subset.R old mode 100644 new mode 100755 From bdff26bada5ae85104171959c11044e77318d3bb Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Fri, 6 Mar 2026 08:20:13 -0500 Subject: [PATCH 32/33] bugfix: ignore case when matching --- .../helpers/convert/resources/usr/bin/sceasy_converter.R | 3 ++- .../local/helpers/subset/resources/usr/bin/seurat_subset.R | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R b/modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R index 4c65a22..b86ac2e 100755 --- a/modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R +++ b/modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R @@ -47,7 +47,8 @@ if(opt$type == "Seurat") { sce_object <- as.SingleCellExperiment( seurat_object) - output <- file.path(opt$outdir, gsub(".RDS", ".h5ad", basename(opt$file))) + output <- file.path(opt$outdir, gsub(".RDS", ".h5ad", basename(opt$file), + ignore.case = TRUE)) convertFormat(sce_object, from = "sce", to = "anndata", outFile = output) diff --git a/modules/local/helpers/subset/resources/usr/bin/seurat_subset.R b/modules/local/helpers/subset/resources/usr/bin/seurat_subset.R index 7708aa5..2e4655e 100755 --- a/modules/local/helpers/subset/resources/usr/bin/seurat_subset.R +++ b/modules/local/helpers/subset/resources/usr/bin/seurat_subset.R @@ -48,5 +48,7 @@ seurat_object <- subset( # -saveRDS( - seurat_object, file = file.path(opt$outdir, gsub(".RDS", "_filtered.RDS", basename(opt$file)))) +saveRDS(seurat_object, + file = file.path(opt$outdir, gsub(".RDS", "_filtered.RDS", + basename(opt$file), + ignore.case = TRUE))) \ No newline at end of file From a76cec790028d58e6c91c65f16b4d34b4d0775d0 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Fri, 6 Mar 2026 09:56:03 -0500 Subject: [PATCH 33/33] Update tests/data/templates/make-test-seurat.r Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/data/templates/make-test-seurat.r | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/data/templates/make-test-seurat.r b/tests/data/templates/make-test-seurat.r index bee9182..a856fcf 100644 --- a/tests/data/templates/make-test-seurat.r +++ b/tests/data/templates/make-test-seurat.r @@ -64,15 +64,17 @@ make_small_seurat <- function(cmdb, ncell_each = 100) { if (ncol(counts) < 2500) { n_genes_to_add <- 2500 - ncol(counts) random_genes <- paste0("DUMMY", seq_len(n_genes_to_add)) - random_counts <- matrix( + random_counts_matrix <- matrix( data = rnbinom(nrow(counts) * n_genes_to_add, size = 1, mu = 100), nrow = nrow(counts), ncol = n_genes_to_add, dimnames = list(rownames(counts), random_genes) - ) |> as.data.frame() - #set 80% of the random counts to 0 to make them more realistic - zeroes <- sample(length(random_counts), size = length(random_counts) * 0.8) - random_counts[zeroes] <- 0 + ) + # set 80% of the random counts to 0 to make them more realistic + n_elements <- length(random_counts_matrix) + zeroes <- sample(n_elements, size = floor(n_elements * 0.8)) + random_counts_matrix[zeroes] <- 0 + random_counts <- as.data.frame(random_counts_matrix) counts <- cbind(counts, random_counts) }