Init test pipeline

.github/workflows/build-push-container.yml

@@ -0,0 +1,58 @@
+#
+name: build-push-container
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  push:
+    paths:
+      - 'docker/scratch-annotation.Dockerfile'
+      - '.github/workflows/build-push-container.yml'
+  workflow_dispatch:
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+      # 
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GHCR_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,format=short,prefix= # Generates a tag like '860c190'
+            main
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          file: docker/scratch-annotation.Dockerfile
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test.yml

@@ -0,0 +1,34 @@
+#inspired by https://docs.cirro.bio/pipelines/development/#automated-testing
+name: test
+
+on:
+  pull_request:
+      branches:
+      - 'main'
+  workflow_dispatch:
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          lfs: true
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: install nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: install nf-test
+        run: |
+          wget -qO- https://get.nf-test.com | bash
+          sudo mv nf-test /usr/local/bin
+
+      - name: run tests
+        run: nf-test test

.gitignore

@@ -10,3 +10,6 @@ figures/
 *.zip
 *.tar.gz
 *.sif
+_nf-test* # temporary files for nf-test - quarto does not produce _freeze and _report if testdir starts with dot
+
+/.quarto/

docker/scratch-annotation.Dockerfile

@@ -6,11 +6,11 @@ WORKDIR /opt
 
 # Timezone settings
 ENV TZ=US/Central
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
-    echo $TZ > /etc/timezone
-# pass your PAT at build time so remotes::install_github can auth
-ARG GITHUB_PAT
-ENV GITHUB_PAT=${GITHUB_PAT}
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
+  && echo $TZ > /etc/timezone
+
+
+ENV R_REPOS="https://packagemanager.posit.co/cran/latest"
 
 # Install system dependencies
 RUN apt-get update && apt-get install -y \
@@ -31,136 +31,111 @@ RUN apt-get update && apt-get install -y \
     libpng-dev \
     libtiff5-dev \
     zlib1g-dev \
-    libxt-dev
+    libxt-dev \
+
+    jags \
+    python3 \
+    python3-pip \
+    python3-venv \
+    libhdf5-dev \
+    libgsl-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 
 # Updating quarto to Quarto v1.4.553
 RUN wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.4.553/quarto-1.4.553-linux-amd64.deb -O quarto-1.4.553-linux-amd64.deb
 RUN dpkg -i quarto-1.4.553-linux-amd64.deb
 
-# # Install remotes package
-# RUN R -e "install.packages('remotes')"
-
-
-RUN apt-get update && apt-get install -y python3 python3-pip python3-venv
 
 # Install core R packages
-RUN Rscript -e "install.packages(c('R.utils','rmarkdown','devtools','tidyverse','readr', 'dplyr', 'ggplot2', 'cowplot', 'remotes', 'BiocManager','reticulate', 'HGNChelper', 'optparse'), repos='http://cran.us.r-project.org')"
-
-
-RUN Rscript -e "BiocManager::install(c('S4Vectors','DelayedMatrixStats','BiocGenerics','Biobase', 'SummarizedExperiment', 'AnnotationDbi', 'org.Hs.eg.db'), ask=FALSE, update=TRUE)"
+RUN Rscript -e "install.packages(c( \
+  'R.utils', \
+  'rmarkdown', \
+  'devtools', \
+  'tidyverse', \
+  'readr', \
+  'dplyr', \
+  'ggplot2', \
+  'cowplot', \
+  'remotes', \
+  'BiocManager',\
+  'reticulate', \
+  'HGNChelper', \
+  'hdf5r', \
+  'optparse' \
+  ), repos='${R_REPOS}')"
+
+# Bioconductor packages
 RUN Rscript -e "BiocManager::install(c( \
-    'HDF5Array','rhdf5','rhdf5lib', \
+  'S4Vectors', \
+  'DelayedMatrixStats', \
+  'BiocGenerics',\
+  'Biobase', \
+  'SummarizedExperiment', \
+  'AnnotationDbi', \
+  'org.Hs.eg.db' \
+  ), ask=FALSE, update=TRUE)" \
+  && Rscript -e "BiocManager::install(c( \
+    'HDF5Array', \
+    'rhdf5', \
+    'rhdf5lib', \
     'SingleCellExperiment', \
-    'GOSemSim','MatrixGenerics','treeio','DOSE','ggtree','enrichplot', \
-    'clusterProfiler','DirichletMultinomial','rtracklayer','GenomicFeatures', \
-    'BSgenome','ensembldb','TFBSTools', \
-    'BSgenome.Hsapiens.UCSC.hg38','EnsDb.Hsapiens.v86' \
-  ), \
-  ask=FALSE, update=FALSE )"
-
-# Setting repository URL
-ARG R_REPO="http://cran.us.r-project.org"
+    'GOSemSim', \
+    'MatrixGenerics', \
+    'treeio', \
+    'DOSE',\
+    'ggtree',\
+    'enrichplot', \
+    'clusterProfiler',\
+    'DirichletMultinomial',\
+    'rtracklayer',\
+    'GenomicFeatures', \
+    'BSgenome',\
+    'ensembldb',\
+    'TFBSTools', \
+    'BSgenome.Hsapiens.UCSC.hg38', \
+    'EnsDb.Hsapiens.v86' \
+  ), ask=FALSE, update=FALSE )"
 
 
-# # Install BiocManager
-# RUN Rscript -e "BiocManager::install(${R_BIOC_DEPS})"
 RUN Rscript -e 'remotes::install_github("ctlab/fgsea")'
 
-
-
-# Install Seurat Wrappers
-RUN wget https://github.com/satijalab/seurat/archive/refs/heads/seurat5.zip -O /opt/seurat-v5.zip
-RUN wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /opt/seurat-data.zip
-RUN wget https://github.com/satijalab/seurat-wrappers/archive/refs/heads/seurat5.zip -O /opt/seurat-wrappers.zip
-
-RUN Rscript -e "devtools::install_local('/opt/seurat-v5.zip')"
-RUN Rscript -e "devtools::install_local('/opt/seurat-data.zip')"
-RUN Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')"
-
-
+# Seurat and wrappers
+RUN wget https://github.com/satijalab/seurat/archive/refs/tags/v5.3.1.zip -O /opt/seurat-v5.zip \
+  && wget https://github.com/satijalab/seurat-data/archive/refs/heads/seurat5.zip -O /opt/seurat-data.zip \
+  && wget https://github.com/satijalab/seurat-wrappers/archive/refs/heads/seurat5.zip -O /opt/seurat-wrappers.zip \
+  && Rscript -e "devtools::install_local('/opt/seurat-v5.zip')" \
+  && Rscript -e "devtools::install_local('/opt/seurat-data.zip')" \
+  && Rscript -e "devtools::install_local('/opt/seurat-wrappers.zip')" \
+  && rm /opt/seurat-v5.zip /opt/seurat-data.zip /opt/seurat-wrappers.zip \
+  && Rscript -e "devtools::install_github('satijalab/azimuth', ref = 'master', dependencies=TRUE, upgrade='never')"
 
 # Install SCP package from GitHub
-# RUN Rscript -e "remotes::install_github('bnprks/BPCells/r')"
-RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, force = TRUE)"
-# RUN Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, force = TRUE)"
-RUN Rscript -e "remotes::install_github('zhanghao-njmu/SCP', upgrade = 'always', dependencies = TRUE, force = TRUE)"
-RUN Rscript -e "remotes::install_github('cellgeni/sceasy', upgrade = 'always', dependencies = TRUE, force = TRUE)"
-
-
+RUN Rscript -e "devtools::install_github('PaulingLiu/ROGUE', dependencies = TRUE, upgrade = 'never', force = TRUE)" \
+  && Rscript -e "devtools::install_github('zhanghao-njmu/SCP', dependencies = TRUE, upgrade = 'never', force = TRUE)" \
+  && Rscript -e "devtools::install_github('cellgeni/sceasy', dependencies = TRUE, upgrade = 'never', force = TRUE)" \
+  # uninstall reactome.db that SCP pulls (1.7Gb)
+  && Rscript -e "remove.packages('reactome.db')"
 
+# Install annotables
+RUN Rscript -e "devtools::install_github('stephenturner/annotables')" \
+  && Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')"
 
-# Create and activate virtual environment
-RUN python3 -m venv /opt/venv
-ENV PATH="/opt/venv/bin:$PATH"
-
-# # Upgrade pip and install Python packages in venv
-# RUN pip install --upgrade pip && \
-#     pip install numpy pandas scikit-learn matplotlib seaborn jupyter jupyter-cache papermill
-
-# Create and activate a virtual environment before installing Python packages
+# Create and activate a virtual environment before installing Python packages 
+# Note the no-gpu version of torch to avoid ~5Gb dependencies and keep image size smaller
 RUN python3 -m venv /opt/venv \
-    && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn matplotlib seaborn jupyter jupyter-cache papermill anndata scanpy scipy session_info scSpectra metatime celltypist \
+    && /opt/venv/bin/pip install --no-cache-dir --upgrade pip \
+    && /opt/venv/bin/pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu \
+    && /opt/venv/bin/pip install --no-cache-dir numpy pandas scikit-learn \
+     matplotlib seaborn jupyter jupyter-cache papermill anndata scanpy scipy \
+     session_info scSpectra metatime celltypist \
     && ln -s /opt/venv/bin/python /usr/local/bin/python \
     && ln -s /opt/venv/bin/pip /usr/local/bin/pip
-# Create and activate virtual environment
-RUN python -m venv /opt/venv
-ENV PATH="/opt/venv/bin:$PATH"
 
-# Setting celltypist variable
+ENV PATH="/opt/venv/bin:$PATH"
 ENV CELLTYPIST_FOLDER=/opt/celltypist
 
-# Installing celltypist models
-COPY setup.py /opt/
-RUN python3 /opt/setup.py
-
-# Additional packages
-RUN apt-get install -y libhdf5-dev
-RUN Rscript -e "install.packages('hdf5r')"
-
-
-# Java + Fortran 
-RUN apt-get update && apt-get install -y default-jre libgfortran5
-
-# JAGS
-RUN apt-get install -y jags
-
-
-# Install annotables
-RUN Rscript -e "devtools::install_github('stephenturner/annotables')"
-# RUN Rscript -e "install.packages('rJava', repos = 'http://cran.us.r-project.org')"
-RUN Rscript -e "devtools::install_github('miccec/yaGST', dependencies = TRUE, upgrade = 'never')"
-
-# RUN Rscript -e "devtools::install_github('AntonioDeFalco/SCEVAN', dependencies = TRUE, upgrade = 'never')"
-# install SCP
-RUN Rscript -e "remotes::install_github( \
-  'zhanghao-njmu/SCP', \
-  dependencies=TRUE, \
-  upgrade='always', \
-  auth_token = Sys.getenv('GITHUB_PAT'))"
-
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-       libgsl-dev \
-    && rm -rf /var/lib/apt/lists/*  
-  # Install DirichletMultinomial + TFBSTools (and all of their R & Bioc deps)
-
-RUN Rscript -e "install.packages(c('DirichletMultinomial','TFBSTools'), dependencies = TRUE, repos = BiocManager::repositories())"
-
-# install Azimuth
-RUN Rscript -e "remotes::install_github( \
-  'satijalab/azimuth', ref = 'master', \
-  dependencies=TRUE, \
-  upgrade='always', \
-  auth_token = Sys.getenv('GITHUB_PAT'))"
-
-
-# Cleaning apt-get cache
-RUN apt-get clean
-RUN rm -rf /var/lib/apt/lists/*
-
-
 # Command to run on container start
 CMD ["bash"]
 

modules/local/helpers/convert/resources/usr/bin/sceasy_converter.R

@@ -47,7 +47,8 @@ if(opt$type == "Seurat") {
   sce_object <- as.SingleCellExperiment(
     seurat_object)
 
-  output <- file.path(opt$outdir, gsub(".RDS", ".h5ad", basename(opt$file)))
+  output <- file.path(opt$outdir, gsub(".RDS", ".h5ad", basename(opt$file),
+                                       ignore.case = TRUE))
 
   convertFormat(sce_object, from = "sce", to = "anndata",
                 outFile = output)

modules/local/helpers/subset/resources/usr/bin/seurat_subset.R

@@ -48,5 +48,7 @@ seurat_object <- subset(
 
 #
 
-saveRDS(
-  seurat_object, file = file.path(opt$outdir, gsub(".RDS", "_filtered.RDS", basename(opt$file))))
+saveRDS(seurat_object,
+        file = file.path(opt$outdir, gsub(".RDS", "_filtered.RDS",
+                                          basename(opt$file),
+                                          ignore.case = TRUE)))

nf-test.config

@@ -0,0 +1,7 @@
+config {
+    testsDir "tests"
+    workDir "_nf-test"
+    configFile "tests/nextflow.config"
+    profile "docker"
+}
+

tests/data/create_data.nf

@@ -0,0 +1,16 @@
+process CREATE_DATA {
+
+    tag    'Creating test data for annotation'
+    label  'process_medium'
+
+    input: 
+        path(cell_markers_database)
+
+    output:
+        path('sr_tiny.rds') ,      emit: sr_tiny
+        path('sr_ref_tiny.rds') ,  emit: sr_ref_tiny
+        path('cell_status.csv') ,  emit: cell_status
+
+    script:
+    template 'make-test-seurat.r'
+}