diff --git a/src/methods_cell_type_annotation/rctd/script.R b/src/methods_cell_type_annotation/rctd/script.R index 6e4524e39..93a9cc345 100644 --- a/src/methods_cell_type_annotation/rctd/script.R +++ b/src/methods_cell_type_annotation/rctd/script.R @@ -43,10 +43,27 @@ filtered_ref <- ref[,colData(ref)$cell_type %in% valid_celltypes] ref_counts <- assay(filtered_ref, "counts") # factor to drop filtered cell types -colData(filtered_ref)$cell_type <- factor(colData(filtered_ref)$cell_type) +colData(filtered_ref)$cell_type <- factor(colData(filtered_ref)$cell_type) cell_types <- colData(filtered_ref)$cell_type names(cell_types) <- colnames(ref_counts) +# --- Diagnostics: confirm what RCTD actually reads. The reference itself has +# ~475/477 panel genes with clear cross-cell-type fold-change, so a "fewer +# than 10 DE genes" failure means the data reaching RCTD is degenerate: +# cell types collapsed to ~1, non-integer/normalized counts, or few genes +# shared with the spatial puck. This block surfaces which. --- +cat("=== RCTD input diagnostics ===\n") +cat(sprintf("spatial puck: %d genes x %d cells\n", nrow(counts), ncol(counts))) +cat(sprintf("reference (>=25 cells/type): %d genes x %d cells, %d cell types\n", + nrow(ref_counts), ncol(ref_counts), length(unique(as.character(cell_types))))) +print(utils::head(sort(table(as.character(cell_types)), decreasing = TRUE), 8)) +cat(sprintf("genes shared reference<->spatial: %d\n", + length(intersect(rownames(ref_counts), rownames(counts))))) +.rcv <- if (methods::is(ref_counts, "sparseMatrix")) ref_counts@x else as.numeric(ref_counts) +if (length(.rcv)) cat(sprintf("reference counts: min=%.4g max=%.4g mean=%.4g all-integer=%s\n", + min(.rcv), max(.rcv), mean(.rcv), isTRUE(all.equal(.rcv, round(.rcv))))) +cat("==============================\n") + # spacexr::check_cell_types() rejects cell-type names containing '/' # (e.g. "Ciliated/secretory cells", "T/NK lineage"). Sanitize the factor levels # before building the Reference, and keep a map (safe name -> original name) so diff --git a/src/methods_expression_correction/split/config.vsh.yaml b/src/methods_expression_correction/split/config.vsh.yaml index d6fe7e14c..9d56d1fba 100644 --- a/src/methods_expression_correction/split/config.vsh.yaml +++ b/src/methods_expression_correction/split/config.vsh.yaml @@ -17,6 +17,34 @@ arguments: type: boolean default: false description: Whether to keep cells with 0 counts (may cause errors if set to TRUE) + # RCTD's default DE-gene / UMI thresholds are tuned for whole-transcriptome + # references; on small iST panels (a few hundred genes, many similar fine-grained + # cell types) fold-changes are compressed and RCTD finds "fewer than 10 DE genes". + # These relaxed defaults mirror the rctd component so create.RCTD below succeeds. + - name: --gene_cutoff + type: double + default: 0.0 + description: "Minimum normalized mean expression for a gene to be a platform-effect DE gene (RCTD default 0.000125)." + - name: --fc_cutoff + type: double + default: 0.1 + description: "Minimum log-fold-change for a gene to be a platform-effect DE gene (RCTD default 0.5)." + - name: --gene_cutoff_reg + type: double + default: 0.0 + description: "Minimum normalized mean expression for a gene to be a regression DE gene (RCTD default 0.0002)." + - name: --fc_cutoff_reg + type: double + default: 0.1 + description: "Minimum log-fold-change for a gene to be a regression DE gene (RCTD default 0.75)." + - name: --umi_min + type: integer + default: 20 + description: "Minimum total UMI per spatial cell to be included (RCTD default 100; iST cells are low-count)." + - name: --umi_min_sigma + type: integer + default: 20 + description: "Minimum UMI for cells used to fit the platform-effect variance (RCTD default 300)." resources: - type: r_script diff --git a/src/methods_expression_correction/split/script.R b/src/methods_expression_correction/split/script.R index aa0e06957..7bcc06f00 100644 --- a/src/methods_expression_correction/split/script.R +++ b/src/methods_expression_correction/split/script.R @@ -12,6 +12,12 @@ par <- list( "input_scrnaseq_reference"= "task_ist_preprocessing/resources_test/task_ist_preprocessing/mouse_brain_combined/scrnaseq_reference.h5ad", "output" = "task_ist_preprocessing/tmp/split_corrected.h5ad", "keep_all_cells" = FALSE, + "gene_cutoff" = 0.0, + "fc_cutoff" = 0.1, + "gene_cutoff_reg" = 0.0, + "fc_cutoff_reg" = 0.1, + "umi_min" = 20, + "umi_min_sigma" = 20 ) meta <- list( @@ -67,7 +73,14 @@ cat(sprintf("Number of cores: %s\n", cores)) # Run the algorithm cat("Running RCTD\n") -myRCTD <- create.RCTD(puck, reference, max_cores = cores) +# Relaxed DE-gene / UMI thresholds (see config): RCTD's defaults find "fewer than +# 10 DE genes" on small iST panels with many fine-grained cell types. +myRCTD <- create.RCTD( + puck, reference, max_cores = cores, + gene_cutoff = par$gene_cutoff, fc_cutoff = par$fc_cutoff, + gene_cutoff_reg = par$gene_cutoff_reg, fc_cutoff_reg = par$fc_cutoff_reg, + UMI_min = par$umi_min, UMI_min_sigma = par$umi_min_sigma +) myRCTD <- run.RCTD(myRCTD, doublet_mode = "doublet") # Get the "spot_class" annotation from RCTD diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index 8dd320557..fdd8872c1 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -45,69 +45,62 @@ engines: # (cudf, cuspatial, torch). Develop/test on a CUDA host; `viash test` # will not run on a machine without an NVIDIA GPU. - type: docker - # NGC 26.06 ships numpy 2.1 and a torch (2.13) compiled against numpy 2.x, so - # the torch<->numpy bridge (used by the `segger segment` subprocess) stays - # alive alongside the numpy-2.x task stack (spatialdata>=0.7.3 hard-requires - # numpy>=2 via multiscale-spatial-image -> xarray-dataclass). The older 25.06 - # image shipped numpy-1.x torch, which cannot coexist with spatialdata>=0.7.3. - image: nvcr.io/nvidia/pytorch:26.06-py3 + # segger is fundamentally a RAPIDS application (cudf + cuspatial + cugraph + + # cuml). Bolting that full stack onto the NGC PyTorch base made RAPIDS's UCX + # (cugraph -> raft_dask) collide with the image's HPC-X UCX and abort with heap + # corruption (SIGABRT, exit 134) that no pip pin could fix. So we base on the + # official RAPIDS image (ships cudf/cugraph/cuml/cuspatial 25.04 + a consistent, + # working UCX/dask stack) and layer torch (GNN) + the spatialdata/anndata I/O + # stack + segger on top. numpy 2.0.2 / pandas 2.2.3 come from the base. + # + # NOTE: this image runs as the NON-root 'rapids' user, so `type: apt` will NOT + # work in a viash build (needs root). Everything below installs via conda/pip + # only (both write to the rapids-owned conda env) — do NOT add apt steps. + image: rapidsai/base:25.04-cuda12.8-py3.12 setup: - # libgl1 + libglib2.0-0: segger's writer imports segger.io -> cv2 (opencv), - # which needs libGL.so.1 / libgthread at import; without them the run crashes - # at write time with "libGL.so.1: cannot open shared object file". - - type: apt - packages: [procps, git, libxcb1, libgl1, libglib2.0-0] + # git (for the segger + openproblems github installs) via conda: apt is + # unavailable as non-root and the RAPIDS base ships no git. + - type: docker + run: conda install -y -c conda-forge git - type: python github: - openproblems-bio/core#subdirectory=packages/python/openproblems - - type: python - packages: - - "spatialdata>=0.7.3" - - "anndata>=0.12.0" - - "zarr>=3.0.0" - - geopandas - - shapely - - "rasterio>=1.3" - - scikit-image - # cuspatial is imported by segger.geometry.conversion and is NOT shipped in - # the 26.06 image (unlike 25.06). Install from the NVIDIA index (its - # libcudf-cu12/libcuspatial-cu12 deps are not on PyPI); it pulls a matching - # cudf-cu12. Unpinned -> resolves to the newest cuda-12 build (25.4, the last - # cuda-12 cuspatial-cu12 on the index). - - type: docker - run: | - pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com cuspatial-cu12 - # segger imports torch_geometric (at import time via _patches), lightning - # (segger segment -> Trainer) and torch_scatter (models: scatter_max) but - # declares NONE of them in its pyproject, so the github install below does - # not pull them. torch_geometric/lightning are pure Python. torch_scatter has - # no prebuilt wheel for the NGC custom torch build, so compile against it - # (--no-build-isolation). The build host has no GPU, so force a CUDA build for - # the target archs (A100=8.0, A10/A40=8.6, L4/L40=8.9, H100=9.0). + # torch for the GNN. Standard PyPI cu124 wheel (numpy-2 compatible) coexists + # with the RAPIDS cuda-12 libs. Unlike the NGC custom torch, torch_scatter has + # a PREBUILT wheel for this torch (the pyg index) — no CUDA compile needed. - type: docker run: + - pip install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124 - pip install --no-cache-dir torch_geometric lightning - - FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" pip install --no-cache-dir --no-build-isolation torch_scatter - # Pin to a specific commit: segger's CLI flags AND output schema change on - # trunk (0.1.0 renamed --prediction-expansion-ratio -> --prediction-graph- - # buffer-ratio and dropped the `keep` column). script.py targets THIS commit's - # contract, so bump this deliberately and re-check `segger segment --help` - # and data/writer.py before doing so. + - pip install --no-cache-dir torch_scatter -f https://data.pyg.org/whl/torch-2.6.0+cu124.html + # I/O stack. spatialdata pinned <0.8 (resolves to 0.7.1): 0.7.2+ require + # dask>=2026.x, but RAPIDS 25.04 hard-pins dask==2025.2.0. 0.7.1 accepts dask + # 2025.2.0; pin dask/distributed so spatialdata can't drag them up (which would + # break RAPIDS's dask-cudf/cugraph). We validated sd.transform works on 2025.2.0. + - type: docker + run: + - pip install --no-cache-dir "spatialdata>=0.7,<0.8" "zarr>=3.0.0" geopandas shapely "rasterio>=1.3" scikit-image "dask==2025.2.0" "distributed==2025.2.0" + # Pin segger to a commit: its CLI flags AND output schema change on trunk + # (0.1.0 renamed --prediction-expansion-ratio -> --prediction-graph-buffer- + # ratio and dropped the `keep` column). script.py targets THIS commit; bump + # deliberately and re-check `segger segment --help` and data/writer.py first. - type: python github: dpeerlab/segger@0233cf62eb177b6e44e49318037705550765a010 - # cudf 25.4 (cuda-12; there is no cuspatial-cu13, so we are stuck on this - # RAPIDS) hard-requires pandas<2.2.4. But segger pulls the modern single-cell - # stack — anndata 0.13 and scanpy 1.12 — which BOTH require pandas>=2.3 - # (anndata 0.13's zarr reader calls pd.StringDtype(na_value=...), an API added - # in pandas 2.3), so under the pinned pandas 2.2.3 reading ANY AnnData/zarr - # table crashes with "StringDtype.__init__() got an unexpected keyword - # argument 'na_value'". Pin pandas back into cudf's range AND hold anndata / - # scanpy at their last pandas-2.2-compatible majors (anndata 0.12.x needs - # pandas>=2.1; scanpy 1.11.x needs pandas>=1.5) as the LAST step so they win - # over the deps above. numpy stays 2.x. See NOTES.md for the full conflict. + # Final step, must be LAST (after segger's github install pulls its deps): + # - cudf 25.4 needs pandas<2.2.4; but anndata 0.13 / scanpy 1.12 (pulled by + # segger) need pandas>=2.3 (anndata 0.13's zarr reader uses + # pd.StringDtype(na_value=...), a pandas-2.3 API) -> reading any zarr table + # crashes under pandas 2.2.3. Hold anndata/scanpy at their last pandas-2.2 + # majors (anndata 0.12.x: pandas>=2.1; scanpy 1.11.x: pandas>=1.5), and + # re-assert spatialdata<0.8 + dask==2025.2.0 in case a dep nudged them. + # - swap opencv-python -> opencv-python-headless: segger pulls opencv-python, + # whose cv2 needs libGL.so.1 (a system lib we cannot apt-install as non-root); + # the headless build provides the same cv2 without it. See NOTES.md. - type: docker run: - - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" + - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" + - pip uninstall -y opencv-python || true + - pip install --no-cache-dir opencv-python-headless - type: native runners: