From 5b7cee1fbe7b71ca1713e1123804ae4b61c81c0c Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Thu, 4 Jun 2026 10:44:51 -0400 Subject: [PATCH 01/14] pin snakebids to 0.14.0 in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d7728ee..6234932 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ SHELL ["/bin/bash", "-c"] # ---- ONE SINGLE RUN ---- RUN set -e && \ conda install -n base -c conda-forge mamba -y && \ - mamba create -y -n snakebids-env -c conda-forge -c bioconda python=3.11 snakebids appdirs unzip && \ + mamba create -y -n snakebids-env -c conda-forge -c bioconda python=3.11 snakebids=0.14.0 appdirs unzip && \ source /opt/conda/etc/profile.d/conda.sh && \ conda activate snakebids-env && \ ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ From 17490de32122cce13b3d74d0bcf21bf8be3275de Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Thu, 4 Jun 2026 10:46:59 -0400 Subject: [PATCH 02/14] allow docker testing on the feature branch for now --- .github/workflows/build_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index e670084..4053e2a 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -2,7 +2,7 @@ name: Build and Push Docker Image on: push: - branches: [main] + branches: [main, 'djay/pin-python-dockerfile'] workflow_dispatch: jobs: build-and-push: From f58aa13cbf0e09d5e3ed42b38c49a22c834cd120 Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Fri, 5 Jun 2026 10:48:54 -0400 Subject: [PATCH 03/14] pin snakemake version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6234932..57e02df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ SHELL ["/bin/bash", "-c"] # ---- ONE SINGLE RUN ---- RUN set -e && \ conda install -n base -c conda-forge mamba -y && \ - mamba create -y -n snakebids-env -c conda-forge -c bioconda python=3.11 snakebids=0.14.0 appdirs unzip && \ + mamba create -y -n snakebids-env -c conda-forge -c bioconda python=3.11 snakebids=0.14.0 snakemake=9.6.2 appdirs unzip && \ source /opt/conda/etc/profile.d/conda.sh && \ conda activate snakebids-env && \ ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ From e0b44ff25d95576cad81593fd9ec50e7c9db6326 Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Fri, 5 Jun 2026 11:00:21 -0400 Subject: [PATCH 04/14] add detect_with_nnlm flag --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 57e02df..2fd4643 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ RUN set -e && \ source /opt/conda/etc/profile.d/conda.sh && \ conda activate snakebids-env && \ ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs --detect_with_nnlm && \ ./autoafids/run.py test_data/bids_T2w test_out participant --modality T2w --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ ./autoafids/run.py test_data/bids_ct test_out participant --modality ct --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --stereotaxy STN --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ From 2f3065c18e69b3e1ab16c4d04e5aff404818f391 Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 11:07:13 -0400 Subject: [PATCH 05/14] use pixi docker image --- Dockerfile | 57 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2fd4643..21b7a85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,33 @@ -FROM --platform=linux/amd64 condaforge/miniforge3:latest +# --- Stage 1: Build & Environment Prep --- +FROM ghcr.io/prefix-dev/pixi:latest AS build -WORKDIR /src/ +WORKDIR /src +COPY . /src -# Copy your code -COPY . /src/ +# Create the environment based on your lockfile and install dependencies +# This creates an isolated, self-contained environment in /src/.pixi/envs/default +RUN pixi install --locked -# Disable user site packages -ENV PYTHONNOUSERSITE=1 +# Run your autoafids pre-caching tasks inside the pixi environment +RUN pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs --detect_with_nnlm && \ + pixi run ./autoafids/run.py test_data/bids_T2w test_out participant --modality T2w --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_ct test_out participant --modality ct --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --stereotaxy STN --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --fidqc --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs + +# --- Stage 2: Tiny Production Image --- +# Use a lightweight glibc base image instead of a heavy Conda/Miniforge image +FROM ubuntu:24.04 AS production + +WORKDIR /src -# Use bash for the following RUNs -SHELL ["/bin/bash", "-c"] - -# ---- ONE SINGLE RUN ---- -RUN set -e && \ - conda install -n base -c conda-forge mamba -y && \ - mamba create -y -n snakebids-env -c conda-forge -c bioconda python=3.11 snakebids=0.14.0 snakemake=9.6.2 appdirs unzip && \ - source /opt/conda/etc/profile.d/conda.sh && \ - conda activate snakebids-env && \ - ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs --detect_with_nnlm && \ - ./autoafids/run.py test_data/bids_T2w test_out participant --modality T2w --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - ./autoafids/run.py test_data/bids_ct test_out participant --modality ct --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --stereotaxy STN --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --fidqc --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - conda clean --all -y && \ - rm -rf /opt/conda/pkgs /root/.caches - -# Set snakemake profile -ENV SNAKEMAKE_PROFILE=/src/autoafids/workflow/profiles/docker-conda - -# Set entrypoint -ENTRYPOINT ["/src/entrypoint.sh"] +# Copy your source code, pre-cached conda environments, and the built Pixi environment +COPY --from=build /src /src + +# Add the pixi environment binaries straight to the PATH so you don't have to "activate" it +ENV PATH="/src/.pixi/envs/default/bin:$PATH" +ENV PYTHONNOUSERSITE=1 +ENV SNAKEMAKE_PROFILE=/src/autoafids/autoafids/workflow/profiles/docker-conda +ENTRYPOINT ["/src/entrypoint.sh"] \ No newline at end of file From c6299fe4e34703ffc1195375a48aceb65c7af913 Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 11:16:32 -0400 Subject: [PATCH 06/14] temporarily remove logic creating conda envs and clean entrypoint.sh --- Dockerfile | 12 ++++++------ entrypoint.sh | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 21b7a85..a3efc4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,12 +9,12 @@ COPY . /src RUN pixi install --locked # Run your autoafids pre-caching tasks inside the pixi environment -RUN pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs --detect_with_nnlm && \ - pixi run ./autoafids/run.py test_data/bids_T2w test_out participant --modality T2w --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - pixi run ./autoafids/run.py test_data/bids_ct test_out participant --modality ct --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --stereotaxy STN --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ - pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --fidqc --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs +# RUN pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ +# pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs --detect_with_nnlm && \ +# pixi run ./autoafids/run.py test_data/bids_T2w test_out participant --modality T2w --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ +# pixi run ./autoafids/run.py test_data/bids_ct test_out participant --modality ct --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ +# pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --stereotaxy STN --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ +# pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --fidqc --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs # --- Stage 2: Tiny Production Image --- # Use a lightweight glibc base image instead of a heavy Conda/Miniforge image diff --git a/entrypoint.sh b/entrypoint.sh index c92d427..910adf8 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,4 +1,2 @@ #!/bin/bash -source /opt/conda/etc/profile.d/conda.sh -conda activate snakebids-env exec /src/autoafids/run.py "$@" \ No newline at end of file From 517eee6aaa563651e1c33e47883f7e145fd1ae41 Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 11:40:07 -0400 Subject: [PATCH 07/14] add back creating conda envs and profile env --- Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a3efc4f..d6838d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,12 +9,12 @@ COPY . /src RUN pixi install --locked # Run your autoafids pre-caching tasks inside the pixi environment -# RUN pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ -# pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs --detect_with_nnlm && \ -# pixi run ./autoafids/run.py test_data/bids_T2w test_out participant --modality T2w --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ -# pixi run ./autoafids/run.py test_data/bids_ct test_out participant --modality ct --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ -# pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --stereotaxy STN --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ -# pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --fidqc --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs +RUN pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs --detect_with_nnlm && \ + pixi run ./autoafids/run.py test_data/bids_T2w test_out participant --modality T2w --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_ct test_out participant --modality ct --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --stereotaxy STN --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs && \ + pixi run ./autoafids/run.py test_data/bids_T1w test_out participant --participant-label 002 --fidqc --use-conda --conda-create-envs-only --cores all --conda-prefix /src/conda-envs # --- Stage 2: Tiny Production Image --- # Use a lightweight glibc base image instead of a heavy Conda/Miniforge image @@ -27,6 +27,7 @@ COPY --from=build /src /src # Add the pixi environment binaries straight to the PATH so you don't have to "activate" it ENV PATH="/src/.pixi/envs/default/bin:$PATH" +ENV SNAKEMAKE_PROFILE=/src/autoafids/workflow/profiles/docker-conda ENV PYTHONNOUSERSITE=1 ENV SNAKEMAKE_PROFILE=/src/autoafids/autoafids/workflow/profiles/docker-conda From a4cc42f99f3b1a3b27763e6597518517f593a53a Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 11:44:36 -0400 Subject: [PATCH 08/14] add git to the system --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d6838d0..1e5670d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ FROM ghcr.io/prefix-dev/pixi:latest AS build WORKDIR /src COPY . /src +# Install git so pip can clone remote repository dependencies +RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* + # Create the environment based on your lockfile and install dependencies # This creates an isolated, self-contained environment in /src/.pixi/envs/default RUN pixi install --locked @@ -29,6 +32,5 @@ COPY --from=build /src /src ENV PATH="/src/.pixi/envs/default/bin:$PATH" ENV SNAKEMAKE_PROFILE=/src/autoafids/workflow/profiles/docker-conda ENV PYTHONNOUSERSITE=1 -ENV SNAKEMAKE_PROFILE=/src/autoafids/autoafids/workflow/profiles/docker-conda ENTRYPOINT ["/src/entrypoint.sh"] \ No newline at end of file From 3fff202d97fdd35f13e177fffcd39c9547cdde3c Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 13:20:31 -0400 Subject: [PATCH 09/14] add libgomp1 --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1e5670d..cc089b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,9 @@ WORKDIR /src COPY . /src # Install git so pip can clone remote repository dependencies -RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* # Create the environment based on your lockfile and install dependencies # This creates an isolated, self-contained environment in /src/.pixi/envs/default From d851c33da76af754b5a349b623cd0c2663d8c4cd Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 13:28:48 -0400 Subject: [PATCH 10/14] add git back --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cc089b1..c17e79f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . /src # Install git so pip can clone remote repository dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ - libgomp1 \ + libgomp1 git \ && rm -rf /var/lib/apt/lists/* # Create the environment based on your lockfile and install dependencies From 8b5a039afeb15487d1a7f9ef661083277f7ace5e Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 15:20:43 -0400 Subject: [PATCH 11/14] add wget and unzip to the docker imgage --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c17e79f..642ce34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . /src # Install git so pip can clone remote repository dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ - libgomp1 git \ + libgomp1 git wget unzip \ && rm -rf /var/lib/apt/lists/* # Create the environment based on your lockfile and install dependencies From 4169acb11f8288edaaf5b5eaf036db334b5c8707 Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Sun, 7 Jun 2026 16:32:12 -0400 Subject: [PATCH 12/14] install system wide packages in the build phase --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 642ce34..0e3288d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,10 @@ FROM ubuntu:24.04 AS production WORKDIR /src +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgomp1 wget unzip \ + && rm -rf /var/lib/apt/lists/* + # Copy your source code, pre-cached conda environments, and the built Pixi environment COPY --from=build /src /src From ec30425e1a4055d93d3a67cc5c757bd01aa2ee6f Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Thu, 11 Jun 2026 10:46:15 -0400 Subject: [PATCH 13/14] use tmpdir directive in resource instead of hardcoding tmp in the path --- autoafids/workflow/rules/nnlm.smk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autoafids/workflow/rules/nnlm.smk b/autoafids/workflow/rules/nnlm.smk index fdbddff..9892837 100644 --- a/autoafids/workflow/rules/nnlm.smk +++ b/autoafids/workflow/rules/nnlm.smk @@ -73,13 +73,14 @@ rule run_nnlm: plans=config.get("nnlm_plans", "nnUNetResEncUNetMPlans"), checkpoint=config.get("nnlm_checkpoint", "checkpoint_final.pth"), device=config.get("nnlm_device", "cuda"), - tmpdir=lambda wildcards: f"/tmp/nnlm_{wildcards.subject}", + tmpdir=lambda wildcards, resources: f"{resources.tmpdir}/nnlm_{wildcards.subject}", conda: "../envs/nnlm.yaml" threads: 4 resources: gpus=lambda wildcards: 1 if config.get("nnlm_device", "cuda") == "cuda" else 0, mem_mb=16000, + tmpdir="tmp", log: bids( root="logs", From 2e7615421d2a5e5ba35f842c55e7225ccd917cb1 Mon Sep 17 00:00:00 2001 From: Dhananjhay Date: Fri, 12 Jun 2026 11:09:03 -0400 Subject: [PATCH 14/14] comment out tmpdir deletion in the shell command --- autoafids/workflow/rules/nnlm.smk | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autoafids/workflow/rules/nnlm.smk b/autoafids/workflow/rules/nnlm.smk index 9892837..462a57b 100644 --- a/autoafids/workflow/rules/nnlm.smk +++ b/autoafids/workflow/rules/nnlm.smk @@ -73,14 +73,13 @@ rule run_nnlm: plans=config.get("nnlm_plans", "nnUNetResEncUNetMPlans"), checkpoint=config.get("nnlm_checkpoint", "checkpoint_final.pth"), device=config.get("nnlm_device", "cuda"), - tmpdir=lambda wildcards, resources: f"{resources.tmpdir}/nnlm_{wildcards.subject}", + tmpdir=lambda wildcards: f"/tmp/nnlm_{wildcards.subject}", conda: "../envs/nnlm.yaml" threads: 4 resources: gpus=lambda wildcards: 1 if config.get("nnlm_device", "cuda") == "cuda" else 0, mem_mb=16000, - tmpdir="tmp", log: bids( root="logs", @@ -124,7 +123,7 @@ rule run_nnlm: cp $TMPDIR/output_dir/{wildcards.subject}.json {output.coords_json} # Cleanup temp dir - rm -rf $TMPDIR + # rm -rf $TMPDIR """