Onboard Tillicum: cluster doc, Slurm port, and first-login recon (#51/#70)

scripts/model_comparison/pack_yolo_dataset.slurm

@@ -0,0 +1,123 @@
+#!/bin/bash
+# Pack one YOLO dataset into a SquashFS image, for transfer klone -> Tillicum.
+#
+# WHY SQUASHFS AND NOT rsync/scp/tar
+# The dataset is 286 GB in ~911,000 files (tiles 210 GB / 718,415; pano 76 GB /
+# 192,938 -- measured 2026-07-30). That is metadata-bound, not bandwidth-bound: `du -sh`
+# over the tiles tree beat three timeouts before finishing on a 550 s budget, and even
+# `ls -lU | head -201` on tiles/images/train timed out.
+#
+#   - rsync/scp of the tree pays ~911k round trips. No.
+#   - tar fixes the TRANSFER but not the DESTINATION: untarring recreates all 911k files
+#     on /gpfs, paying the metadata cost again and permanently.
+#   - SquashFS pays it ONCE, here, and the destination only ever sees ONE file. The
+#     training job mounts it read-only and the kernel serves the tree from a compact
+#     in-image index. This is also what UW-IT recommends for many-small-file datasets
+#     (Sumaiya, 2026-07-30).
+#
+# WHY A SLURM JOB AND NOT A LOGIN NODE
+# klone reaps heavy login processes, and that reap also kills the SSH control master
+# every agent session depends on. Never run this interactively.
+#
+# WHERE IT WRITES
+# /gscratch/scrubbed, NOT /gscratch/makelab: lab storage is 84% full with ~168 GB free
+# (checked 2026-07-30) and cannot hold this. scrubbed has ~15 TB free, and its ~21-day
+# purge does not matter for a transient staging area.
+#
+#   DS=pano  sbatch -A ckpt-makelab scripts/model_comparison/pack_yolo_dataset.slurm
+#   DS=tiles sbatch -A ckpt-makelab scripts/model_comparison/pack_yolo_dataset.slurm
+#
+#SBATCH --job-name=yolo_pack
+#SBATCH -p ckpt
+#SBATCH -q ckpt
+#SBATCH --requeue
+#SBATCH --time=12:00:00
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32G
+#SBATCH --mail-type=NONE
+#SBATCH --output=logs/yolo_pack_%j.out
+#SBATCH --error=logs/yolo_pack_%j.err
+
+set -euo pipefail
+
+DS="${DS:?set DS to pano or tiles}"
+SRC="${SRC:-/gscratch/scrubbed/$USER/yolo}"
+OUT="${OUT:-/gscratch/scrubbed/$USER/yolo_squashfs}"
+mkdir -p "$OUT"
+
+IMG="$OUT/${DS}.sqfs"
+SUM="$IMG.sha256"
+
+echo "--- pack $DS ---"
+echo "src: $SRC/$DS   out: $IMG"
+df -h "$OUT" | tail -1
+
+# Idempotent: a completed, verified image is left alone, so a ckpt requeue or a manual
+# resubmit is free rather than a restart from zero.
+if [ -f "$IMG" ] && [ -f "$SUM" ]; then
+    echo "[skip] image exists; verifying..."
+    if ( cd "$OUT" && sha256sum -c "$(basename "$SUM")" ); then
+        echo "[skip] verified, nothing to do"; exit 0
+    fi
+    echo "[warn] checksum FAILED -- rebuilding"
+    rm -f "$IMG" "$SUM"
+fi
+
+# ---------------------------------------------------------------------------------
+# THE ULTRALYTICS LABEL CACHE IS THE SUBTLE PART -- READ BEFORE CHANGING THIS.
+#
+# Ultralytics writes labels/<split>.cache next to the labels and validates it against a
+# hash of the label+image FILE PATHS. Two consequences:
+#
+#   1. The .cache files MUST be inside the image. They already exist on klone (built
+#      2026-07-25/26), and a read-only mount cannot create them. Without them every run
+#      rescans ~911k files -- exactly the cost this whole exercise exists to avoid.
+#
+#   2. The hash covers absolute paths, so mounting at a DIFFERENT path than the one the
+#      cache was built under INVALIDATES it and forces that same rescan. Mount the image
+#      on Tillicum at a stable path and expect to regenerate the cache ONCE there (it
+#      needs a writable overlay, or a one-off run with labels/ bind-mounted writable).
+#      Budget one slow first epoch; do not mistake it for the steady-state cost.
+#
+# This is called out in docs/tillicum.md too, because getting it wrong is invisible --
+# training still works, it is just silently slow forever.
+# ---------------------------------------------------------------------------------
+echo "[check] label caches that will be packed:"
+ls -la "$SRC/$DS/labels/"*.cache 2>/dev/null || echo "  NONE -- expect a full rescan on first use"
+
+# -noD: do NOT compress data blocks. The payload is JPEG, already compressed; deflating
+# it would burn hours of CPU for ~nothing and make reads slower. Metadata/inodes are
+# still compressed, which is where the win actually is.
+# -no-xattrs: nothing here uses them, and they bloat the index.
+echo "[mksquashfs] starting $(date)"
+mksquashfs "$SRC/$DS" "$IMG" \
+    -noD \
+    -no-xattrs \
+    -processors "${SLURM_CPUS_PER_TASK:-8}" \
+    -info
+echo "[mksquashfs] done $(date)"
+
+# The checksum travels with the image and is verified after transfer, same discipline as
+# the weight snapshot's MANIFEST.md.
+( cd "$OUT" && sha256sum "$(basename "$IMG")" > "$(basename "$SUM")" )
+
+ls -lh "$IMG"
+cat "$SUM"
+
+cat <<EOF
+
+[ok] $DS packed.
+
+Next, on Tillicum -- note /gpfs/scrubbed, not the 1 TB /gpfs/projects/makelab quota
+(UW-IT recommends scrubbed for active many-small-file datasets):
+
+  # mount read-only, no untar, no 911k files ever created on /gpfs
+  apptainer exec --nv \\
+      --bind /gpfs/scrubbed/\$USER/${DS}.sqfs:/data/${DS}:image-src=/ \\
+      <image.sif> <command>
+
+Verify the checksum on arrival BEFORE trusting it:
+  sha256sum -c $(basename "$SUM")
+EOF

scripts/model_comparison/run_yolo_data_prep_tillicum.slurm

@@ -0,0 +1,92 @@
+#!/bin/bash
+# Rebuild the YOLO training dataset ON TILLICUM, from Hugging Face. Download + prep.
+#
+# WHY REGENERATE RATHER THAN TRANSFER FROM KLONE
+# There is no automated path from klone to Tillicum: no shared filesystem (/gscratch and
+# /mmfs1 do not exist here), and Tillicum cannot authenticate to klone non-interactively
+# — both ends are Duo/keyboard-interactive with no publickey, so BatchMode fails. Globus
+# CLI is on neither side. Verified 2026-07-30.
+#
+# Regenerating sidesteps all of it, and is SAFE because the prep is deterministic by
+# construction: prepare_yolo_dataset.py thins background tiles with an md5 of the file
+# stem specifically so the choice is stable across processes and runs (see
+# `_keep_background`, and its comment about salted hash()).
+#
+# VERIFY, DO NOT ASSUME. The committed record gives the exact counts klone produced:
+#     tiles 557,413 train / 161,002 val      pano 150,063 train / 42,875 val
+# This job prints its own counts at the end. If they do not match, the datasets are NOT
+# equivalent and nothing trained here is comparable to the #51 runs — stop and diff.
+#
+# WHY A GPU JOB FOR CPU WORK
+# Tillicum prohibits CPU-only jobs; every job must request >=1 GPU. The runbook also
+# warns that login nodes reap heavy long-running processes, and on Tillicum that would
+# take the Duo-authenticated SSH master with it. So we hold one GPU and eat the cost:
+# roughly 6-12 h at $0.90/h, i.e. $5-11. Cheap against a $1,500/month cap.
+#
+#   sbatch scripts/model_comparison/run_yolo_data_prep_tillicum.slurm
+#
+#SBATCH --job-name=yolo_data_prep
+#SBATCH --qos=normal
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=200G
+#SBATCH --time=12:00:00
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --mail-type=NONE
+#SBATCH --output=logs/yolo_data_prep_%j.out
+#SBATCH --error=logs/yolo_data_prep_%j.err
+
+set -euo pipefail
+
+export REPO="${REPO:-/gpfs/home/$USER/RampNet}"
+# /gpfs/scrubbed, NOT the 1 TB /gpfs/projects/makelab quota: the HF source is ~462 GB
+# and the prepared YOLO layout another ~286 GB, so this needs ~750 GB. scrubbed has
+# ~484 TB free and is what UW-IT recommends for active many-small-file datasets.
+export SCRATCH="${SCRATCH:-/gpfs/scrubbed/$USER}"
+export PYBIN="${PYBIN:-/gpfs/projects/makelab/$USER/envs/rampnet-yolo/bin/python}"
+export HF_HOME="$SCRATCH/hf"
+export YOLO_CONFIG_DIR="$SCRATCH/ultralytics"
+
+mkdir -p "$SCRATCH"
+
+echo "=== node ==="
+hostname; date
+df -h "$SCRATCH" | tail -1
+echo "=== interpreter (must match the klone #51 runs exactly) ==="
+"$PYBIN" -c "import sys, torch, ultralytics; print('python', sys.version.split()[0], '| torch', torch.__version__, '| ultralytics', ultralytics.__version__)"
+
+cd "$REPO"
+
+echo
+echo "=== STAGE: data (HF download, ~462 GB) ==="
+date
+bash hyak_yolo_runbook.sh data
+
+echo
+echo "=== STAGE: prep (tiles + pano) ==="
+date
+bash hyak_yolo_runbook.sh prep
+
+echo
+echo "=== VERIFICATION — counts must match the klone record ==="
+date
+for ds in tiles pano; do
+    for split in train val; do
+        d="$SCRATCH/yolo/$ds/images/$split"
+        printf "  %-6s %-6s %s\n" "$ds" "$split" "$(ls -U "$d" 2>/dev/null | grep -c '\.jpg$')"
+    done
+done
+cat <<'EOF'
+
+  Expected, from scripts/model_comparison/yolo_baseline/README.md provenance:
+    tiles  train  557413
+    tiles  val    161002
+    pano   train  150063
+    pano   val    42875
+
+  A mismatch means the regenerated dataset is NOT the one the #51 arms trained on.
+  Do not train against it until the difference is understood.
+EOF
+date
+echo "Cost so far: run `hyakusage`"

scripts/model_comparison/run_yolo_train_tillicum.slurm

@@ -0,0 +1,161 @@
+#!/bin/bash
+# Train one YOLO curb-ramp detector on the RampNet dataset, on TILLICUM.
+#
+# Port of run_yolo_train.slurm (klone). Kept as a SEPARATE file rather than
+# branching inside the klone script, because nearly every scheduler directive
+# differs and the klone version is the preserved record of the #51 runs -- it
+# should not churn. See docs/tillicum.md for the directive-by-directive diff.
+#
+# WHAT CHANGED FROM THE KLONE VERSION, AND WHY
+#   -p ckpt-g2 / -q ckpt-gpu  ->  --qos=normal   Tillicum has NO partitions; QoS only.
+#   --requeue                 ->  dropped        Nothing preempts on Tillicum.
+#   --time=72:00:00           ->  24:00:00       normal QoS caps at 24 h. 7-day jobs
+#                                                need `long` QoS (pre-approval).
+#   --gpus-per-node=1         ->  --gres=gpu:N   Tillicum's documented spelling.
+#   --cpus-per-task=12        ->  8 x N GPUs     HARD 8:1 ratio; 12-on-1 is REJECTED.
+#   (no mail directive)       ->  --mail-type=NONE   Be explicit so the site default
+#                                                cannot decide for us (on klone an
+#                                                inherited END,FAIL,TIME_LIMIT produced
+#                                                ~1 email per minute under preemption).
+#
+#   YOLO_DATA=$DATA/pano/data.yaml YOLO_IMGSZ=1280 BATCH=4 NAME=y26_pano_till \
+#       sbatch scripts/model_comparison/run_yolo_train_tillicum.slurm
+#
+# COST. Tillicum BILLS: $0.90/GPU-hour where GPU-hour = elapsed x N GPUs. A 24 h
+# 2-GPU job is 48 GPU-hours (~$43). Do not leave allocations idle. Check `hyakusage`.
+#SBATCH --qos=normal
+#SBATCH --job-name=yolo_curb_ramp_train
+#SBATCH --time=24:00:00
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --mail-type=NONE
+#SBATCH --output=logs/yolo_train_till_%j.out
+#SBATCH --error=logs/yolo_train_till_%j.err
+# gres/cpus/mem are NOT hardcoded -- see the GPUS block below, which must be kept in
+# step with them. Override at submit time with: sbatch --gres=gpu:2 --cpus-per-task=16
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=200G
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------------
+# The 2-GPU dataloader trick, and why DEVICE is separate from the allocation.
+#
+# Tillicum fixes CPUs at 8 per GPU. Our tiles arm is I/O-bound (the same epoch took
+# 4.1-9.5 h on klone depending only on which node it landed on), so more dataloader
+# workers is the obvious lever -- but on Tillicum more CPUs means more GPUs.
+#
+# The naive move -- allocate 2 GPUs and let ultralytics use both -- is NOT a clean
+# experiment: 2 GPUs puts ultralytics into DDP, which changes the effective batch and
+# the LR dynamics, so the result is no longer comparable to the klone single-GPU runs.
+#
+# So: allocate 2 GPUs for their 16 CPUs, but pin DEVICE=0 so training math is
+# IDENTICAL to a 1-GPU run. The second GPU sits idle and we pay for it. That is the
+# point -- it isolates dataloader throughput as the only changed variable, and it
+# answers whether the 2x bill buys more than a 2x speedup. Set DEVICE=0,1 only if you
+# deliberately want a DDP run, and label it as a different arm.
+# ---------------------------------------------------------------------------------
+DEVICE="${DEVICE:-0}"
+
+YOLO_CKPT="${YOLO_CKPT:-yolo11l.pt}"
+YOLO_DATA="${YOLO_DATA:?set YOLO_DATA to a prepared data.yaml}"
+YOLO_IMGSZ="${YOLO_IMGSZ:-1024}"
+EPOCHS="${EPOCHS:-60}"
+BATCH="${BATCH:--1}"                          # pin per config to match the klone runs
+PATIENCE="${PATIENCE:-20}"
+
+# Keep a checkpoint every N epochs. Ultralytics defaults save_period=-1, keeping ONLY
+# last.pt and best.pt -- which forecloses, permanently and retroactively, any analysis
+# that needs a checkpoint from a specific epoch.
+#
+# The one we care about: RampNet's published model was trained for ~1 epoch / ~9.4k
+# steps at constant LR (#84), while this baseline gets 60 epochs with a schedule. If
+# RampNet wins anyway that is a STRONGER result -- but only if we can also report a
+# COMPUTE-MATCHED YOLO point, i.e. the checkpoint at the epoch where YOLO has consumed
+# comparable training, not just the converged one. That requires the checkpoint to
+# still exist. results.csv gives per-epoch val metrics but no weights, so it cannot be
+# reconstructed after the fact.
+#
+# Cost is disk (~150 MB per checkpoint per arm) against a 1 TB allocation. Cheap
+# insurance for an option that cannot be bought back later.
+SAVE_PERIOD="${SAVE_PERIOD:-5}"
+# On klone this defaulted to 0 because ultralytics `time=` restarts its clock on every
+# preemption. Tillicum never preempts, so `time=` is safe here -- but leave it 0 for
+# any run meant to be comparable to the #51 klone results, which are epoch-based.
+TRAIN_HOURS="${TRAIN_HOURS:-0}"
+NAME="${NAME:-yolo_run_till}"
+
+# Tillicum storage. /gpfs/projects/makelab is 1 TB, backed up daily, and PURGED AT END
+# OF PROJECT -- it is NOT archival. Only best.pt matters downstream: rsync it back to
+# klone /gscratch/makelab, which stays the system of record.
+GPFS="${GPFS:-/gpfs/projects/makelab/$USER}"
+export HF_HOME="${HF_HOME:-$GPFS/hf}"
+PROJECT="${PROJECT:-$GPFS/yolo_runs}"
+export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$GPFS/ultralytics}"
+mkdir -p "$PROJECT" "$YOLO_CONFIG_DIR" "$HF_HOME"
+
+# PYTHON points at an interpreter with ultralytics. On Tillicum the DOCUMENTED path is
+# an Apptainer container (NGC PyTorch + `pip install ultralytics`), not a conda module
+# -- our environment.yml pins CUDA 11.8 against what is now a Rocky 9 / H200 host and
+# should not be assumed to transfer. Set PYTHON, or set APPTAINER_IMG to run in one.
+PYTHON="${PYTHON:-python}"
+APPTAINER_IMG="${APPTAINER_IMG:-}"
+
+echo "--- YOLO train on TILLICUM (issue #51 / #70) ---"
+echo "base:   ${YOLO_CKPT}"
+echo "data:   ${YOLO_DATA}"
+echo "imgsz:  ${YOLO_IMGSZ}   epochs: ${EPOCHS}   batch: ${BATCH}   patience: ${PATIENCE}"
+echo "alloc:  ${SLURM_GPUS_ON_NODE:-?} GPU(s), ${SLURM_CPUS_PER_TASK:-?} CPUs on ${SLURMD_NODENAME:-?}"
+echo "device: ${DEVICE}   (allocation and device differ on purpose -- see header)"
+echo "out:    ${PROJECT}/${NAME}/weights/best.pt"
+nvidia-smi --query-gpu=name,memory.total --format=csv || true
+echo "------------------------------------------------"
+
+run_train() {
+    "$@" - "$YOLO_CKPT" "$YOLO_DATA" "$YOLO_IMGSZ" "$EPOCHS" "$BATCH" "$PATIENCE" \
+            "$PROJECT" "$NAME" "$TRAIN_HOURS" "$DEVICE" "$SAVE_PERIOD" <<'PY'
+import os, sys
+from ultralytics import YOLO
+(ckpt, data, imgsz, epochs, batch, patience, project, name, hours, device,
+ save_period) = sys.argv[1:12]
+last = os.path.join(project, name, "weights", "last.pt")
+# Still needed on Tillicum, but for a different reason than on klone: not preemption,
+# but the 24 h normal-QoS ceiling. A 60-epoch tiles schedule does not fit in one job,
+# so chain jobs and resume. Unlike klone the boundary is predictable, not random.
+#
+# CAVEAT: resume=True reuses every training arg SAVED IN THE CHECKPOINT, so a run
+# resumed from a klone checkpoint keeps that run's save_period=-1 -- SAVE_PERIOD above
+# does NOT apply to it. Per-epoch checkpoints therefore exist only for runs STARTED
+# under this script. For the klone arms the early-epoch weights are already gone.
+#
+# That is less costly than it sounds: the compute-matched point we most want (YOLO at
+# roughly RampNet's ~1-epoch budget) is exactly the ep1 best.pt that four of the six
+# klone arms are still holding. The "one-epoch models" are the matched comparison.
+if os.path.exists(last):
+    print(f"[resume] {last} exists -> resuming", flush=True)
+    YOLO(last).train(resume=True)
+else:
+    bf = float(batch)
+    batch_arg = int(bf) if bf.is_integer() else bf
+    dev = [int(d) for d in device.split(",")] if "," in device else int(device)
+    kw = dict(
+        data=data, imgsz=int(imgsz), epochs=int(epochs), batch=batch_arg,
+        patience=int(patience), project=project, name=name, device=dev, exist_ok=True,
+        save_period=int(save_period),
+    )
+    if float(hours) > 0:
+        kw["time"] = float(hours)
+    YOLO(ckpt).train(**kw)
+PY
+}
+
+# run_train takes the interpreter as "$@", so the container case is just a longer
+# command word -- the heredoc still lands on its stdin and every shell variable is
+# expanded HERE, outside the container, before apptainer is ever invoked.
+# --nv exposes the GPUs; bind /gpfs so the container sees data, runs and cache.
+if [ -n "$APPTAINER_IMG" ]; then
+    run_train apptainer exec --nv --bind /gpfs:/gpfs "$APPTAINER_IMG" python
+else
+    run_train "$PYTHON"
+fi