From 3774b3a2608dec768c2d70a0b1495b06bc4607fd Mon Sep 17 00:00:00 2001 From: Jean Mertz Date: Fri, 31 Jul 2026 14:22:07 +0200 Subject: [PATCH 1/3] windows CI test Signed-off-by: Jean Mertz --- .github/workflows/rust.yml | 35 +++++++++++++++++++++++++++++++++-- crates/jp_tombmap/src/lib.rs | 25 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a51a66b4b..ad0199073 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -96,7 +96,10 @@ jobs: strategy: matrix: os: [ubuntu-latest] - task: [lint, fmt, fmt-comments, fmt-markdown, test, docs, coverage, deny, insta, shear, vet] + # EXPERIMENT: trimmed to `test` while measuring Windows CI overhead. + # Restore before merging: + # [lint, fmt, fmt-comments, fmt-markdown, test, docs, coverage, deny, insta, shear, vet] + task: [test] include: - task: test os: windows-latest @@ -142,6 +145,21 @@ jobs: if [ "$run" != "true" ]; then echo "Skipping $TASK because no relevant files changed." fi + # Defender scans every file cargo, rustc and tar touch. The cache restore + # extracts ~4 GB of small files and cargo re-hydrates `registry/src` on + # top of that, so the scan tax dominates the job. + - name: Exclude build directories from Defender scanning + if: ${{ steps.task.outputs.run == 'true' && runner.os == 'Windows' }} + shell: pwsh + run: | + try { + "realtime protection: $((Get-MpComputerStatus).RealTimeProtectionEnabled)" + Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE", "$env:USERPROFILE\.cargo", "$env:USERPROFILE\.rustup", "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess "cargo.exe", "rustc.exe", "sccache.exe", "tar.exe", "zstd.exe", "rust-lld.exe", "link.exe" + "excluded paths: $((Get-MpPreference).ExclusionPath -join '; ')" + } catch { + "::warning::could not configure Defender exclusions: $_" + } - if: ${{ steps.task.outputs.run == 'true' }} uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6 - if: ${{ steps.task.outputs.run == 'true' }} @@ -150,7 +168,11 @@ jobs: with: path: | ~/.rustup - key: toolchain-${{ matrix.task }}${{ matrix.cache_suffix }} + key: toolchain-${{ matrix.task }}${{ matrix.cache_suffix }}-${{ hashFiles('rust-toolchain.toml') }} + # Prefix match, so a toolchain bump falls back to the previous entry + # instead of paying for a full rustup download. + restore-keys: | + toolchain-${{ matrix.task }}${{ matrix.cache_suffix }} - if: ${{ steps.task.outputs.run == 'true' }} uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3 - if: ${{ steps.task.outputs.run == 'true' }} @@ -171,6 +193,15 @@ jobs: run: | echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + # PROBE: `cargo nextest run` starts with a silent `cargo metadata`, which + # has to re-extract every `.crate` into `registry/src` because + # `rust-cache` strips that directory before saving. Running it as its own + # step attributes that cost instead of hiding it inside the test step. + # Remove once measured. + - name: Warm the registry source tree (timing probe) + if: ${{ steps.task.outputs.run == 'true' && matrix.task == 'test' }} + shell: bash + run: cargo metadata --locked --format-version 1 > /dev/null - if: ${{ steps.task.outputs.run == 'true' }} run: just ${{ matrix.task }}-ci - if: ${{ matrix.task == 'coverage' && steps.task.outputs.run == 'true' }} diff --git a/crates/jp_tombmap/src/lib.rs b/crates/jp_tombmap/src/lib.rs index 436fe97c9..3311307ea 100644 --- a/crates/jp_tombmap/src/lib.rs +++ b/crates/jp_tombmap/src/lib.rs @@ -1,3 +1,11 @@ +//! A hash map that remembers what changed. +//! +//! [`TombMap`] wraps a [`HashMap`] and records which keys were removed (leaving +//! a tombstone behind) and which were inserted or mutated. +//! A persistence layer can use those two sets to write only the entries that +//! changed and delete the ones that went away, instead of rewriting the whole +//! map. + use std::{ borrow::Borrow, collections::{HashMap, HashSet, TryReserveError, hash_map}, @@ -12,6 +20,23 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use self::Entry::*; +/// A [`HashMap`] that tracks removals and modifications. +/// +/// Removing a key records a tombstone, so [`removed_keys`] still reports it +/// afterwards. +/// Inserting a key or handing out a mutable reference to its value marks the +/// key as modified. +/// The `_untracked` methods (such as [`remove_untracked`] and +/// [`iter_mut_untracked`]) skip that bookkeeping. +/// +/// Only the live entries are serialized. +/// Deserializing therefore yields a map with no tombstones and nothing marked +/// modified, so both sets describe the changes made since the map was last +/// read. +/// +/// [`iter_mut_untracked`]: TombMap::iter_mut_untracked +/// [`remove_untracked`]: TombMap::remove_untracked +/// [`removed_keys`]: TombMap::removed_keys pub struct TombMap { live: HashMap, dead: HashSet, From a85a1eadffb4945b128c504b953984a079ced273 Mon Sep 17 00:00:00 2001 From: Jean Mertz Date: Fri, 31 Jul 2026 14:47:03 +0200 Subject: [PATCH 2/3] fixup! windows CI test Signed-off-by: Jean Mertz --- .github/workflows/rust.yml | 42 ++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ad0199073..16c038aef 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -105,7 +105,19 @@ jobs: os: windows-latest name_suffix: " (windows)" cache_suffix: "-windows" + # EXPERIMENT: stop caching `target/` on Windows and let sccache + # carry compilation instead. Dropping the path also changes the + # `actions/cache` version hash, so the existing 3.9 GB entry no + # longer matches and a registry-only entry is written in its place. + # `cache_save` keeps that entry warm on this branch so the second + # run onwards measures the steady state rather than a cold start. + # Revert both before merging. + cache_targets: "false" + cache_save: "true" name: ${{ matrix.task }}${{ matrix.name_suffix }} + # A run without a target cache falls back to a full build. Cap it so a + # pathological run doesn't burn an hour of Windows minutes unnoticed. + timeout-minutes: 60 steps: - name: Decide whether to run task id: task @@ -145,21 +157,6 @@ jobs: if [ "$run" != "true" ]; then echo "Skipping $TASK because no relevant files changed." fi - # Defender scans every file cargo, rustc and tar touch. The cache restore - # extracts ~4 GB of small files and cargo re-hydrates `registry/src` on - # top of that, so the scan tax dominates the job. - - name: Exclude build directories from Defender scanning - if: ${{ steps.task.outputs.run == 'true' && runner.os == 'Windows' }} - shell: pwsh - run: | - try { - "realtime protection: $((Get-MpComputerStatus).RealTimeProtectionEnabled)" - Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE", "$env:USERPROFILE\.cargo", "$env:USERPROFILE\.rustup", "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess "cargo.exe", "rustc.exe", "sccache.exe", "tar.exe", "zstd.exe", "rust-lld.exe", "link.exe" - "excluded paths: $((Get-MpPreference).ExclusionPath -join '; ')" - } catch { - "::warning::could not configure Defender exclusions: $_" - } - if: ${{ steps.task.outputs.run == 'true' }} uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6 - if: ${{ steps.task.outputs.run == 'true' }} @@ -181,7 +178,8 @@ jobs: key: ${{ matrix.task }} cache-bin: false cache-workspace-crates: true - save-if: ${{ github.ref == 'refs/heads/main' }} + cache-targets: ${{ matrix.cache_targets || 'true' }} + save-if: ${{ matrix.cache_save == 'true' || github.ref == 'refs/heads/main' }} - if: ${{ steps.task.outputs.run == 'true' && matrix.task != 'coverage' && matrix.task != 'shear' }} uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # V0.10 - if: ${{ steps.task.outputs.run == 'true' && matrix.task != 'coverage' && matrix.task != 'shear' }} @@ -193,12 +191,12 @@ jobs: run: | echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - # PROBE: `cargo nextest run` starts with a silent `cargo metadata`, which - # has to re-extract every `.crate` into `registry/src` because - # `rust-cache` strips that directory before saving. Running it as its own - # step attributes that cost instead of hiding it inside the test step. - # Remove once measured. - - name: Warm the registry source tree (timing probe) + # `rust-cache` strips `~/.cargo/registry/src` before saving, so the first + # cargo invocation of the job re-extracts every `.crate` archive. That is + # the single most expensive non-build phase on Windows. Doing it in a + # named step keeps the cost visible instead of burying it in the silent + # `cargo metadata` that `cargo nextest run` shells out to on startup. + - name: Warm the registry source tree if: ${{ steps.task.outputs.run == 'true' && matrix.task == 'test' }} shell: bash run: cargo metadata --locked --format-version 1 > /dev/null From f47efc1fd76493320133095bb7eb50e38657574f Mon Sep 17 00:00:00 2001 From: Jean Mertz Date: Fri, 31 Jul 2026 15:54:33 +0200 Subject: [PATCH 3/3] fixup! windows CI test Signed-off-by: Jean Mertz --- .github/workflows/rust.yml | 25 +++++++++++-------------- justfile | 16 ++++++++++++++-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 16c038aef..20ba888f4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -96,24 +96,21 @@ jobs: strategy: matrix: os: [ubuntu-latest] - # EXPERIMENT: trimmed to `test` while measuring Windows CI overhead. - # Restore before merging: - # [lint, fmt, fmt-comments, fmt-markdown, test, docs, coverage, deny, insta, shear, vet] - task: [test] + task: [lint, fmt, fmt-comments, fmt-markdown, test, docs, coverage, deny, insta, shear, vet] include: - task: test os: windows-latest name_suffix: " (windows)" cache_suffix: "-windows" - # EXPERIMENT: stop caching `target/` on Windows and let sccache - # carry compilation instead. Dropping the path also changes the - # `actions/cache` version hash, so the existing 3.9 GB entry no - # longer matches and a registry-only entry is written in its place. - # `cache_save` keeps that entry warm on this branch so the second - # run onwards measures the steady state rather than a cold start. - # Revert both before merging. + # Restoring a 3.9 GB `target/` cache costs Windows more than + # compiling from a warm sccache does, so this job caches only + # `~/.cargo` and lets sccache carry compilation. cache_targets: "false" - cache_save: "true" + # Windows pays a steep per-file cost to unpack, compile and link, so + # PR runs cover only the crates that make up `jp` itself. The Linux + # `test` job still builds and tests everything on every PR, and + # `main` covers the full workspace on Windows too. + scope: ${{ github.ref == 'refs/heads/main' && 'workspace' || 'jp-only' }} name: ${{ matrix.task }}${{ matrix.name_suffix }} # A run without a target cache falls back to a full build. Cap it so a # pathological run doesn't burn an hour of Windows minutes unnoticed. @@ -179,7 +176,7 @@ jobs: cache-bin: false cache-workspace-crates: true cache-targets: ${{ matrix.cache_targets || 'true' }} - save-if: ${{ matrix.cache_save == 'true' || github.ref == 'refs/heads/main' }} + save-if: ${{ github.ref == 'refs/heads/main' }} - if: ${{ steps.task.outputs.run == 'true' && matrix.task != 'coverage' && matrix.task != 'shear' }} uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # V0.10 - if: ${{ steps.task.outputs.run == 'true' && matrix.task != 'coverage' && matrix.task != 'shear' }} @@ -201,7 +198,7 @@ jobs: shell: bash run: cargo metadata --locked --format-version 1 > /dev/null - if: ${{ steps.task.outputs.run == 'true' }} - run: just ${{ matrix.task }}-ci + run: just ${{ matrix.task }}-ci ${{ matrix.scope }} - if: ${{ matrix.task == 'coverage' && steps.task.outputs.run == 'true' }} uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2 with: diff --git a/justfile b/justfile index 09f96122e..cabf3da82 100644 --- a/justfile +++ b/justfile @@ -12,6 +12,14 @@ vet_version := "0.10.2" quiet_flag := if env_var_or_default("CI", "") == "true" { "" } else { "--quiet" } +# Workspace crates that no `jp_*` crate depends on: standalone tooling binaries, +# docs infrastructure, and command plugins. Skipping them drops their +# dependencies from the build entirely, rather than merely skipping their tests. +# +# `grizzly` is deliberately absent: `jp_attachment_bear_note` depends on it, so +# it is compiled either way. +non_jp_excludes := "--exclude bookworm --exclude build-registry --exclude comfort --exclude jp-path --exclude jp-serve-web --exclude json_edit --exclude tools" + alias r := run alias i := install alias c := check @@ -2082,9 +2090,13 @@ fmt-markdown-ci: _install-comfort _install_ci_matchers comfort --check --workspace --language markdown --format-markdown --reference-links --prune-reference-links # Test the code on CI. +# +# `SCOPE` selects the crates to test: `workspace` covers every member, `jp-only` +# skips the crates listed in `non_jp_excludes`. An unrecognised scope tests the +# full workspace. [group('ci')] -test-ci: (_install "cargo-nextest@" + nextest_version) _install_ci_matchers - cargo nextest run --locked --lib --tests --cargo-profile=nextest --status-level=slow --failure-output=immediate-final --workspace --no-fail-fast +test-ci SCOPE="workspace": (_install "cargo-nextest@" + nextest_version) _install_ci_matchers + cargo nextest run --locked --lib --tests --cargo-profile=nextest --status-level=slow --failure-output=immediate-final --workspace --no-fail-fast {{ if SCOPE == "jp-only" { non_jp_excludes } else { "" } }} # Generate documentation on CI. [group('ci')]