Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
|| 'bash --noprofile --norc -euo pipefail {0}' }}
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v7
with:
fetch-depth: 1
persist-credentials: false
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/codeql.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ jobs:
build-mode: none
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v7
with:
fetch-depth: 1
persist-credentials: false
Expand Down Expand Up @@ -95,7 +95,7 @@ jobs:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
# actions/unpinned-tag is excluded below as a deliberate policy choice here, not an actionable defect.
# This repo intentionally references Actions by version tag (e.g. actions/checkout@v4) rather than by
# This repo intentionally references Actions by version tag (e.g. actions/checkout@v7) rather than by
# full-length commit SHA, so the query's supply-chain "pin to a commit" finding does not apply. Filters
# need a config to attach to, hence the inline config block rather than the queries input.
config: |
Expand Down
33 changes: 28 additions & 5 deletions .github/workflows/coverage.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
# actions/checkout defaults to the PR *merge* commit, which does not exist upstream, so DeepSource can't
# match it to the run it analyzed and the coverage reads as "never reported" (the check then times out).
# Check out the PR head SHA instead; `|| github.sha` keeps the push-to-main path working.
uses: actions/checkout@v4
uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
fetch-depth: 1
Expand All @@ -57,6 +57,29 @@ jobs:
github-token: ${{ github.token }}
install-action-tools: cargo-llvm-cov,deepsource

# Counted here, before the build and coverage steps, because sloc needs only git plus the two counters.
# Running it early means the figures land in the job summary even when a later coverage step fails, and a
# misclassification (the task's partition-sum check) fails in seconds rather than after the long run.
# sloc-crosscheck depends on sloc, so this one invocation emits both the tables and the gocloc deltas.
- name: Count source lines (scc, cross-checked with gocloc)
timeout-minutes: 5
run: |
mise run sloc-crosscheck | tee "$RUNNER_TEMP/sloc-report.txt"
{
echo '### Source lines of code'
echo '```'
cat "$RUNNER_TEMP/sloc-report.txt"
echo '```'
} >>"$GITHUB_STEP_SUMMARY"

- name: Upload sloc reports as artifacts
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: sloc-reports
path: target/sloc
if-no-files-found: error

- name: Prefetch dependencies
timeout-minutes: 15
env:
Expand Down Expand Up @@ -106,7 +129,7 @@ jobs:

- name: Upload coverage reports as artifacts
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7
with:
name: coverage-reports
path: |
Expand All @@ -115,15 +138,15 @@ jobs:
if-no-files-found: error

- name: Upload Rust coverage to Codecov (OIDC)
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@v7
with:
use_oidc: true
files: lcov.info
flags: rust
fail_ci_if_error: true

- name: Upload Python coverage to Codecov (OIDC)
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@v7
with:
use_oidc: true
files: coverage-python.xml
Expand All @@ -141,7 +164,7 @@ jobs:
# matter most; `report_type: test_results` selects the JUnit upload path.
- name: Upload test results to Codecov (OIDC)
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@v7
with:
use_oidc: true
report_type: test_results
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/dependencies.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
timeout-minutes: 25
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v7
with:
fetch-depth: 1
persist-credentials: false
Expand Down Expand Up @@ -73,7 +73,7 @@ jobs:
# timeout. Key on Cargo.lock so a dependency change refreshes; the prefix restore-key reuses the previous
# cache for crates that didn't change.
- name: Cache cargo-unmaintained lookups
uses: actions/cache@v4
uses: actions/cache@v6
with:
path: ~/.cache/cargo-unmaintained
key: cargo-unmaintained-${{ hashFiles('**/Cargo.lock') }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ jobs:
) }}
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v7
with:
fetch-depth: 1
persist-credentials: false
Expand Down Expand Up @@ -189,7 +189,7 @@ jobs:
shell: C:\Users\runneradmin\AppData\Local\mise\installs\http-busybox\1.37.0\ash.exe -euo pipefail {0}
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v7
with:
fetch-depth: 1
persist-credentials: false
Expand Down
106 changes: 106 additions & 0 deletions .mise/config.coverage.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@
# Windows is deliberately out of scope for this env (hence the os scoping here): coverage collection runs only
# on the ubuntu CI lane, and there is no intention to make the wasm coverage pipeline work on Windows.
[tools]
# scc is the primary line counter behind the sloc task.
# Picked over the alternatives on three grounds: a prebuilt binary for every target platform, counting rules
# documented upstream, and json/csv output so a reported figure can be regenerated rather than transcribed.
"aqua:boyter/scc" = "latest"
# gocloc re-counts the same file sets for sloc-crosscheck, as an independent implementation of cloc's rules.
# cloc itself cannot be a mise tool: upstream ships a Perl script plus a Windows .exe, so every non-Windows
# platform would fall back to host perl. gocloc is the closest thing that installs as a prebuilt binary.
"aqua:hhatto/gocloc" = "latest"
"conda:clang" = { version = "latest", os = ["linux", "macos"] }

# The wasm coverage builds run on nightly (-Zno-profiler-runtime) with the wasm-capable conda clang on PATH.
Expand Down Expand Up @@ -454,3 +462,101 @@ rpt="$covdir/report.txt"
rg "ws-modules/pic-viewer/src/lib.rs|Filename|TOTAL" "$rpt" || true
"""
shell = "bash -euo pipefail -c"

[tasks.sloc]
description = "Count source lines of code: non-test vs test totals plus a per-language split (scc)"
run = """
out=target/sloc
coreutils rm -rf "$out"
coreutils mkdir -p "$out"

# Count the repository of record rather than the working tree.
# `git ls-files` fixes the input set without relying on either counter's own ignore handling, and it hands this
# task and sloc-crosscheck byte-identical lists so a difference between the two tools is only ever a counting
# rule. verification/ is the one exclusion: those files are expected-output fixtures a generator emits for
# byte-comparison, so they are data the counts would inflate, not source anyone reads.
excluded='^verification/'
# A test is a file under a `tests/` directory, or one whose basename starts with `test_`.
# That is the repo's stated test convention and, checked against the tree, the only two shapes in it.
tests='(^|/)(tests/|test_[^/]*$)'
# generated/ counts as source, and is also measured on its own so a report can state how much is machine-made.
# It holds no test-shaped path, so it sits entirely inside the non-test partition and cannot split across the
# two -- which is what keeps the identity check below meaningful.
generated='^generated/'

git ls-files | rg -v "$excluded" >"$out/all.txt"
rg "$tests" <"$out/all.txt" >"$out/test.txt"
rg -v "$tests" <"$out/all.txt" >"$out/non-test.txt"
rg "$generated" <"$out/all.txt" >"$out/generated.txt"

# --no-cocomo drops the cost estimate: COCOMO is a model with its own assumptions, not a measurement.
# Each partition also lands as json + csv so a figure in prose can be regenerated instead of transcribed.
count() {
echo "== $2 =="
fmt="tabular:stdout,json:$out/$1.json,csv:$out/$1.csv"
xargs scc --no-cocomo --sort code --format-multi "$fmt" <"$out/$1.txt"
}
count all "all source"
count non-test "non-test source"
count test "test source"
count generated "generated source (a subset of non-test, shown for reference)"

# Per-language test vs non-test, joined from the same json the totals come from so the two cannot disagree.
# scc counts one partition per run, so the split has to be assembled here rather than read off a single table.
{
printf 'language\\tnon_test_code\\ttest_code\\ttotal_code\\n'
# $doc and $l are jaq bindings, not shell variables, so the program stays single-quoted.
# shellcheck disable=SC2016
printf '{"nontest":%s,"test":%s}' "$(coreutils cat "$out/non-test.json")" "$(coreutils cat "$out/test.json")" |
jaq -r '. as $doc
| (reduce $doc.nontest[] as $l ({}; .[$l.Name] = {nontest: $l.Code, test: 0}))
| (reduce $doc.test[] as $l (.; .[$l.Name] = {nontest: (.[$l.Name].nontest // 0), test: $l.Code}))
| to_entries
| map({lang: .key, nontest: .value.nontest, test: .value.test, total: (.value.nontest + .value.test)})
| sort_by(-.total)
| .[]
| [.lang, .nontest, .test, .total]
| @tsv'
} >"$out/by-language.tsv"

# The test share is derived at print time rather than stored, so the tsv holds only measured counts.
echo "== per-language test vs non-test =="
goawk -F '\\t' 'NR == 1 { printf "%-30s %10s %10s %10s %7s\\n", "Language", $2, $3, $4, "test%"; next }
{ nt += $2; t += $3; total += $4
printf "%-30s %10d %10d %10d %7s\\n", substr($1, 1, 30), $2, $3, $4,
($4 > 0 ? sprintf("%.1f", 100 * $3 / $4) : "-") }
END { printf "%-30s %10d %10d %10d %7s\\n", "TOTAL", nt, t, total,
(total > 0 ? sprintf("%.1f", 100 * t / total) : "-") }' "$out/by-language.tsv"

# Every counted line must fall in exactly one partition, so a classification miss fails instead of passing.
code() { jaq '[.[].Code] | add' <"$out/$1.json"; }
total=$(code all)
production=$(code non-test)
tested=$(code test)
machine=$(code generated)
if [ "$total" -ne "$((production + tested))" ]; then
echo "sloc partitions do not sum: all=$total non-test=$production test=$tested" >&2
exit 1
fi
echo "sloc: $total code lines = $production non-test + $tested test; $machine of the non-test lines generated"
"""
shell = "bash -euo pipefail -c"

[tasks.sloc-crosscheck]
depends = ["sloc"]
description = "Re-count sloc's file lists with gocloc, an independent implementation of cloc's rules"
# Corroboration, not a gate.
# Two implementations disagree by a few lines wherever their comment and blank-line rules differ, and the
# point is to show the primary figure does not rest on a single tool. A wide gap means something is being
# counted that should not be, so the deltas are printed for a human to read rather than asserted on.
run = """
out=target/sloc
for part in all non-test test generated; do
xargs gocloc --output-type=json <"$out/$part.txt" >"$out/gocloc-$part.json"
scc_code=$(jaq '[.[].Code] | add' <"$out/$part.json")
gocloc_code=$(jaq '.total.code' <"$out/gocloc-$part.json")
delta=$((scc_code - gocloc_code))
printf "%-9s scc=%s gocloc=%s delta=%s\\n" "$part" "$scc_code" "$gocloc_code" "$delta"
done
"""
shell = "bash -euo pipefail -c"
43 changes: 34 additions & 9 deletions .mise/config.dotnet.toml
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,41 @@ description = "Format .NET sources"
depends = ["roslynator-fix"]
description = "Apply .NET lint-fix passes (roslynator fix)"

[tasks.dotnet-workload-wasm-tools]
description = "Install the wasm-tools workload once, before any dotnet module build"
# One installer for a workload two module builds share, because a second concurrent install destroys the first.
# `dotnet workload install` unpacks into the single mise dotnet-root, and a second copy racing the first hits
# the half-written nupkg, fails, and *rolls back* -- uninstalling the packs the winning build is mid-publish
# against. The victim then dies on a pack that vanished underneath it:
# Workload installation failed: The file '<dotnet-root>/library-packs/
# microsoft.net.sdk.webassembly.pack.10.0.8.nupkg' already exists.
# error MSB4175: The task factory "JsonToItemsTaskFactory.JsonToItemsTaskFactory" could not be loaded from
# the assembly '<dotnet-root>/packs/Microsoft.NET.Runtime.MonoTargets.Sdk/10.0.8/Sdk/../tasks/net10.0/
# MonoTargetsTasks.dll'. The system cannot find the file specified.
# Seen on linux-arm64 running build-modules-all, where the two dotnet builds start close enough together to
# overlap.
#
# The install runs unconditionally: `dotnet workload list` naming wasm-tools does NOT mean the packs are
# usable, and skipping on that basis broke every Windows lane. There the publish silently left the
# WebAssembly SDK path and ILLink demanded an entry point these `OutputType=Library` modules do not have:
# ILLink : error IL1034: Root assembly 'dotnet-data1, Version=1.0.0.0, Culture=neutral,
# PublicKeyToken=null' does not have entry point.
# error NETSDK1144: Optimizing assemblies for size failed.
# All three Windows lanes at once on commit 017a9aa4035a50db663ad6a800d1005051ca9a78,
# https://github.com/edge-toolkit/core/actions/runs/32226380541/job/95986792705 -- serializing the install was
# the fix for the race, guarding it was not part of that and only hid a broken workload state.
run = "dotnet workload install wasm-tools --skip-manifest-update"
shell = "bash -euo pipefail -c"

[tasks.build-ws-dotnet-data1-module]
depends = ["dotnet-workload-wasm-tools"]
description = "Build the dotnet-data1 C# WASM workflow module"
dir = "services/ws-modules/dotnet-data1"
run = '''
# `set -x` so every command is echoed to the log.
# The Windows lane is newly re-enabled; keep the trace until it is reliably green in CI.
set -x

dotnet workload install wasm-tools --skip-manifest-update

# Windows: cap the PATH the SDK's emcc link <Exec> constructs, or bare `emcc` becomes unresolvable.
# BrowserWasmApp.targets(522) runs the link as `<Exec Command='emcc ...'>` with an EnvironmentVariables PATH
# composed of the Emscripten workload pack dirs + $([MSBuild]::Escape($(PATH))). cmd.exe cannot read an
Expand Down Expand Up @@ -105,14 +130,13 @@ cp "$PUBLISH"/*.js "$PUBLISH"/*.wasm "$PUBLISH"/*.dat pkg/
shell = "bash -euo pipefail -c"

[tasks.build-ws-dotnet-math1-module]
depends = ["dotnet-workload-wasm-tools"]
description = "Build the dotnet-math1 C# WASM FedAvg module"
dir = "services/ws-modules/dotnet-math1"
# Same publish flow as build-ws-dotnet-data1-module, including its Windows PATH-cap workaround.
run = '''
set -x

dotnet workload install wasm-tools --skip-manifest-update

publish_args=""
if [ "${OS:-}" = "Windows_NT" ]; then
publish_args="-p:PATH=C:\\Windows\\System32"
Expand All @@ -126,11 +150,12 @@ PUBLISH=bin/Release/net10.0/publish/wwwroot/_framework
cp "$PUBLISH"/*.js "$PUBLISH"/*.wasm "$PUBLISH"/*.dat pkg/
'''
shell = "bash -euo pipefail -c"
# Both dotnet publishes drive emscripten out of one shared workload pack tree, so they run one at a time.
# `wait_for` rather than `depends` because the edge only matters when both are already scheduled -- building
# math1 alone stays one task.
wait_for = ["build-ws-dotnet-data1-module"]

[tasks."prefetch:dotnet"]
depends = ["dotnet-workload-wasm-tools"]
description = "Prefetch .NET dependencies and the wasm-tools workload"
run = """
dotnet restore
dotnet workload install wasm-tools --skip-manifest-update
"""
shell = "bash -euo pipefail -c"
run = "dotnet restore"
Loading
Loading