diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index 7ec06c9..d46fa2a 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -1,8 +1,11 @@ # Dart toolchain + tasks. Loaded when MISE_ENV includes `dart`. [vars] -# Every tracked Dart package: the two generated clients plus the two modules. Shared by analyze/format. -dart_paths = "generated/dart-ws generated/dart-rest services/ws-modules/dart-comm1 services/ws-modules/dart-data1" +# Every tracked Dart package: the two generated clients plus the three modules. Shared by analyze/format. +# Split in two only to stay inside the editorconfig line length; dart_paths is the one callers use. +dart_generated_paths = "generated/dart-ws generated/dart-rest" +dart_module_paths = "services/ws-modules/dart-comm1 services/ws-modules/dart-data1 services/ws-modules/dart-math1" +dart_paths = "{{ vars.dart_generated_paths }} {{ vars.dart_module_paths }}" [tools] "cargo:dart-typegen" = { version = "latest", os = ["linux", "macos"] } diff --git a/.mise/config.macos.toml b/.mise/config.macos.toml index 9226723..bf7940c 100644 --- a/.mise/config.macos.toml +++ b/.mise/config.macos.toml @@ -88,7 +88,9 @@ missing="" brewy="" for pkg in $pkgs; do case "$pkg" in - ca-certificates | libc6-dev | libicu* | binutils) cmd="" ;; + # binutils names the package, `as` is the binary -- so the Homebrew check below has something to resolve. + binutils) cmd="as" ;; + ca-certificates | libc6-dev | libicu*) cmd="" ;; *) cmd="$pkg" ;; esac case "$pkg" in diff --git a/.mise/config.maint.toml b/.mise/config.maint.toml index 4e8547b..a84d147 100644 --- a/.mise/config.maint.toml +++ b/.mise/config.maint.toml @@ -304,12 +304,13 @@ if [ "${OS:-}" = "Windows_NT" ]; then # lookup below works regardless of which envs were exported. The conda-m2-gnupg directory is only needed by # this repo's own build (NOT rustpython's), so dropping it for the cargo install is safe. PATH on git-bash # uses `:` as the separator. - PATH=$(echo "$PATH" | tr ':' '\\n' | grep -v 'conda-m2-gnupg' | paste -sd ':' -) + PATH=$(echo "$PATH" | coreutils tr ':' '\\n' | rg -v 'conda-m2-gnupg' | coreutils paste -sd ':' -) export PATH # Pick the first where.exe match under an MSVC tools dir. # where.exe returns every match newline-separated. If none is found we # leave the default lookup alone (the PATH strip above is usually enough). - msvc_link=$(where.exe link.exe 2>/dev/null | tr -d '\\r' | grep -iE 'MSVC[\\/].*Hostx64' | head -1 || true) + msvc_where=$(where.exe link.exe 2>/dev/null | coreutils tr -d '\\r' | rg -i 'MSVC[\\/].*Hostx64' || true) + msvc_link=$(printf '%s\\n' "$msvc_where" | coreutils head -1) if [ -n "${msvc_link:-}" ]; then export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER="$msvc_link" fi diff --git a/.mise/config.toml b/.mise/config.toml index 8f68bb6..6fdfd68 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -1295,7 +1295,7 @@ description = "Prefetch the Node (pnpm) workspace" env = { CI = "true" } # Installs the whole workspace from the repo root, so no --dir. # The members come from pnpm-workspace.yaml and the resolution lands in the single root pnpm-lock.yaml. -run = "pnpm install --ignore-scripts --config.ignoredBuiltDependencies=protobufjs" +run = "pnpm install --ignore-scripts" # `shell = "bash"` on Windows for the same base-config-task PATH dropout. # Same dropout as `prefetch:rust` and `build-et-cli` -- cmd subprocess can't find pnpm. Captured on # 3ac0d60b windows test job 82282265424 (after the prefetch:rust fix, the same failure surfaced one step diff --git a/CLAUDE.md b/CLAUDE.md index ab3abb9..8508683 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -895,9 +895,10 @@ standalone file or a mise task `run`. The available linters: - **shfmt on mise task bodies** (`.mise/shfmt-mise.awk`, via `shfmt-mise-fmt` and `shfmt-mise-check`) -- splits those same bodies out, shfmt-formats each one, and merges the result back over the config, so a task body is held to the same formatting as any other shell in the repo. The check reports shfmt's own diff and names `shfmt-mise-fmt` as the - fix. Its one demand on how a body is written: inside a `"""` block every backslash escape must be doubled, since a - lone `\n` / `\r` / `\b` is TOML syntax that folds into a control character rather than the two bytes the shell wants - -- the split pass rejects one instead of guessing which was meant. + fix. Its one demand on how a body is written: inside a `"""` block a backslash meant to reach the shell must be + doubled, since a lone `\n` / `\r` / `\b` is TOML syntax that folds into a control character rather than the two bytes + the shell wants -- the split pass rejects one instead of guessing which was meant. `\\` and `\"` are the two escapes + it does decode, so an embedded quote stays written the TOML way. - plus hadolint, ls-lint (file/dir naming), zizmor (Actions security), ryl (YAML), lychee (links), clang-format / clang-tidy / cpplint / flawfinder (C, in the zig config), editorconfig-checker, typos, and action-validator for their domains. diff --git a/README.md b/README.md index 55270e4..a348d57 100644 --- a/README.md +++ b/README.md @@ -303,9 +303,9 @@ the other two runners). A test-harness "fake agent" uploads a canonical input fi pointer to it over the hub; each module reads the input, runs the same FedAvg simulation -- rounds of local gradient-descent epochs per simulated client, merged with a sample-count-weighted average, using only `+ - * /` on IEEE-754 doubles -- and stores its resulting global model back to storage, where the test harness verifies that -every language produced bit-identical weights. To trigger the twins manually, run the `math1-sender` module in -another browser tab: it plays the fake-agent side itself, uploading the canonical input and broadcasting the -pointer once a second for a minute. +every language's weights land on the expected model within the tolerance its shared contract defines. To trigger the +twins manually, run the `math1-sender` module in another browser tab: it plays the fake-agent side itself, uploading +the canonical input and broadcasting the pointer once a second for a minute. ## Root module diff --git a/config/semgrep/ls-lint-no-glob.yaml b/config/semgrep/ls-lint-no-glob.yaml index 7581337..97dbef5 100644 --- a/config/semgrep/ls-lint-no-glob.yaml +++ b/config/semgrep/ls-lint-no-glob.yaml @@ -4,7 +4,9 @@ rules: paths: include: - "/config/ls-lint.yaml" - pattern-regex: '\*\*' + # Matches a list item (the `ignore:` entries are the file's only sequence) carrying any glob metacharacter. + # Scoping to `- ` keeps the `ls:` rules above clear, whose `regex:` values legitimately contain `[`. + pattern-regex: '-\s+\S*[*?\[{]' message: >- ls-lint config entries must be literal paths -- `**` (and any other glob syntax) is banned. ls-lint snapshots glob-based ignores once at startup and deletes every key that matches nothing at that moment diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c9f444e..191ddcd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -13,5 +13,13 @@ packages: - services/ws-modules/js-data1 # esbuild ships its platform binary via a postinstall build script; allow it so the js-data1 bundle can build. # Everything else stays blocked by pnpm's default supply-chain policy. +# +# A blocked script still has to be named here with `false`. pnpm aborts an install that silently ignored one +# (`[ERR_PNPM_IGNORED_BUILDS] Ignored build scripts: protobufjs@7.6.5`, telling you to run +# `pnpm approve-builds`), so an undeclared entry fails every cold tree while a warm one passes on recorded +# state -- which is how build-ws-js-data1-module broke on a fresh linux-arm64 clone but nowhere else. +# protobufjs (transitive, via the aws-sdk graph) only uses its postinstall to pick a native codec, and the +# bundle runs the pure-JS path, so blocking it is the outcome we want -- just declared rather than implied. allowBuilds: esbuild: true + protobufjs: false diff --git a/services/ws-modules/dart-math1/lib/dart_math1.dart b/services/ws-modules/dart-math1/lib/dart_math1.dart index cd90d2c..048a2bc 100644 --- a/services/ws-modules/dart-math1/lib/dart_math1.dart +++ b/services/ws-modules/dart-math1/lib/dart_math1.dart @@ -76,13 +76,22 @@ Future waitFor(String what, T? Function() ready) async { /// The broadcast pointer naming the storage bucket + filename of the input JSON. ({String bucket, String filename})? inputPointer; +/// One path segment of a storage URL, with no separator or traversal syntax. +/// +/// The pointer arrives in a relayed broadcast from an arbitrary peer, so a `bucket` of `../..` would +/// otherwise steer the read at another storage path. The java, kotlin, and dotnet twins hold their pointers +/// to the same shape, so every math1 twin shares one trust model. +final _safeSegment = RegExp(r'^[A-Za-z0-9][A-Za-z0-9._-]*$'); + void captureInputPointer(String frame) { try { final msg = jsonDecode(frame); if (msg is Map && msg['type'] == 'math1-input' && msg['bucket'] is String && - msg['filename'] is String) { + msg['filename'] is String && + _safeSegment.hasMatch(msg['bucket'] as String) && + _safeSegment.hasMatch(msg['filename'] as String)) { inputPointer = ( bucket: msg['bucket'] as String, filename: msg['filename'] as String, @@ -102,7 +111,8 @@ List fedAvg(Map input) { .map( (samples) => (samples as List) .map( - (sample) => (sample as List).map((v) => (v as num).toDouble()).toList(), + (sample) => + (sample as List).map((v) => (v as num).toDouble()).toList(), ) .toList(), ) @@ -155,7 +165,10 @@ Future run() async { }).toJS, ); client.connect(); - await waitFor('WebSocket connection', () => client.get_state() == 'connected' ? true : null); + await waitFor( + 'WebSocket connection', + () => client.get_state() == 'connected' ? true : null, + ); final agentId = await waitFor('agent_id', () { final id = client.get_agent_id(); return id.isEmpty ? null : id; @@ -177,13 +190,19 @@ Future run() async { final input = jsonDecode(utf8.decode(inputBytes)) as Map; final clientCount = (input['clients'] as List).length; - log('running FedAvg - $clientCount clients x ${input['rounds']} rounds x ${input['epochs']} local epochs'); + log( + 'running FedAvg - $clientCount clients x ${input['rounds']} rounds x ${input['epochs']} local epochs', + ); final model = fedAvg(input); final weight = model[0]; final bias = model[1]; log('global model weight=$weight bias=$bias'); - final output = jsonEncode({'module': 'dart-math1', 'weight': weight, 'bias': bias}); + final output = jsonEncode({ + 'module': 'dart-math1', + 'weight': weight, + 'bias': bias, + }); await rest.storage.putFile( agentId: agentId, filename: 'math1-output.json', diff --git a/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js index c24aebc..f8bb3cd 100644 --- a/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js +++ b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js @@ -23,6 +23,8 @@ export default async function init() { setModuleImports("dotnet-math1", { wsConnect: (url) => { + // Cleared per connect: the page caches this module, so a second run() must not see the first run's id. + agentId = ""; ws = new WebSocket(url); wsState = "connecting"; ws.onopen = () => { diff --git a/services/ws-modules/java-math1/pkg/et_ws_java_math1.js b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js index b38f741..bf332db 100644 --- a/services/ws-modules/java-math1/pkg/et_ws_java_math1.js +++ b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js @@ -22,6 +22,8 @@ export default async function init() { // TeaVM @JSBody calls reference `host` as a global globalThis.host = { wsConnect: (url) => { + // Cleared per connect: the page caches this module, so a second run() must not see the first run's id. + agentId = ""; ws = new WebSocket(url); wsState = "connecting"; ws.onopen = () => { diff --git a/services/ws-modules/js-data1/src/index.js b/services/ws-modules/js-data1/src/index.js index da0dea5..0e8adb2 100644 --- a/services/ws-modules/js-data1/src/index.js +++ b/services/ws-modules/js-data1/src/index.js @@ -51,7 +51,18 @@ function httpBase() { function connectAgent() { const ws = new WebSocket(websocketUrl()); return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("timed out waiting for et-connect-ack")), 10000); + // `run`'s finally block only closes the socket it was handed, so a rejection here has to close its own -- + // otherwise a failed handshake leaves the runner holding an open socket the server keeps registered. + // `settled` keeps a late ack from resolving a promise already rejected, which would hand back a closed ws. + let settled = false; + const fail = (message) => { + if (settled) return; + settled = true; + clearTimeout(timer); + ws.close(); + reject(new Error(message)); + }; + const timer = setTimeout(() => fail("timed out waiting for et-connect-ack"), 10000); ws.addEventListener("message", (event) => { let frame; try { @@ -60,14 +71,13 @@ function connectAgent() { return; } if (frame.type === "et-connect-ack" && frame.agent_id) { + if (settled) return; + settled = true; clearTimeout(timer); resolve({ agentId: frame.agent_id, ws }); } }); - ws.addEventListener("error", () => { - clearTimeout(timer); - reject(new Error("websocket error before et-connect-ack")); - }); + ws.addEventListener("error", () => fail("websocket error before et-connect-ack")); ws.addEventListener("open", () => ws.send(JSON.stringify({ agent_id: null, type: "et-connect" }))); }); } diff --git a/services/ws-modules/kotlin-data1/pkg/et_ws_kotlin_data1.js b/services/ws-modules/kotlin-data1/pkg/et_ws_kotlin_data1.js index bec293b..7fa8e7a 100644 --- a/services/ws-modules/kotlin-data1/pkg/et_ws_kotlin_data1.js +++ b/services/ws-modules/kotlin-data1/pkg/et_ws_kotlin_data1.js @@ -9,6 +9,8 @@ export default async function init() { // The Kotlin js() interop bridges reference `host` as a global globalThis.host = { wsConnect: (url) => { + // Cleared per connect: the page caches this module, so a second run() must not see the first run's id. + agentId = ""; ws = new WebSocket(url); wsState = "connecting"; ws.onopen = () => { diff --git a/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js b/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js index 67a33c8..fb11736 100644 --- a/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js +++ b/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js @@ -21,6 +21,8 @@ export default async function init() { // The Kotlin js() interop bridges reference `host` as a global globalThis.host = { wsConnect: (url) => { + // Cleared per connect: the page caches this module, so a second run() must not see the first run's id. + agentId = ""; ws = new WebSocket(url); wsState = "connecting"; ws.onopen = () => { diff --git a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js index 3a6ef28..61881a6 100644 --- a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js +++ b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js @@ -72,6 +72,8 @@ export async function run() { ); return; case 1: + // Cleared per connect: the page caches this module, so a second run() must not see the first run's id. + agentId = ""; ws = new WebSocket(payload); wsState = "connecting"; ws.onopen = () => { @@ -157,6 +159,8 @@ export async function run() { worker.terminate(); if (e.data.ret === 0) { resolve(); + } else if (e.data.error) { + reject(new Error("zig-math1: worker failed: " + e.data.error)); } else { reject(new Error("zig-math1: run() returned " + e.data.ret)); } diff --git a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js index 805f973..51d1ed5 100644 --- a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js +++ b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js @@ -78,8 +78,14 @@ self.onmessage = async (e) => { // Resolve the module wasm from this worker's own location (self.location), never from a postMessage value, // so the fetch URL cannot depend on message data. The wasm is a fixed-name sibling of this worker script. const wasmUrl = new URL("et_ws_zig_math1.wasm", self.location.href); - const { instance } = await WebAssembly.instantiateStreaming(fetch(wasmUrl), imports); - wasmMemory = instance.exports.memory; - const ret = instance.exports.run(); - self.postMessage({ done: true, ret }); + // Every failure still has to post `done`: instantiateStreaming can reject and run() can trap, and an + // unhandled rejection here posts nothing at all, leaving the page waiting on a message that never arrives. + try { + const { instance } = await WebAssembly.instantiateStreaming(fetch(wasmUrl), imports); + wasmMemory = instance.exports.memory; + const ret = instance.exports.run(); + self.postMessage({ done: true, ret }); + } catch (error) { + self.postMessage({ done: true, error: String(error), ret: -1 }); + } };