From 26c11da12a256c6c7f6a3174ecbd36eaa751b641 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 17 Aug 2026 04:47:11 +0800 Subject: [PATCH 1/8] Add math1 modules --- .codacy.yaml | 1 + .dockerignore | 2 + .gitignore | 2 + .mise/config.dart.toml | 10 + .mise/config.dotnet.toml | 35 ++- .mise/config.kotlin.toml | 19 +- .mise/config.python.toml | 11 + .mise/config.r.toml | 13 + .mise/config.rust.toml | 17 ++ .mise/config.zig.toml | 5 + CLAUDE.md | 30 ++- Cargo.lock | 32 +++ Cargo.toml | 2 + .../rules/doc-summary-ends-with-period.yaml | 5 + .../rules/no-relative-path-literal.yaml | 3 + config/coverage.toml | 1 + config/ls-lint.yaml | 22 ++ config/pyrefly.toml | 1 + config/semgrep/prefer-yaml-toml.yaml | 1 + core.slnx | 1 + pom.xml | 37 +++ pubspec.yaml | 1 + pyproject.toml | 1 + .../ws-modules/dart-math1/lib/dart_math1.dart | 212 +++++++++++++++ services/ws-modules/dart-math1/pkg/.gitignore | 1 + .../dart-math1/pkg/et_ws_dart_math1.js | 37 +++ .../ws-modules/dart-math1/pkg/package.json | 8 + services/ws-modules/dart-math1/pubspec.yaml | 18 ++ services/ws-modules/dotnet-math1/.gitignore | 8 + services/ws-modules/dotnet-math1/Program.cs | 126 +++++++++ .../dotnet-math1/dotnet-math1.csproj | 11 + .../ws-modules/dotnet-math1/pkg/.gitignore | 4 + .../dotnet-math1/pkg/et_ws_dotnet_math1.js | 77 ++++++ .../ws-modules/dotnet-math1/pkg/package.json | 8 + services/ws-modules/java-math1/pkg/.gitignore | 1 + .../java-math1/pkg/et_ws_java_math1.js | 96 +++++++ .../ws-modules/java-math1/pkg/package.json | 8 + .../src/main/java/au/edu/curtin/et/Math1.java | 209 ++++++++++++++ .../ws-modules/js-math1/pkg/et_ws_js_math1.js | 114 ++++++++ services/ws-modules/js-math1/pkg/package.json | 8 + .../ws-modules/kotlin-math1/build.gradle.kts | 31 +++ .../ws-modules/kotlin-math1/pkg/.gitignore | 1 + .../kotlin-math1/pkg/et_ws_kotlin_math1.js | 89 ++++++ .../ws-modules/kotlin-math1/pkg/package.json | 8 + .../kotlin-math1/settings.gradle.kts | 1 + .../src/wasmJsMain/kotlin/Math1.kt | 143 ++++++++++ services/ws-modules/math1/Cargo.toml | 32 +++ services/ws-modules/math1/src/lib.rs | 255 ++++++++++++++++++ services/ws-modules/pymath1/pkg/.gitignore | 2 + .../ws-modules/pymath1/pkg/et_ws_pymath1.js | 110 ++++++++ .../ws-modules/pymath1/pymath1/__init__.py | 58 ++++ services/ws-modules/pymath1/pyproject.toml | 19 ++ .../ws-modules/rmath1/pkg/et_ws_rmath1.js | 59 ++++ services/ws-modules/rmath1/pkg/module.R | 148 ++++++++++ services/ws-modules/rmath1/pkg/package.json | 8 + services/ws-modules/wasi-math1/.gitignore | 4 + services/ws-modules/wasi-math1/Cargo.toml | 34 +++ services/ws-modules/wasi-math1/build.rs | 10 + .../ws-modules/wasi-math1/src/coverage.rs | 16 ++ services/ws-modules/wasi-math1/src/lib.rs | 215 +++++++++++++++ services/ws-modules/zig-math1/build.zig | 50 ++++ services/ws-modules/zig-math1/build.zig.zon | 9 + services/ws-modules/zig-math1/pkg/.gitignore | 2 + .../zig-math1/pkg/et_ws_zig_math1.js | 178 ++++++++++++ .../zig-math1/pkg/et_ws_zig_math1_worker.js | 85 ++++++ services/ws-modules/zig-math1/src/main.zig | 212 +++++++++++++++ services/ws-pyo3-runner/python/math1.py | 82 ++++++ services/ws-pyo3-runner/tests/modules.rs | 20 ++ services/ws-test-server/Cargo.toml | 1 + services/ws-test-server/data/math1-input.json | 25 ++ services/ws-test-server/src/lib.rs | 14 +- services/ws-test-server/src/math1.rs | 163 +++++++++++ services/ws-wasi-runner/tests/modules.rs | 60 +++++ services/ws-web-runner/Cargo.toml | 1 + services/ws-web-runner/tests/modules.rs | 109 ++++++++ 75 files changed, 3433 insertions(+), 19 deletions(-) create mode 100644 services/ws-modules/dart-math1/lib/dart_math1.dart create mode 100644 services/ws-modules/dart-math1/pkg/.gitignore create mode 100644 services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js create mode 100644 services/ws-modules/dart-math1/pkg/package.json create mode 100644 services/ws-modules/dart-math1/pubspec.yaml create mode 100644 services/ws-modules/dotnet-math1/.gitignore create mode 100644 services/ws-modules/dotnet-math1/Program.cs create mode 100644 services/ws-modules/dotnet-math1/dotnet-math1.csproj create mode 100644 services/ws-modules/dotnet-math1/pkg/.gitignore create mode 100644 services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js create mode 100644 services/ws-modules/dotnet-math1/pkg/package.json create mode 100644 services/ws-modules/java-math1/pkg/.gitignore create mode 100644 services/ws-modules/java-math1/pkg/et_ws_java_math1.js create mode 100644 services/ws-modules/java-math1/pkg/package.json create mode 100644 services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java create mode 100644 services/ws-modules/js-math1/pkg/et_ws_js_math1.js create mode 100644 services/ws-modules/js-math1/pkg/package.json create mode 100644 services/ws-modules/kotlin-math1/build.gradle.kts create mode 100644 services/ws-modules/kotlin-math1/pkg/.gitignore create mode 100644 services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js create mode 100644 services/ws-modules/kotlin-math1/pkg/package.json create mode 100644 services/ws-modules/kotlin-math1/settings.gradle.kts create mode 100644 services/ws-modules/kotlin-math1/src/wasmJsMain/kotlin/Math1.kt create mode 100644 services/ws-modules/math1/Cargo.toml create mode 100644 services/ws-modules/math1/src/lib.rs create mode 100644 services/ws-modules/pymath1/pkg/.gitignore create mode 100644 services/ws-modules/pymath1/pkg/et_ws_pymath1.js create mode 100644 services/ws-modules/pymath1/pymath1/__init__.py create mode 100644 services/ws-modules/pymath1/pyproject.toml create mode 100644 services/ws-modules/rmath1/pkg/et_ws_rmath1.js create mode 100644 services/ws-modules/rmath1/pkg/module.R create mode 100644 services/ws-modules/rmath1/pkg/package.json create mode 100644 services/ws-modules/wasi-math1/.gitignore create mode 100644 services/ws-modules/wasi-math1/Cargo.toml create mode 100644 services/ws-modules/wasi-math1/build.rs create mode 100644 services/ws-modules/wasi-math1/src/coverage.rs create mode 100644 services/ws-modules/wasi-math1/src/lib.rs create mode 100644 services/ws-modules/zig-math1/build.zig create mode 100644 services/ws-modules/zig-math1/build.zig.zon create mode 100644 services/ws-modules/zig-math1/pkg/.gitignore create mode 100644 services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js create mode 100644 services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js create mode 100644 services/ws-modules/zig-math1/src/main.zig create mode 100644 services/ws-pyo3-runner/python/math1.py create mode 100644 services/ws-test-server/data/math1-input.json create mode 100644 services/ws-test-server/src/math1.rs diff --git a/.codacy.yaml b/.codacy.yaml index 68d2b2bd..a1fa936c 100644 --- a/.codacy.yaml +++ b/.codacy.yaml @@ -43,6 +43,7 @@ exclude_paths: - "services/ws-modules/dotnet-data1/Program.cs" - "services/ws-modules/wasi-comm1/src/coverage.rs" - "services/ws-modules/wasi-data1/src/coverage.rs" + - "services/ws-modules/wasi-math1/src/coverage.rs" - "services/ws-test-server/src/bin/cov-server.rs" - "services/ws-web-runner/mingw-shim/msvc_crt_alloc.c" - "services/ws-web-runner/mingw-shim/msvc_crt_locale_cache.c" diff --git a/.dockerignore b/.dockerignore index 88a012f6..d2e08ef6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,7 @@ services/ws-wasm-agent/pkg/ services/ws-modules/pywasm1/pkg/ services/ws-modules/rdata1/pkg/webr/ services/ws-modules/rcomm1/pkg/webr/ +services/ws-modules/rmath1/pkg/webr/ services/ws-modules/js-data1/pkg/et_ws_js_data1.js services/ws-server/static/models/ **/.zig-cache/ @@ -34,6 +35,7 @@ services/ws-modules/dotnet-data1/bin/ **/.gradle/ **/.kotlin/ services/ws-modules/kotlin-data1/build/ +services/ws-modules/kotlin-math1/build/ # Coverage-report artifacts produced by the coverage workflow, uploaded to Codecov and never committed. # Covers cargo-llvm-cov's lcov, pytest-cov's Cobertura xml, and pytest-cov's .coverage data file. lcov.info diff --git a/.gitignore b/.gitignore index 70ab5754..7d35d491 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ services/ws-wasm-agent/pkg/ services/ws-modules/pywasm1/pkg/ services/ws-modules/rdata1/pkg/webr/ services/ws-modules/rcomm1/pkg/webr/ +services/ws-modules/rmath1/pkg/webr/ services/ws-modules/js-data1/pkg/et_ws_js_data1.js services/ws-server/static/models/ .zig-cache/ @@ -30,6 +31,7 @@ services/ws-modules/dotnet-data1/bin/ .gradle/ .kotlin/ services/ws-modules/kotlin-data1/build/ +services/ws-modules/kotlin-math1/build/ # Coverage-report artifacts produced by the coverage workflow, uploaded to Codecov and never committed. # Covers cargo-llvm-cov's lcov, pytest-cov's Cobertura xml, and pytest-cov's .coverage data file. /lcov.info diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index e6435d7a..ae2485df 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -76,12 +76,22 @@ dart compile js lib/dart_data1.dart -o pkg/et_ws_dart_data1_compiled.js --no-sou """ shell = "bash -euo pipefail -c" +[tasks.build-ws-dart-math1-module] +description = "Build the dart-math1 FedAvg module" +dir = "services/ws-modules/dart-math1" +run = """ +dart pub get +dart compile js lib/dart_math1.dart -o pkg/et_ws_dart_math1_compiled.js --no-source-maps +""" +shell = "bash -euo pipefail -c" + [tasks."prefetch:dart"] description = "Prefetch Dart (pub) dependencies" run = """ dart pub get --directory generated/dart-rest dart pub get --directory services/ws-modules/dart-comm1 dart pub get --directory services/ws-modules/dart-data1 +dart pub get --directory services/ws-modules/dart-math1 """ shell = "bash -euo pipefail -c" diff --git a/.mise/config.dotnet.toml b/.mise/config.dotnet.toml index eb3d789a..855a5dde 100644 --- a/.mise/config.dotnet.toml +++ b/.mise/config.dotnet.toml @@ -38,11 +38,19 @@ description = "Analyze C# sources (Roslynator)" # setting a forward-slashed DOTNET_ROOT). The env var itself is fine as the `-m` argument -- mise's core # dotnet plugin exports DOTNET_ROOT on every OS, and the forward-slashed form ash delivers on Windows is # accepted there; only MSBuildLocator's own probing of it breaks. -run = 'roslynator analyze -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-data1/dotnet-data1.csproj' +run = """ +roslynator analyze -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-data1/dotnet-data1.csproj +roslynator analyze -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-math1/dotnet-math1.csproj +""" +shell = "bash -euo pipefail -c" [tasks.roslynator-fix] description = "Apply Roslynator diagnostics' machine-applicable C# fixes in place" -run = 'roslynator fix -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-data1/dotnet-data1.csproj' +run = """ +roslynator fix -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-data1/dotnet-data1.csproj +roslynator fix -m "{{ vars.dotnet_msbuild }}" services/ws-modules/dotnet-math1/dotnet-math1.csproj +""" +shell = "bash -euo pipefail -c" # Namespaced aggregators picked up by the default config's globbed `check`/`fmt`/`fix`. [tasks."check:dotnet"] @@ -96,6 +104,29 @@ cp "$PUBLISH"/*.js "$PUBLISH"/*.wasm "$PUBLISH"/*.dat pkg/ ''' shell = "bash -euo pipefail -c" +[tasks.build-ws-dotnet-math1-module] +description = "Build the dotnet-math1 C# WASM FedAvg module" +dir = "services/ws-modules/dotnet-math1" +# Same publish flow as build-ws-dotnet-data1-module, including its Windows PATH-cap workaround. +run = ''' +set -x + +dotnet workload install wasm-tools --skip-manifest-update + +publish_args="" +if [ "${OS:-}" = "Windows_NT" ]; then + publish_args="-p:PATH=C:\\Windows\\System32" +fi + +# $publish_args holds zero or one CLI flag; word-splitting is intentional. +# shellcheck disable=SC2086 +dotnet publish -c Release $publish_args + +PUBLISH=bin/Release/net10.0/publish/wwwroot/_framework +cp "$PUBLISH"/*.js "$PUBLISH"/*.wasm "$PUBLISH"/*.dat pkg/ +''' +shell = "bash -euo pipefail -c" + [tasks."prefetch:dotnet"] description = "Prefetch .NET dependencies and the wasm-tools workload" run = """ diff --git a/.mise/config.kotlin.toml b/.mise/config.kotlin.toml index ba80a588..2fe024a8 100644 --- a/.mise/config.kotlin.toml +++ b/.mise/config.kotlin.toml @@ -31,16 +31,27 @@ dir = "services/ws-modules/kotlin-data1" run = "\"$GRADLE\" --console=plain pkgDist" shell = "bash -euo pipefail -c" +[tasks.build-ws-kotlin-math1-module] +description = "Build the kotlin-math1 FedAvg module (Kotlin/Wasm -> WasmGC)" +dir = "services/ws-modules/kotlin-math1" +run = "\"$GRADLE\" --console=plain pkgDist" +shell = "bash -euo pipefail -c" + # Namespaced aggregator picked up by the default config's globbed `check`. # The compile runs kotlinc with allWarningsAsErrors (set in build.gradle.kts), the Kotlin analogue of -Werror. +# Each Kotlin module is its own Gradle root project, so the check compiles them one -p at a time. [tasks."check:kotlin"] description = "Run Kotlin checks (kotlinc allWarningsAsErrors via the Gradle compile)" -dir = "services/ws-modules/kotlin-data1" -run = "\"$GRADLE\" --console=plain compileKotlinWasmJs" +run = """ +"$GRADLE" --console=plain -p services/ws-modules/kotlin-data1 compileKotlinWasmJs +"$GRADLE" --console=plain -p services/ws-modules/kotlin-math1 compileKotlinWasmJs +""" shell = "bash -euo pipefail -c" [tasks."prefetch:kotlin"] description = "Prefetch Kotlin (Gradle plugin + Maven Central) dependencies" -dir = "services/ws-modules/kotlin-data1" -run = "\"$GRADLE\" --console=plain dependencies" +run = """ +"$GRADLE" --console=plain -p services/ws-modules/kotlin-data1 dependencies +"$GRADLE" --console=plain -p services/ws-modules/kotlin-math1 dependencies +""" shell = "bash -euo pipefail -c" diff --git a/.mise/config.python.toml b/.mise/config.python.toml index 5fa35997..80ece8c6 100644 --- a/.mise/config.python.toml +++ b/.mise/config.python.toml @@ -129,6 +129,17 @@ uv build --wheel --out-dir pkg """ shell = "bash -euo pipefail -c" +[tasks.build-ws-pymath1-module] +depends = ["build-et-cli"] +description = "Build the pymath1 FedAvg module" +dir = "services/ws-modules/pymath1" +run = """ +coreutils rm -f pkg/*.whl +uv build --wheel --out-dir pkg +{{ vars.et_cli }} module-package-json +""" +shell = "bash -euo pipefail -c" + [tasks.build-ws-pydata1-module] depends = ["build-et-rest-client-wheel"] description = "Build the pydata1 Python workflow module" diff --git a/.mise/config.r.toml b/.mise/config.r.toml index 0f2d4178..5d131cc9 100644 --- a/.mise/config.r.toml +++ b/.mise/config.r.toml @@ -35,6 +35,19 @@ coreutils rm -f pkg/webr/metadata.json """ shell = "bash -euo pipefail -c" +[tasks.build-ws-rmath1-module] +description = "Vendor the webR distribution into rmath1's pkg/webr/ (served at /modules/et-ws-rmath1/webr/)" +dir = "services/ws-modules/rmath1" +run = """ +src="$(mise where "http:webr")" +[ -n "$src" ] || { echo "http:webr not installed; run 'MISE_ENV=r mise install' first" >&2; exit 1; } +coreutils rm -rf pkg/webr +coreutils mkdir -p pkg/webr +coreutils cp -R "$src/." pkg/webr/ +coreutils rm -f pkg/webr/metadata.json +""" +shell = "bash -euo pipefail -c" + [tasks.build-ws-rcomm1-module] description = "Vendor the webR distribution into rcomm1's pkg/webr/ (served at /modules/et-ws-rcomm1/webr/)" dir = "services/ws-modules/rcomm1" diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 953e85c9..98420e5f 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -24,6 +24,12 @@ description = "Build the data1 workflow WASM module" dir = "services/ws-modules/data1" run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +[tasks.build-ws-math1-module] +depends = ["build-wasm-cov-wrapper"] +description = "Build the math1 FedAvg WASM module" +dir = "services/ws-modules/math1" +run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" + [tasks.build-ws-comm1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the comm1 workflow WASM module" @@ -107,6 +113,17 @@ cp target/wasm32-wasip2/release/et_ws_wasi_data1.wasm services/ws-modules/wasi-d """ shell = "bash -euo pipefail -c" +[tasks.build-ws-wasi-math1-module] +depends = ["build-et-cli"] +description = "Build the Rust WASI math1 module as a WASI Preview 2 component" +run = """ +{{ vars.wasm_cov }} cargo build --release -p et-ws-wasi-math1 --target wasm32-wasip2{{ vars.wasi_cov_feat }} +mkdir -p services/ws-modules/wasi-math1/pkg +cp target/wasm32-wasip2/release/et_ws_wasi_math1.wasm services/ws-modules/wasi-math1/pkg/et_ws_wasi_math1.wasm +{{ vars.et_cli }} module-package-json --module-dir services/ws-modules/wasi-math1 +""" +shell = "bash -euo pipefail -c" + [tasks.build-ws-wasi-comm1-module] depends = ["build-et-cli"] description = "Build the Rust WASI comm1 module as a WASI Preview 2 component" diff --git a/.mise/config.zig.toml b/.mise/config.zig.toml index 5fc1b193..4eeaebd3 100644 --- a/.mise/config.zig.toml +++ b/.mise/config.zig.toml @@ -185,6 +185,11 @@ description = "Build the zig-data1 workflow WASM module" dir = "services/ws-modules/zig-data1" run = "zig build -Doptimize=ReleaseSmall" +[tasks.build-ws-zig-math1-module] +description = "Build the zig-math1 FedAvg WASM module" +dir = "services/ws-modules/zig-math1" +run = "zig build -Doptimize=ReleaseSmall" + [tasks.build-ws-zig-except1-module] description = "Build the zig-except1 exception-handling demo WASM module" dir = "services/ws-modules/zig-except1" diff --git a/CLAUDE.md b/CLAUDE.md index cdd81b4c..d63e088a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -361,16 +361,28 @@ The server only serves them from disk. Languages: - **Rust -> WASM** (wasm-pack): audio1, bluetooth, comm1, data1, except1, face-detection, geolocation, graphics-info, - har1, nfc, sensor1, speech-recognition, video1 -- **Dart -> JS**: dart-comm1 -- **Kotlin -> WASM (WasmGC)**: kotlin-data1 -- compiled by the Kotlin Gradle plugin's `wasmJs` target; the - module is a WasmGC binary (browser GC manages the Kotlin heap), so it needs a WasmGC-capable engine -- **Python (Pyodide)**: pydata1, pyeye1, pyface1 -- **C# (.NET WASM)**: dotnet-data1 -- **Java (TeaVM -> JS)**: java-data1 -- **R (webR -> WASM)**: rdata1, rcomm1 -- browser-only (webR spawns a classic Worker, unsupported by Deno's + har1, math1, nfc, sensor1, speech-recognition, video1 +- **JavaScript**: js-data1 (esbuild bundle of the AWS SDK v3 twin), js-math1 (dependency-free, committed as-is) +- **Dart -> JS**: dart-comm1, dart-data1, dart-math1 +- **Kotlin -> WASM (WasmGC)**: kotlin-data1, kotlin-math1 -- compiled by the Kotlin Gradle plugin's `wasmJs` target; + each module is a WasmGC binary (browser GC manages the Kotlin heap), so it needs a WasmGC-capable engine +- **Python (Pyodide)**: pydata1, pyeye1, pyface1, pymath1 +- **C# (.NET WASM)**: dotnet-data1, dotnet-math1 +- **Java (TeaVM -> JS)**: java-data1, java-math1 -- both built by the single root `pom.xml` (one compilation, one + teavm-maven-plugin execution per module) +- **R (webR -> WASM)**: rdata1, rcomm1, rmath1 -- browser-only (webR spawns a classic Worker, unsupported by Deno's ws-web-runner). Their JS shims are linted by the `js` env; `MISE_ENV=r` supplies webR + the vendoring build tasks. -- **Zig -> WASM**: zig-data1, zig-except1 (C++ wasm-exception-handling demo) +- **Zig -> WASM**: zig-data1, zig-except1 (C++ wasm-exception-handling demo), zig-math1 + +The math1 modules are one family, driven by a storage exchange: a fake agent (the shared helper in +`et-ws-test-server`, embedding the canonical committed input at its `data/math1-input.json`) injects the +input JSON into ws-server storage and broadcasts a `math1-input` pointer over the hub; each module reads the +input, runs the same FedAvg simulation (`+ - * /` on f64 only), and stores its global model to +`math1-output.json` in its own bucket, which the test harness reads and verifies against the expected +weights -- proving bit-identical float math across every guest runtime. Beyond the browser twins the family +also covers wasi-math1 (a WASI Preview 2 component under `et-ws-wasi-runner`'s wasmtime) and the `math1.py` +module under `et-ws-pyo3-runner` (native CPython) -- the two non-browser executors. + - **Python (componentize-py -> WASI Preview 2 component)**: wasi-graphics-info -- runs in `et-ws-wasi-runner` rather than the browser. The WIT world the component implements is at `services/ws-wasi-runner/wit/world.wit` and is mirrored under the module's own `wit/`. diff --git a/Cargo.lock b/Cargo.lock index 951d3b9a..4a798bce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4553,6 +4553,25 @@ dependencies = [ "web-sys", ] +[[package]] +name = "et-ws-math1" +version = "0.1.0" +dependencies = [ + "et-rest-client", + "et-web", + "et-ws-wasm-agent", + "futures-util", + "js-sys", + "serde", + "serde_json", + "tracing", + "tracing-wasm", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", +] + [[package]] name = "et-ws-nfc" version = "0.1.0" @@ -4758,6 +4777,7 @@ dependencies = [ "retry", "serde_json", "tempfile", + "thiserror 2.0.19", "tokio", "tokio-tungstenite", "tracing-actix-web", @@ -4803,6 +4823,18 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "et-ws-wasi-math1" +version = "0.1.0" +dependencies = [ + "et-path", + "fs-err", + "minicov", + "serde", + "serde_json", + "wit-bindgen", +] + [[package]] name = "et-ws-wasi-runner" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 44084de9..8af8251b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "services/ws-modules/geolocation", "services/ws-modules/graphics-info", "services/ws-modules/har1", + "services/ws-modules/math1", "services/ws-modules/nfc", "services/ws-modules/pic-viewer", "services/ws-modules/sensor1", @@ -30,6 +31,7 @@ members = [ "services/ws-modules/video1", "services/ws-modules/wasi-comm1", "services/ws-modules/wasi-data1", + "services/ws-modules/wasi-math1", "services/modules", "services/storage", "services/websockify", diff --git a/config/ast-grep/rules/doc-summary-ends-with-period.yaml b/config/ast-grep/rules/doc-summary-ends-with-period.yaml index 468add0a..125c6522 100644 --- a/config/ast-grep/rules/doc-summary-ends-with-period.yaml +++ b/config/ast-grep/rules/doc-summary-ends-with-period.yaml @@ -43,13 +43,18 @@ files: - services/websockify/src/lib.rs - services/websockify/tests/relay.rs - services/ws-modules/except1/src/lib.rs + - services/ws-modules/math1/src/lib.rs - services/ws-modules/pic-viewer/build.rs - services/ws-modules/pic-viewer/src/lib.rs - services/ws-modules/pic-viewer/tests/parse.rs - services/ws-modules/pic-viewer/tests/show_image.rs - services/ws-modules/wasi-comm1/src/lib.rs - services/ws-modules/wasi-data1/src/lib.rs + - services/ws-modules/wasi-math1/src/coverage.rs + - services/ws-modules/wasi-math1/src/lib.rs - services/ws-pyo3-runner/tests/modules.rs + - services/ws-test-server/src/lib.rs + - services/ws-test-server/src/math1.rs - services/ws-server/src/config.rs - services/ws-server/src/lib.rs - services/ws-server/src/main.rs diff --git a/config/ast-grep/rules/no-relative-path-literal.yaml b/config/ast-grep/rules/no-relative-path-literal.yaml index cd9d863c..8a02927c 100644 --- a/config/ast-grep/rules/no-relative-path-literal.yaml +++ b/config/ast-grep/rules/no-relative-path-literal.yaml @@ -17,5 +17,8 @@ ignores: - libs/path/src/lib.rs # --module-dir defaults to the current directory. - utilities/cli/src/cli.rs + # include_str! resolves its path against this source file at compile time, not the process cwd. + # The one `..` up to the crate's committed data/ dir is therefore stable by construction. + - services/ws-test-server/src/math1.rs # Asserts the relative paths that the generators above produce. - utilities/cli/tests/scenario_generation.rs diff --git a/config/coverage.toml b/config/coverage.toml index a55165b9..dbf33e74 100644 --- a/config/coverage.toml +++ b/config/coverage.toml @@ -16,4 +16,5 @@ pydata1 = ["services/ws-modules/pydata1/pydata1", "pydata1", "**/pydata1"] pydemo1 = ["services/ws-modules/pydemo1/pydemo1", "pydemo1", "**/pydemo1"] pyeye1 = ["services/ws-modules/pyeye1/pyeye1", "pyeye1", "**/pyeye1"] pyface1 = ["services/ws-modules/pyface1/pyface1", "pyface1", "**/pyface1"] +pymath1 = ["services/ws-modules/pymath1/pymath1", "pymath1", "**/pymath1"] pyspeech1 = ["services/ws-modules/pyspeech1/pyspeech1", "pyspeech1", "**/pyspeech1"] diff --git a/config/ls-lint.yaml b/config/ls-lint.yaml index a6afb9dc..f0949985 100644 --- a/config/ls-lint.yaml +++ b/config/ls-lint.yaml @@ -48,34 +48,44 @@ ignore: - services/ws-modules/comm1/pkg - services/ws-modules/dart-comm1/pkg - services/ws-modules/dart-data1/pkg + - services/ws-modules/dart-math1/pkg - services/ws-modules/data1/pkg - services/ws-modules/dotnet-data1/pkg + - services/ws-modules/dotnet-math1/pkg - services/ws-modules/except1/pkg - services/ws-modules/face-detection/pkg - services/ws-modules/geolocation/pkg - services/ws-modules/graphics-info/pkg - services/ws-modules/har1/pkg - services/ws-modules/java-data1/pkg + - services/ws-modules/java-math1/pkg - services/ws-modules/js-data1/pkg + - services/ws-modules/js-math1/pkg - services/ws-modules/kotlin-data1/pkg + - services/ws-modules/kotlin-math1/pkg + - services/ws-modules/math1/pkg - services/ws-modules/nfc/pkg - services/ws-modules/pic-viewer/pkg - services/ws-modules/pydata1/pkg - services/ws-modules/pydemo1/pkg - services/ws-modules/pyeye1/pkg - services/ws-modules/pyface1/pkg + - services/ws-modules/pymath1/pkg - services/ws-modules/pyspeech1/pkg - services/ws-modules/pywasm1/pkg - services/ws-modules/rcomm1/pkg - services/ws-modules/rdata1/pkg + - services/ws-modules/rmath1/pkg - services/ws-modules/sensor1/pkg - services/ws-modules/speech-recognition/pkg - services/ws-modules/video1/pkg - services/ws-modules/wasi-comm1/pkg - services/ws-modules/wasi-data1/pkg + - services/ws-modules/wasi-math1/pkg - services/ws-modules/wasi-graphics-info/pkg - services/ws-modules/zig-data1/pkg - services/ws-modules/zig-except1/pkg + - services/ws-modules/zig-math1/pkg - services/ws-wasm-agent/pkg # Python venvs plus the pytest and bytecode caches each Python module's test run leaves behind. - .venv @@ -83,6 +93,7 @@ ignore: - .ruff_cache - services/ws-modules/pydata1/pydata1/__pycache__ - services/ws-modules/pydemo1/pydemo1/__pycache__ + - services/ws-modules/pymath1/pymath1/__pycache__ - services/ws-modules/pydemo1/tests/__pycache__ - services/ws-modules/pyeye1/pyeye1/__pycache__ - services/ws-modules/pyeye1/tests/__pycache__ @@ -106,6 +117,8 @@ ignore: - services/ws-modules/pyeye1/.pytest_cache - services/ws-modules/pyface1/.venv - services/ws-modules/pyface1/.pytest_cache + - services/ws-modules/pymath1/.venv + - services/ws-modules/pymath1/.pytest_cache - services/ws-modules/pyspeech1/.venv - services/ws-modules/pyspeech1/.pytest_cache - services/ws-modules/pywasm1/.venv @@ -116,11 +129,16 @@ ignore: - services/ws-modules/dotnet-data1/obj - services/ws-modules/dotnet-data1/bin/Debug - services/ws-modules/dotnet-data1/bin/Release + - services/ws-modules/dotnet-math1/obj + - services/ws-modules/dotnet-math1/bin/Debug + - services/ws-modules/dotnet-math1/bin/Release # Zig build output and caches, plus the codegen templates dir whose name carries a literal `.in` suffix. - services/ws-modules/zig-data1/.zig-cache - services/ws-modules/zig-data1/zig-out - services/ws-modules/zig-except1/.zig-cache - services/ws-modules/zig-except1/zig-out + - services/ws-modules/zig-math1/.zig-cache + - services/ws-modules/zig-math1/zig-out - utilities/int-gen/src/zig.in # Gradle build + cache trees, plus the one tracked source dir whose name ls-lint would reject. # wasmJsMain stays tracked but its camelCase name is fixed by Kotlin Multiplatform's @@ -129,3 +147,7 @@ ignore: - services/ws-modules/kotlin-data1/.kotlin - services/ws-modules/kotlin-data1/build - services/ws-modules/kotlin-data1/src/wasmJsMain + - services/ws-modules/kotlin-math1/.gradle + - services/ws-modules/kotlin-math1/.kotlin + - services/ws-modules/kotlin-math1/build + - services/ws-modules/kotlin-math1/src/wasmJsMain diff --git a/config/pyrefly.toml b/config/pyrefly.toml index 6f101823..317089c7 100644 --- a/config/pyrefly.toml +++ b/config/pyrefly.toml @@ -24,6 +24,7 @@ search-path = [ "../services/ws-modules/pydemo1", "../services/ws-modules/pyeye1", "../services/ws-modules/pyface1", + "../services/ws-modules/pymath1", "../services/ws-modules/pyspeech1", "../services/ws-pyo3-runner/python", ] diff --git a/config/semgrep/prefer-yaml-toml.yaml b/config/semgrep/prefer-yaml-toml.yaml index a25f1fcd..dcc57f14 100644 --- a/config/semgrep/prefer-yaml-toml.yaml +++ b/config/semgrep/prefer-yaml-toml.yaml @@ -18,6 +18,7 @@ rules: - "oxlintrc.jsonc" # oxlint: only JSON / JSONC supported - "oxfmtrc.jsonc" # oxfmt: only JSON / JSONC supported - "tsconfig.json" # TypeScript/tsgolint: mandates the JSON `tsconfig.json` name (oxlint --type-aware) + - "math1-input.json" # math1 wire payload: served to modules verbatim, parsed by their JSON parsers pattern-regex: \A message: >- Prefer YAML or TOML over JSON/JSONC/JSONL for config. diff --git a/core.slnx b/core.slnx index 8fb8b43d..bdf4a32d 100644 --- a/core.slnx +++ b/core.slnx @@ -2,5 +2,6 @@ + diff --git a/pom.xml b/pom.xml index e0400652..69771f06 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,27 @@ ${project.basedir}/services/ws-modules/java-data1/src/main/java + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + add-java-math1-source + generate-sources + + add-source + + + + ${project.basedir}/services/ws-modules/java-math1/src/main/java + + + + + org.apache.maven.plugins maven-compiler-plugin @@ -102,6 +123,22 @@ true + + build-math1-js + + compile + + package + + au.edu.curtin.et.Math1 + ${project.basedir}/services/ws-modules/java-math1/pkg + classes.js + JAVASCRIPT + ES2015 + ADVANCED + true + + diff --git a/pubspec.yaml b/pubspec.yaml index e8d01a25..ca30a858 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -14,3 +14,4 @@ workspace: - generated/dart-rest - services/ws-modules/dart-comm1 - services/ws-modules/dart-data1 + - services/ws-modules/dart-math1 diff --git a/pyproject.toml b/pyproject.toml index 365e7e26..43540060 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ members = [ "services/ws-modules/pydemo1", "services/ws-modules/pyeye1", "services/ws-modules/pyface1", + "services/ws-modules/pymath1", "services/ws-modules/pyspeech1", "services/ws-modules/wasi-graphics-info", ] diff --git a/services/ws-modules/dart-math1/lib/dart_math1.dart b/services/ws-modules/dart-math1/lib/dart_math1.dart new file mode 100644 index 00000000..cd90d2cb --- /dev/null +++ b/services/ws-modules/dart-math1/lib/dart_math1.dart @@ -0,0 +1,212 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:js_interop'; +import 'dart:typed_data'; + +import 'package:dio/dio.dart'; +import 'package:et_rest/export.dart'; + +// JS interop declarations for et_ws_wasm_agent +@JS() +extension type WsClientConfig._(JSObject _) implements JSObject { + external factory WsClientConfig(String serverUrl); +} + +@JS() +extension type WsClient._(JSObject _) implements JSObject { + external factory WsClient(WsClientConfig config); + external void connect(); + external void disconnect(); + // ignore: non_constant_identifier_names + external String get_state(); + // ignore: non_constant_identifier_names + external String get_agent_id(); + // ignore: non_constant_identifier_names + external void set_on_message(JSFunction callback); +} + +// JS interop for browser globals +@JS('window.location.protocol') +external String get locationProtocol; + +@JS('window.location.host') +external String get locationHost; + +@JS('document.getElementById') +external JSObject? getElementById(String id); + +@JS() +extension type _TextArea._(JSObject _) implements JSObject { + external String get value; + external set value(String v); +} + +void appendOutput(String msg) { + final el = getElementById('module-output'); + if (el != null) { + final ta = el as _TextArea; + ta.value = ta.value.isEmpty ? msg : '${ta.value}\n$msg'; + } +} + +void log(String msg) { + appendOutput('[dart-math1] $msg'); +} + +String get wsUrl { + final proto = locationProtocol == 'https:' ? 'wss:' : 'ws:'; + return '$proto//$locationHost/ws'; +} + +Future sleep(int ms) { + final c = Completer(); + Timer(Duration(milliseconds: ms), c.complete); + return c.future; +} + +Future waitFor(String what, T? Function() ready) async { + for (var i = 0; i < 100; i++) { + final value = ready(); + if (value != null) return value; + await sleep(100); + } + throw Exception('Timeout waiting for $what'); +} + +/// The broadcast pointer naming the storage bucket + filename of the input JSON. +({String bucket, String filename})? inputPointer; + +void captureInputPointer(String frame) { + try { + final msg = jsonDecode(frame); + if (msg is Map && + msg['type'] == 'math1-input' && + msg['bucket'] is String && + msg['filename'] is String) { + inputPointer = ( + bucket: msg['bucket'] as String, + filename: msg['filename'] as String, + ); + } + } on FormatException { + // Not JSON -- some other relayed frame; ignore. + } +} + +/// Runs the FedAvg simulation on the fetched input and returns the final global [weight, bias]. +/// +/// Only + - * / on doubles, in a fixed evaluation order, so the result is bit-identical to the +/// other math1 language twins. +List fedAvg(Map input) { + final clients = (input['clients'] as List) + .map( + (samples) => (samples as List) + .map( + (sample) => (sample as List).map((v) => (v as num).toDouble()).toList(), + ) + .toList(), + ) + .toList(); + final rounds = input['rounds'] as int; + final epochs = input['epochs'] as int; + final learningRate = (input['learning_rate'] as num).toDouble(); + + var weight = 0.0; + var bias = 0.0; + var totalSamples = 0.0; + for (final samples in clients) { + totalSamples += samples.length.toDouble(); + } + for (var round = 0; round < rounds; round++) { + var mergedWeight = 0.0; + var mergedBias = 0.0; + for (final samples in clients) { + final count = samples.length.toDouble(); + var clientWeight = weight; + var clientBias = bias; + for (var epoch = 0; epoch < epochs; epoch++) { + var gradWeight = 0.0; + var gradBias = 0.0; + for (final sample in samples) { + final residual = clientWeight * sample[0] + clientBias - sample[1]; + gradWeight += residual * sample[0]; + gradBias += residual; + } + clientWeight -= learningRate * (2.0 * gradWeight / count); + clientBias -= learningRate * (2.0 * gradBias / count); + } + mergedWeight += clientWeight * count; + mergedBias += clientBias * count; + } + weight = mergedWeight / totalSamples; + bias = mergedBias / totalSamples; + } + return [weight, bias]; +} + +Future run() async { + log('entered run()'); + + final client = WsClient(WsClientConfig(wsUrl)); + client.set_on_message( + ((JSAny? frame) { + final text = frame.dartify(); + if (text is String) captureInputPointer(text); + }).toJS, + ); + client.connect(); + await waitFor('WebSocket connection', () => client.get_state() == 'connected' ? true : null); + final agentId = await waitFor('agent_id', () { + final id = client.get_agent_id(); + return id.isEmpty ? null : id; + }); + log('connected as $agentId'); + + log('waiting for the math1-input pointer broadcast'); + final pointer = await waitFor('math1-input pointer', () => inputPointer); + + // Empty baseUrl -> requests resolve against the page origin. Every browser module is served from + // the same ws-server that owns its storage (mirrors the Rust math1 module). + final rest = RestClient(Dio(BaseOptions(baseUrl: ''))); + + log('reading input from /storage/${pointer.bucket}/${pointer.filename}'); + final inputBytes = await rest.storage.getFile( + agentId: pointer.bucket, + filename: pointer.filename, + ); + final input = jsonDecode(utf8.decode(inputBytes)) as Map; + + final clientCount = (input['clients'] as List).length; + log('running FedAvg - $clientCount clients x ${input['rounds']} rounds x ${input['epochs']} local epochs'); + final model = fedAvg(input); + final weight = model[0]; + final bias = model[1]; + log('global model weight=$weight bias=$bias'); + + final output = jsonEncode({'module': 'dart-math1', 'weight': weight, 'bias': bias}); + await rest.storage.putFile( + agentId: agentId, + filename: 'math1-output.json', + body: Uint8List.fromList(utf8.encode(output)), + ); + log('stored the global model to /storage/$agentId/math1-output.json'); + + await sleep(2000); + client.disconnect(); + log('workflow complete'); +} + +@JS('dartMath1Run') +external set _dartMath1Run(JSFunction f); + +void main() { + _dartMath1Run = (() { + return (() async { + try { + await run(); + } catch (e, st) { + throw '$e\n$st'.toJS; + } + }().toJS); + }.toJS); +} diff --git a/services/ws-modules/dart-math1/pkg/.gitignore b/services/ws-modules/dart-math1/pkg/.gitignore new file mode 100644 index 00000000..50ebc2be --- /dev/null +++ b/services/ws-modules/dart-math1/pkg/.gitignore @@ -0,0 +1 @@ +*_compiled* diff --git a/services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js b/services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js new file mode 100644 index 00000000..205c1de2 --- /dev/null +++ b/services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js @@ -0,0 +1,37 @@ +// et_ws_dart_math1.js -- ES module shim for dart-math1 + +export default async function init() { + await new Promise((resolve, reject) => { + const s = document.createElement("script"); + s.src = new URL("et_ws_dart_math1_compiled.js", import.meta.url).href; + s.onload = resolve; + s.onerror = reject; + document.head.appendChild(s); + }); +} + +export async function run() { + if (typeof globalThis.dartMath1Run !== "function") { + throw new Error("dart-math1: not initialized"); + } + // Dart @JS() interop resolves against globalThis, so expose the wasm-agent + // classes there for the duration of the call. The FedAvg kernel itself is + // pure local computation and needs no further globals. + const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + await wasmAgent.default(); + const { WsClient, WsClientConfig } = wasmAgent; + globalThis.WsClient = WsClient; + globalThis.WsClientConfig = WsClientConfig; + try { + const result = globalThis.dartMath1Run(); + console.log("dart-math1 dartMath1Run returned:", result, typeof result); + await result; + } catch (e) { + console.error("dart-math1 raw error:", e, "boxed:", e?.error); + const msg = e?.error?.toString?.() ?? e?.message ?? String(e); + throw new Error(msg, { cause: e }); + } finally { + delete globalThis.WsClient; + delete globalThis.WsClientConfig; + } +} diff --git a/services/ws-modules/dart-math1/pkg/package.json b/services/ws-modules/dart-math1/pkg/package.json new file mode 100644 index 00000000..93a26baa --- /dev/null +++ b/services/ws-modules/dart-math1/pkg/package.json @@ -0,0 +1,8 @@ +{ + "name": "et-ws-dart-math1", + "type": "module", + "description": "dart math1", + "version": "0.1.0", + "license": "Apache-2.0 OR MIT", + "main": "et_ws_dart_math1.js" +} diff --git a/services/ws-modules/dart-math1/pubspec.yaml b/services/ws-modules/dart-math1/pubspec.yaml new file mode 100644 index 00000000..abe69d33 --- /dev/null +++ b/services/ws-modules/dart-math1/pubspec.yaml @@ -0,0 +1,18 @@ +name: et_dart_math1 +description: dart-math1 FedAvg module +version: 0.1.0 +repository: https://github.com/edge-toolkit/core +publish_to: none + +environment: + sdk: ^3.11.5 + +resolution: workspace + +dependencies: + dio: ^5.9.2 + et_rest: + path: ../../../generated/dart-rest + +dev_dependencies: + lints: ^6.0.0 diff --git a/services/ws-modules/dotnet-math1/.gitignore b/services/ws-modules/dotnet-math1/.gitignore new file mode 100644 index 00000000..9ef46866 --- /dev/null +++ b/services/ws-modules/dotnet-math1/.gitignore @@ -0,0 +1,8 @@ +bin/ +obj/ +*.wasm +*.dll +*.pdb +dotnet.js +dotnet.runtime.js +dotnet.native.js diff --git a/services/ws-modules/dotnet-math1/Program.cs b/services/ws-modules/dotnet-math1/Program.cs new file mode 100644 index 00000000..5697f914 --- /dev/null +++ b/services/ws-modules/dotnet-math1/Program.cs @@ -0,0 +1,126 @@ +using System; +using System.Runtime.InteropServices.JavaScript; +using System.Text.Json; +using System.Threading.Tasks; + +namespace EtWsModules; + +// JS-imported host functions provided by the shim +partial class Host +{ + [JSImport("wsConnect", "dotnet-math1")] internal static partial void WsConnect(string url); + [JSImport("wsDisconnect", "dotnet-math1")] internal static partial void WsDisconnect(); + [JSImport("wsGetState", "dotnet-math1")] internal static partial string WsGetState(); + [JSImport("wsGetAgentId", "dotnet-math1")] internal static partial string WsGetAgentId(); + [JSImport("hasInput", "dotnet-math1")] internal static partial bool HasInput(); + [JSImport("fetchInputJson", "dotnet-math1")] internal static partial Task FetchInputJsonAsync(); + [JSImport("putOutput", "dotnet-math1")] + internal static partial Task PutOutputAsync(string module, double weight, double bias); + [JSImport("log", "dotnet-math1")] internal static partial void Log(string msg); + [JSImport("setStatus", "dotnet-math1")] internal static partial void SetStatus(string msg); + // skipcq: CS-A1000 -- [JSImport] marshals a string; System.Uri is not a supported JS-interop return type. + [JSImport("getWsUrl", "dotnet-math1")] internal static partial string GetWsUrl(); + [JSImport("sleep", "dotnet-math1")] internal static partial Task SleepAsync(int ms); +} + +public partial class DotnetMath1 +{ + [JSExport] + public static async Task RunAsync() + { + Status("entered Run()"); + + Host.WsConnect(Host.GetWsUrl()); + await WaitForAsync("WebSocket connection", () => Host.WsGetState() == "connected"); + await WaitForAsync("agent_id", () => !string.IsNullOrEmpty(Host.WsGetAgentId())); + Status($"connected as {Host.WsGetAgentId()}"); + + Status("waiting for the math1-input pointer broadcast"); + await WaitForAsync("math1-input pointer", Host.HasInput); + var inputJson = await Host.FetchInputJsonAsync(); + using var input = JsonDocument.Parse(inputJson); + var root = input.RootElement; + + var clients = root.GetProperty("clients"); + var rounds = root.GetProperty("rounds").GetInt32(); + var epochs = root.GetProperty("epochs").GetInt32(); + var learningRate = root.GetProperty("learning_rate").GetDouble(); + Status($"running FedAvg - {clients.GetArrayLength()} clients x {rounds} rounds x {epochs} local epochs"); + + var (weight, bias) = FedAvg(clients, rounds, epochs, learningRate); + Status($"global model weight={weight:R} bias={bias:R}"); + + await Host.PutOutputAsync("dotnet-math1", weight, bias); + Status("stored the global model to math1-output.json"); + + await Host.SleepAsync(2000); + Host.WsDisconnect(); + Status("workflow complete"); + } + + // Runs the FedAvg simulation on the fetched input and returns the final global (weight, bias). + // Only + - * / on double in a fixed evaluation order, so the result is bit-identical to the + // other math1 language twins. + private static (double Weight, double Bias) FedAvg( + JsonElement clients, int rounds, int epochs, double learningRate) + { + var weight = 0.0; + var bias = 0.0; + var totalSamples = 0.0; + foreach (var samples in clients.EnumerateArray()) + { + totalSamples += samples.GetArrayLength(); + } + for (int round = 0; round < rounds; round++) + { + var mergedWeight = 0.0; + var mergedBias = 0.0; + foreach (var samples in clients.EnumerateArray()) + { + double count = samples.GetArrayLength(); + var clientWeight = weight; + var clientBias = bias; + for (int epoch = 0; epoch < epochs; epoch++) + { + var gradWeight = 0.0; + var gradBias = 0.0; + foreach (var sample in samples.EnumerateArray()) + { + var feature = sample[0].GetDouble(); + var target = sample[1].GetDouble(); + var residual = clientWeight * feature + clientBias - target; + gradWeight += residual * feature; + gradBias += residual; + } + clientWeight -= learningRate * (2.0 * gradWeight / count); + clientBias -= learningRate * (2.0 * gradBias / count); + } + mergedWeight += clientWeight * count; + mergedBias += clientBias * count; + } + weight = mergedWeight / totalSamples; + bias = mergedBias / totalSamples; + } + return (weight, bias); + } + + private static void Status(string msg) + { + var line = $"[dotnet-math1] {msg}"; + Host.Log(line); + Host.SetStatus(line); + } + + private static async Task WaitForAsync(string what, Func ready) + { + for (int i = 0; i < 100; i++) + { + if (ready()) + { + return; + } + await Host.SleepAsync(100); + } + throw new TimeoutException($"Timeout waiting for {what}"); + } +} diff --git a/services/ws-modules/dotnet-math1/dotnet-math1.csproj b/services/ws-modules/dotnet-math1/dotnet-math1.csproj new file mode 100644 index 00000000..4201f97a --- /dev/null +++ b/services/ws-modules/dotnet-math1/dotnet-math1.csproj @@ -0,0 +1,11 @@ + + + net10.0 + browser-wasm + false + enable + Library + true + CA1416 + + diff --git a/services/ws-modules/dotnet-math1/pkg/.gitignore b/services/ws-modules/dotnet-math1/pkg/.gitignore new file mode 100644 index 00000000..4db69560 --- /dev/null +++ b/services/ws-modules/dotnet-math1/pkg/.gitignore @@ -0,0 +1,4 @@ +*.wasm +*.dat +dotnet.js +dotnet.*.js diff --git a/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js new file mode 100644 index 00000000..b2d6c4d2 --- /dev/null +++ b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js @@ -0,0 +1,77 @@ +// et_ws_dotnet_math1.js — .NET WASM shim for dotnet-math1 +// Interface: default(), run() +// +// Storage-driven FedAvg: the shim owns the browser I/O -- the WebSocket (including capturing the +// broadcast math1-input pointer), fetching the input JSON from storage, and storing the output -- +// while the C# guest parses the JSON with System.Text.Json and owns the kernel. + +let exports = null; + +// skipcq: JS-0833 -- committed .NET WASM ES-module shim; DeepSource's script-mode parse is a false positive +export default async function init() { + const { dotnet } = await import(new URL("dotnet.js", import.meta.url).href); + const { getAssemblyExports, setModuleImports } = await dotnet.create(); + + let ws = null, + wsState = "disconnected", + agentId = "", + inputPointer = null; + + setModuleImports("dotnet-math1", { + wsConnect: (url) => { + ws = new WebSocket(url); + wsState = "connecting"; + ws.onopen = () => { + wsState = "connected"; + ws.send(JSON.stringify({ type: "et-connect" })); + }; + ws.onmessage = (e) => { + try { + const msg = JSON.parse(e.data); + if (msg.type === "et-connect-ack" && msg.agent_id) agentId = msg.agent_id; + if (msg.type === "math1-input" && msg.bucket && msg.filename) inputPointer = msg; + } catch {} + }; + ws.onclose = ws.onerror = () => { + wsState = "disconnected"; + }; + }, + wsDisconnect: () => { + ws?.close(); + wsState = "disconnected"; + }, + wsGetState: () => wsState, + wsGetAgentId: () => agentId ?? "", + hasInput: () => inputPointer !== null, + fetchInputJson: () => + fetch(`/storage/${inputPointer.bucket}/${inputPointer.filename}`).then((r) => { + if (!r.ok) throw new Error(`input GET failed: ${r.status}`); + return r.text(); + }), + putOutput: (module, weight, bias) => { + const body = JSON.stringify({ module, weight, bias }); + return fetch(`/storage/${agentId}/math1-output.json`, { method: "PUT", body }).then((r) => { + if (!r.ok) throw new Error(`output PUT failed: ${r.status}`); + }); + }, + log: (msg) => { + console.log(msg); + appendOutput(msg); + }, + setStatus: (msg) => appendOutput(msg), + getWsUrl: () => `${location.protocol === "https:" ? "wss:" : "ws:"}//${location.host}/ws`, + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), + }); + + exports = await getAssemblyExports("dotnet-math1"); +} + +export async function run() { + if (!exports) throw new Error("dotnet-math1: not initialized"); + await exports.EtWsModules.DotnetMath1.RunAsync(); +} + +function appendOutput(msg) { + const el = document.getElementById("module-output"); + if (el) el.value = (el.value ? el.value + "\n" : "") + msg; +} diff --git a/services/ws-modules/dotnet-math1/pkg/package.json b/services/ws-modules/dotnet-math1/pkg/package.json new file mode 100644 index 00000000..5612b887 --- /dev/null +++ b/services/ws-modules/dotnet-math1/pkg/package.json @@ -0,0 +1,8 @@ +{ + "name": "et-ws-dotnet-math1", + "type": "module", + "description": "dotnet math1", + "version": "0.1.0", + "license": "Apache-2.0 OR MIT", + "main": "et_ws_dotnet_math1.js" +} diff --git a/services/ws-modules/java-math1/pkg/.gitignore b/services/ws-modules/java-math1/pkg/.gitignore new file mode 100644 index 00000000..b6e03704 --- /dev/null +++ b/services/ws-modules/java-math1/pkg/.gitignore @@ -0,0 +1 @@ +classes.js diff --git a/services/ws-modules/java-math1/pkg/et_ws_java_math1.js b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js new file mode 100644 index 00000000..9e6105a8 --- /dev/null +++ b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js @@ -0,0 +1,96 @@ +// et_ws_java_math1.js — TeaVM JS shim for java-math1 +// Interface: default(), run() +// +// Storage-driven FedAvg: the shim owns the browser I/O -- the WebSocket (including capturing the +// broadcast math1-input pointer), fetching the input JSON from storage, and storing the output -- +// while the TeaVM guest owns the kernel, reading the parsed input through the typed host accessors +// below (the guest carries no JSON parser of its own; the accessors keep it dependency-free). + +let javaRun = null; + +export default async function init() { + let ws = null, + wsState = "disconnected", + agentId = "", + inputPointer = null, + input = null; + + // TeaVM @JSBody calls reference `host` as a global + globalThis.host = { + wsConnect: (url) => { + ws = new WebSocket(url); + wsState = "connecting"; + ws.onopen = () => { + wsState = "connected"; + ws.send(JSON.stringify({ type: "et-connect" })); + }; + ws.onmessage = (e) => { + try { + const msg = JSON.parse(e.data); + if (msg.type === "et-connect-ack" && msg.agent_id) agentId = msg.agent_id; + if (msg.type === "math1-input" && msg.bucket && msg.filename) inputPointer = msg; + } catch {} + }; + ws.onclose = ws.onerror = () => { + wsState = "disconnected"; + }; + }, + wsDisconnect: () => { + ws?.close(); + wsState = "disconnected"; + }, + wsGetState: () => wsState, + wsGetAgentId: () => agentId ?? "", + hasInput: () => inputPointer !== null, + loadInput: () => + fetch(`/storage/${inputPointer.bucket}/${inputPointer.filename}`).then((r) => { + if (!r.ok) throw new Error(`input GET failed: ${r.status}`); + return r.json().then((json) => { + input = json; + }); + }), + inputClientCount: () => input.clients.length, + inputSampleCount: (client) => input.clients[client].length, + inputFeature: (client, index) => input.clients[client][index][0], + inputTarget: (client, index) => input.clients[client][index][1], + inputRounds: () => input.rounds, + inputEpochs: () => input.epochs, + inputLearningRate: () => input.learning_rate, + inputDescribe: () => `${input.clients.length} clients x ${input.rounds} rounds x ${input.epochs} local epochs`, + putOutput: (module, weight, bias) => { + const body = JSON.stringify({ module, weight, bias }); + return fetch(`/storage/${agentId}/math1-output.json`, { method: "PUT", body }).then((r) => { + if (!r.ok) throw new Error(`output PUT failed: ${r.status}`); + }); + }, + log: (msg) => { + console.log(msg); + appendOutput(msg); + }, + setStatus: (msg) => appendOutput(msg), + getWsUrl: () => `${location.protocol === "https:" ? "wss:" : "ws:"}//${location.host}/ws`, + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), + }; + + const jsUrl = new URL("classes.js", import.meta.url).href; + + await new Promise((resolve, reject) => { + const s = document.createElement("script"); + s.src = jsUrl; + s.onload = resolve; + s.onerror = reject; + document.head.appendChild(s); + }); + + javaRun = globalThis.run; +} + +export async function run() { + if (!javaRun) throw new Error("java-math1: not initialized"); + await javaRun(); +} + +function appendOutput(msg) { + const el = document.getElementById("module-output"); + if (el) el.value = (el.value ? el.value + "\n" : "") + msg; +} diff --git a/services/ws-modules/java-math1/pkg/package.json b/services/ws-modules/java-math1/pkg/package.json new file mode 100644 index 00000000..326d1583 --- /dev/null +++ b/services/ws-modules/java-math1/pkg/package.json @@ -0,0 +1,8 @@ +{ + "name": "et-ws-java-math1", + "type": "module", + "description": "java math1", + "version": "0.1.0", + "license": "Apache-2.0 OR MIT", + "main": "et_ws_java_math1.js" +} diff --git a/services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java b/services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java new file mode 100644 index 00000000..d0847d71 --- /dev/null +++ b/services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java @@ -0,0 +1,209 @@ +package au.edu.curtin.et; + +import org.teavm.jso.JSBody; +import org.teavm.jso.JSExport; +import org.teavm.jso.JSObject; +import org.teavm.jso.core.JSPromise; +import org.teavm.jso.function.JSConsumer; + +public final class Math1 { + + @JSBody(params = {"msg"}, script = "host.log(msg);") + static native void log(String msg); + + @JSBody(params = {"msg"}, script = "host.setStatus(msg);") + static native void setStatus(String msg); + + @JSBody(script = "return host.getWsUrl();") + static native String getWsUrl(); + + @JSBody(params = {"url"}, script = "host.wsConnect(url);") + static native void wsConnect(String url); + + @JSBody(script = "host.wsDisconnect();") + static native void wsDisconnect(); + + @JSBody(script = "return host.wsGetState();") + static native String wsGetState(); + + @JSBody(script = "return host.wsGetAgentId();") + static native String wsGetAgentId(); + + @JSBody(params = {"ms"}, script = "return host.sleep(ms);") + static native JSPromise sleep(int ms); + + @JSBody(script = "return host.hasInput();") + static native boolean hasInput(); + + @JSBody(script = "return host.loadInput();") + static native JSPromise loadInput(); + + @JSBody(script = "return host.inputClientCount();") + static native int inputClientCount(); + + @JSBody(params = {"client"}, script = "return host.inputSampleCount(client);") + static native int inputSampleCount(int client); + + @JSBody(params = {"client", "index"}, script = "return host.inputFeature(client, index);") + static native double inputFeature(int client, int index); + + @JSBody(params = {"client", "index"}, script = "return host.inputTarget(client, index);") + static native double inputTarget(int client, int index); + + @JSBody(script = "return host.inputRounds();") + static native int inputRounds(); + + @JSBody(script = "return host.inputEpochs();") + static native int inputEpochs(); + + @JSBody(script = "return host.inputLearningRate();") + static native double inputLearningRate(); + + @JSBody(script = "return host.inputDescribe();") + static native String inputDescribe(); + + @JSBody(params = {"module", "weight", "bias"}, script = "return host.putOutput(module, weight, bias);") + static native JSPromise putOutput(String module, double weight, double bias); + + @JSBody(params = {"msg"}, script = "return new Error(msg);") + static native JSObject jsError(String msg); + + private Math1() {} + + @JSExport + public static JSPromise run() { + return new JSPromise<>((resolve, reject) -> runAsync(resolve, reject)); + } + + private static void status(String msg) { + log("[java-math1] " + msg); + setStatus("[java-math1] " + msg); + } + + private static void runAsync(JSConsumer resolve, JSConsumer reject) { + status("entered run()"); + wsConnect(getWsUrl()); + waitForConnected(0, resolve, reject); + } + + private static void waitForConnected(int attempt, JSConsumer resolve, JSConsumer reject) { + if (attempt >= 100) { + reject.accept(jsError("Timeout waiting for WebSocket connection")); + return; + } + if ("connected".equals(wsGetState())) { + waitForAgentId(0, resolve, reject); + return; + } + sleep(100).then(v -> { + waitForConnected(attempt + 1, resolve, reject); + return null; + }); + } + + private static void waitForAgentId(int attempt, JSConsumer resolve, JSConsumer reject) { + if (attempt >= 100) { + reject.accept(jsError("Timeout waiting for agent_id")); + return; + } + String agentId = wsGetAgentId(); + if (agentId != null && !agentId.isEmpty()) { + status("connected as " + agentId); + status("waiting for the math1-input pointer broadcast"); + waitForInput(0, resolve, reject); + return; + } + sleep(100).then(v -> { + waitForAgentId(attempt + 1, resolve, reject); + return null; + }); + } + + private static void waitForInput(int attempt, JSConsumer resolve, JSConsumer reject) { + if (attempt >= 100) { + reject.accept(jsError("Timeout waiting for the math1-input pointer")); + return; + } + if (hasInput()) { + loadInput().then(v -> { + computeAndStore(resolve, reject); + return null; + }); + return; + } + sleep(100).then(v -> { + waitForInput(attempt + 1, resolve, reject); + return null; + }); + } + + /** + * Runs the FedAvg simulation over the host-parsed input and returns the global {weight, bias}. + * + *

Only + - * / on double in a fixed evaluation order, so the result is bit-identical to the + * other math1 language twins. The input is read through the shim's typed accessors because the + * TeaVM guest carries no JSON parser of its own. + */ + private static double[] fedAvg() { + int rounds = inputRounds(); + int epochs = inputEpochs(); + double learningRate = inputLearningRate(); + int clientCount = inputClientCount(); + double weight = 0.0; + double bias = 0.0; + double totalSamples = 0.0; + for (int client = 0; client < clientCount; client++) { + totalSamples += inputSampleCount(client); + } + for (int round = 0; round < rounds; round++) { + double mergedWeight = 0.0; + double mergedBias = 0.0; + for (int client = 0; client < clientCount; client++) { + int sampleCount = inputSampleCount(client); + double count = sampleCount; + double clientWeight = weight; + double clientBias = bias; + for (int epoch = 0; epoch < epochs; epoch++) { + double gradWeight = 0.0; + double gradBias = 0.0; + for (int index = 0; index < sampleCount; index++) { + double feature = inputFeature(client, index); + double target = inputTarget(client, index); + double residual = clientWeight * feature + clientBias - target; + gradWeight += residual * feature; + gradBias += residual; + } + clientWeight -= learningRate * (2.0 * gradWeight / count); + clientBias -= learningRate * (2.0 * gradBias / count); + } + mergedWeight += clientWeight * count; + mergedBias += clientBias * count; + } + weight = mergedWeight / totalSamples; + bias = mergedBias / totalSamples; + } + return new double[] {weight, bias}; + } + + private static void computeAndStore(JSConsumer resolve, JSConsumer reject) { + status("running FedAvg - " + inputDescribe()); + double[] model = fedAvg(); + double weight = model[0]; + double bias = model[1]; + status("global model weight=" + weight + " bias=" + bias); + putOutput("java-math1", weight, bias).then(v -> { + status("stored the global model to math1-output.json"); + finish(resolve); + return null; + }); + } + + private static void finish(JSConsumer resolve) { + sleep(2000).then(v -> { + wsDisconnect(); + status("workflow complete"); + resolve.accept(null); + return null; + }); + } +} diff --git a/services/ws-modules/js-math1/pkg/et_ws_js_math1.js b/services/ws-modules/js-math1/pkg/et_ws_js_math1.js new file mode 100644 index 00000000..0751ec3a --- /dev/null +++ b/services/ws-modules/js-math1/pkg/et_ws_js_math1.js @@ -0,0 +1,114 @@ +// et_ws_js_math1.js -- math1 twin in plain JavaScript (no bundler, no dependencies). +// +// Storage-driven FedAvg: waits for the broadcast math1-input pointer, reads the input JSON +// (client datasets + hyperparameters) from ws-server storage, runs the kernel -- only + - * / on +// IEEE-754 doubles, bit-identical to the other math1 twins -- and stores the global model to +// math1-output.json in its own bucket, where the test harness reads and verifies it. + +function appendOutput(msg) { + const el = document.getElementById("module-output"); + if (el) { + el.value = el.value ? `${el.value}\n${msg}` : msg; + } +} + +function log(msg) { + const line = `[js-math1] ${msg}`; + console.log(line); + appendOutput(line); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitFor(what, ready) { + for (let attempt = 0; attempt < 100; attempt++) { + const value = ready(); + if (value) { + return value; + } + await sleep(100); + } + throw new Error(`Timeout waiting for ${what}`); +} + +function fedAvg(input) { + let weight = 0.0; + let bias = 0.0; + let totalSamples = 0.0; + for (const samples of input.clients) { + totalSamples += samples.length; + } + for (let round = 0; round < input.rounds; round++) { + let mergedWeight = 0.0; + let mergedBias = 0.0; + for (const samples of input.clients) { + const count = samples.length; + let clientWeight = weight; + let clientBias = bias; + for (let epoch = 0; epoch < input.epochs; epoch++) { + let gradWeight = 0.0; + let gradBias = 0.0; + for (const [feature, target] of samples) { + const residual = clientWeight * feature + clientBias - target; + gradWeight += residual * feature; + gradBias += residual; + } + clientWeight -= input.learning_rate * ((2.0 * gradWeight) / count); + clientBias -= input.learning_rate * ((2.0 * gradBias) / count); + } + mergedWeight += clientWeight * count; + mergedBias += clientBias * count; + } + weight = mergedWeight / totalSamples; + bias = mergedBias / totalSamples; + } + return [weight, bias]; +} + +export default async function init() {} + +export async function run() { + log("entered run()"); + + const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + await wasmAgent.default(); + const proto = window.location.protocol === "https:" ? "wss:" : "ws:"; + const wsUrl = `${proto}//${window.location.host}/ws`; + const client = new wasmAgent.WsClient(new wasmAgent.WsClientConfig(wsUrl)); + + let pointer = null; + client.set_on_message((frame) => { + if (typeof frame !== "string") return; + try { + const msg = JSON.parse(frame); + if (msg.type === "math1-input" && msg.bucket && msg.filename) pointer = msg; + } catch {} + }); + + client.connect(); + await waitFor("WebSocket connection", () => client.get_state() === "connected"); + const agentId = await waitFor("agent_id", () => client.get_agent_id()); + log(`connected as ${agentId}`); + + log("waiting for the math1-input pointer broadcast"); + const input_ptr = await waitFor("math1-input pointer", () => pointer); + log(`reading input from /storage/${input_ptr.bucket}/${input_ptr.filename}`); + const inputResponse = await fetch(`/storage/${input_ptr.bucket}/${input_ptr.filename}`); + if (!inputResponse.ok) throw new Error(`input GET failed: ${inputResponse.status}`); + const input = await inputResponse.json(); + + log(`running FedAvg - ${input.clients.length} clients x ${input.rounds} rounds x ${input.epochs} local epochs`); + const [weight, bias] = fedAvg(input); + log(`global model weight=${weight} bias=${bias}`); + + const output = JSON.stringify({ module: "js-math1", weight, bias }); + const putResponse = await fetch(`/storage/${agentId}/math1-output.json`, { method: "PUT", body: output }); + if (!putResponse.ok) throw new Error(`output PUT failed: ${putResponse.status}`); + log(`stored the global model to /storage/${agentId}/math1-output.json`); + + await sleep(2000); + client.disconnect(); + log("workflow complete"); +} diff --git a/services/ws-modules/js-math1/pkg/package.json b/services/ws-modules/js-math1/pkg/package.json new file mode 100644 index 00000000..7a48871c --- /dev/null +++ b/services/ws-modules/js-math1/pkg/package.json @@ -0,0 +1,8 @@ +{ + "name": "et-ws-js-math1", + "type": "module", + "description": "js math1", + "version": "0.1.0", + "license": "Apache-2.0 OR MIT", + "main": "et_ws_js_math1.js" +} diff --git a/services/ws-modules/kotlin-math1/build.gradle.kts b/services/ws-modules/kotlin-math1/build.gradle.kts new file mode 100644 index 00000000..2c5ee990 --- /dev/null +++ b/services/ws-modules/kotlin-math1/build.gradle.kts @@ -0,0 +1,31 @@ +plugins { + kotlin("multiplatform") version "2.4.10" +} + +repositories { + mavenCentral() +} + +kotlin { + wasmJs { + // The compiled loader + wasm artifact names derive from this; the committed pkg/ shim imports + // `et_ws_kotlin_math1_compiled.mjs`, mirroring kotlin-data1's naming. + outputModuleName.set("et_ws_kotlin_math1_compiled") + browser() + binaries.executable() + } + compilerOptions { + allWarningsAsErrors.set(true) + } +} + +// Copy the linked production executable (WasmGC module + its ES-module loader glue) into pkg/, where the +// modules service serves it next to the committed package.json and shim. `preserve` keeps the committed files. +tasks.register("pkgDist") { + dependsOn("wasmJsProductionExecutableCompileSync") + from(layout.buildDirectory.dir("compileSync/wasmJs/main/productionExecutable/kotlin")) + into(layout.projectDirectory.dir("pkg")) + preserve { + include("package.json", "et_ws_kotlin_math1.js", ".gitignore") + } +} diff --git a/services/ws-modules/kotlin-math1/pkg/.gitignore b/services/ws-modules/kotlin-math1/pkg/.gitignore new file mode 100644 index 00000000..50ebc2be --- /dev/null +++ b/services/ws-modules/kotlin-math1/pkg/.gitignore @@ -0,0 +1 @@ +*_compiled* diff --git a/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js b/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js new file mode 100644 index 00000000..1e36f0e0 --- /dev/null +++ b/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js @@ -0,0 +1,89 @@ +// et_ws_kotlin_math1.js -- ES module shim for kotlin-math1 (Kotlin/Wasm, WasmGC) +// Interface: default(), run() +// +// Storage-driven FedAvg: the shim owns the browser I/O -- the WebSocket (including capturing the +// broadcast math1-input pointer), fetching the input JSON from storage, and storing the output -- +// while the WasmGC guest owns the kernel, reading the parsed input through the typed host +// accessors below (WasmGC has no JSON parser of its own; the accessors keep the guest +// dependency-free). + +export default async function init() { + let ws = null, + wsState = "disconnected", + agentId = "", + inputPointer = null, + input = null; + + // The Kotlin js() interop bridges reference `host` as a global + globalThis.host = { + wsConnect: (url) => { + ws = new WebSocket(url); + wsState = "connecting"; + ws.onopen = () => { + wsState = "connected"; + ws.send(JSON.stringify({ type: "et-connect" })); + }; + ws.onmessage = (e) => { + try { + const msg = JSON.parse(e.data); + if (msg.type === "et-connect-ack" && msg.agent_id) agentId = msg.agent_id; + if (msg.type === "math1-input" && msg.bucket && msg.filename) inputPointer = msg; + } catch {} + }; + ws.onclose = ws.onerror = () => { + wsState = "disconnected"; + }; + }, + wsDisconnect: () => { + ws?.close(); + wsState = "disconnected"; + }, + wsGetState: () => wsState, + wsGetAgentId: () => agentId ?? "", + hasInput: () => inputPointer !== null, + loadInput: () => + fetch(`/storage/${inputPointer.bucket}/${inputPointer.filename}`).then((r) => { + if (!r.ok) throw new Error(`input GET failed: ${r.status}`); + return r.json().then((json) => { + input = json; + }); + }), + inputClientCount: () => input.clients.length, + inputSampleCount: (client) => input.clients[client].length, + inputFeature: (client, index) => input.clients[client][index][0], + inputTarget: (client, index) => input.clients[client][index][1], + inputRounds: () => input.rounds, + inputEpochs: () => input.epochs, + inputLearningRate: () => input.learning_rate, + inputDescribe: () => `${input.clients.length} clients x ${input.rounds} rounds x ${input.epochs} local epochs`, + putOutput: (module, weight, bias) => { + const body = JSON.stringify({ module, weight, bias }); + return fetch(`/storage/${agentId}/math1-output.json`, { method: "PUT", body }).then((r) => { + if (!r.ok) throw new Error(`output PUT failed: ${r.status}`); + }); + }, + log: (msg) => { + console.log(msg); + appendOutput(msg); + }, + setStatus: (msg) => appendOutput(msg), + getWsUrl: () => `${location.protocol === "https:" ? "wss:" : "ws:"}//${location.host}/ws`, + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), + }; + + // Instantiates the WasmGC module (browser path: instantiateStreaming over fetch) and runs Kotlin main(), + // which installs globalThis.kotlinMath1Run. + await import(new URL("et_ws_kotlin_math1_compiled.mjs", import.meta.url).href); +} + +export async function run() { + if (typeof globalThis.kotlinMath1Run !== "function") { + throw new Error("kotlin-math1: not initialized"); + } + await globalThis.kotlinMath1Run(); +} + +function appendOutput(msg) { + const el = document.getElementById("module-output"); + if (el) el.value = (el.value ? el.value + "\n" : "") + msg; +} diff --git a/services/ws-modules/kotlin-math1/pkg/package.json b/services/ws-modules/kotlin-math1/pkg/package.json new file mode 100644 index 00000000..eed2ca29 --- /dev/null +++ b/services/ws-modules/kotlin-math1/pkg/package.json @@ -0,0 +1,8 @@ +{ + "name": "et-ws-kotlin-math1", + "type": "module", + "description": "kotlin math1", + "version": "0.1.0", + "license": "Apache-2.0 OR MIT", + "main": "et_ws_kotlin_math1.js" +} diff --git a/services/ws-modules/kotlin-math1/settings.gradle.kts b/services/ws-modules/kotlin-math1/settings.gradle.kts new file mode 100644 index 00000000..3c3663cf --- /dev/null +++ b/services/ws-modules/kotlin-math1/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "et-ws-kotlin-math1" diff --git a/services/ws-modules/kotlin-math1/src/wasmJsMain/kotlin/Math1.kt b/services/ws-modules/kotlin-math1/src/wasmJsMain/kotlin/Math1.kt new file mode 100644 index 00000000..2af35bef --- /dev/null +++ b/services/ws-modules/kotlin-math1/src/wasmJsMain/kotlin/Math1.kt @@ -0,0 +1,143 @@ +// Kotlin 2.x still stability-gates the whole wasmJs interop surface (js(), JsAny, Promise) behind this +// opt-in; every declaration in this file is an interop bridge, so the opt-in is file-scoped. +@file:OptIn(ExperimentalWasmJsInterop::class) + +package au.edu.curtin.et + +import kotlin.js.ExperimentalWasmJsInterop +import kotlin.js.Promise + +// The pkg/ shim installs a `host` global carrying the browser-side WebSocket, storage, and parsed +// math1-input accessors; each js() body below is a single-expression bridge. The kernel reads the +// input through the typed accessors because the WasmGC guest carries no JSON parser of its own. +private fun hostLog(msg: String): Unit = js("host.log(msg)") + +private fun hostSetStatus(msg: String): Unit = js("host.setStatus(msg)") + +private fun hostGetWsUrl(): String = js("host.getWsUrl()") + +private fun hostWsConnect(url: String): Unit = js("host.wsConnect(url)") + +private fun hostWsDisconnect(): Unit = js("host.wsDisconnect()") + +private fun hostWsGetState(): String = js("host.wsGetState()") + +private fun hostWsGetAgentId(): String = js("host.wsGetAgentId()") + +private fun hostSleep(ms: Int): Promise = js("host.sleep(ms)") + +private fun hostHasInput(): Boolean = js("host.hasInput()") + +private fun hostLoadInput(): Promise = js("host.loadInput()") + +private fun hostInputClientCount(): Int = js("host.inputClientCount()") + +private fun hostInputSampleCount(client: Int): Int = js("host.inputSampleCount(client)") + +private fun hostInputFeature(client: Int, index: Int): Double = js("host.inputFeature(client, index)") + +private fun hostInputTarget(client: Int, index: Int): Double = js("host.inputTarget(client, index)") + +private fun hostInputRounds(): Int = js("host.inputRounds()") + +private fun hostInputEpochs(): Int = js("host.inputEpochs()") + +private fun hostInputLearningRate(): Double = js("host.inputLearningRate()") + +private fun hostInputDescribe(): String = js("host.inputDescribe()") + +private fun hostPutOutput(module: String, weight: Double, bias: Double): Promise = + js("host.putOutput(module, weight, bias)") + +private fun resolvedPromise(): Promise = js("Promise.resolve(null)") + +private fun rejectedPromise(msg: String): Promise = js("Promise.reject(new Error(msg))") + +private fun installRun(f: () -> Promise): Unit = js("globalThis.kotlinMath1Run = f") + +private fun status(msg: String) { + hostLog("[kotlin-math1] $msg") + hostSetStatus("[kotlin-math1] $msg") +} + +private fun waitUntil(what: String, attempt: Int = 0, ready: () -> Boolean): Promise = when { + ready() -> resolvedPromise() + attempt >= 100 -> rejectedPromise("Timeout waiting for $what") + else -> hostSleep(100).then { waitUntil(what, attempt + 1, ready) } +} + +// Runs the FedAvg simulation over the host-parsed input and returns the final global (weight, bias). +// Only + - * / on Double in a fixed evaluation order, so the result is bit-identical to the other +// math1 language twins. +private fun fedAvg(): Pair { + val rounds = hostInputRounds() + val epochs = hostInputEpochs() + val learningRate = hostInputLearningRate() + val clientCount = hostInputClientCount() + var weight = 0.0 + var bias = 0.0 + var totalSamples = 0.0 + for (client in 0 until clientCount) totalSamples += hostInputSampleCount(client).toDouble() + repeat(rounds) { + var mergedWeight = 0.0 + var mergedBias = 0.0 + for (client in 0 until clientCount) { + val sampleCount = hostInputSampleCount(client) + val count = sampleCount.toDouble() + var clientWeight = weight + var clientBias = bias + repeat(epochs) { + var gradWeight = 0.0 + var gradBias = 0.0 + for (index in 0 until sampleCount) { + val feature = hostInputFeature(client, index) + val target = hostInputTarget(client, index) + val residual = clientWeight * feature + clientBias - target + gradWeight += residual * feature + gradBias += residual + } + clientWeight -= learningRate * (2.0 * gradWeight / count) + clientBias -= learningRate * (2.0 * gradBias / count) + } + mergedWeight += clientWeight * count + mergedBias += clientBias * count + } + weight = mergedWeight / totalSamples + bias = mergedBias / totalSamples + } + return weight to bias +} + +private fun computeAndStore(): Promise { + status("running FedAvg - ${hostInputDescribe()}") + val (weight, bias) = fedAvg() + status("global model weight=$weight bias=$bias") + return hostPutOutput("kotlin-math1", weight, bias).then { + status("stored the global model to math1-output.json") + null + } +} + +private fun runWorkflow(): Promise { + status("entered run()") + hostWsConnect(hostGetWsUrl()) + return waitUntil("WebSocket connection") { hostWsGetState() == "connected" } + .then { waitUntil("agent_id") { hostWsGetAgentId().isNotEmpty() } } + .then { + status("connected as ${hostWsGetAgentId()}") + status("waiting for the math1-input pointer broadcast") + waitUntil("math1-input pointer") { hostHasInput() } + } + .then { hostLoadInput() } + .then { computeAndStore() } + .then { hostSleep(2000) } + .then { + hostWsDisconnect() + status("workflow complete") + null + } +} + +fun main() { + installRun { runWorkflow() } +} diff --git a/services/ws-modules/math1/Cargo.toml b/services/ws-modules/math1/Cargo.toml new file mode 100644 index 00000000..86aa0601 --- /dev/null +++ b/services/ws-modules/math1/Cargo.toml @@ -0,0 +1,32 @@ +[package] +description = "math 1" +name = "et-ws-math1" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +crate-type = ["cdylib", "rlib"] +doctest = false +test = false + +[dependencies] +et-rest-client.workspace = true +et-web.workspace = true +et-ws-wasm-agent.workspace = true +futures-util.workspace = true +js-sys.workspace = true +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true +tracing-wasm.workspace = true +wasm-bindgen.workspace = true +wasm-bindgen-futures.workspace = true +web-sys = { workspace = true, features = ["Window", "console"] } + +[dev-dependencies] +wasm-bindgen-test.workspace = true + +[lints] +workspace = true diff --git a/services/ws-modules/math1/src/lib.rs b/services/ws-modules/math1/src/lib.rs new file mode 100644 index 00000000..13e7556c --- /dev/null +++ b/services/ws-modules/math1/src/lib.rs @@ -0,0 +1,255 @@ +//! math1: federated-averaging (`FedAvg`) demo in a browser WASM module. +//! +//! Storage-driven twin family: a fake agent injects the canonical input JSON (client datasets + +//! hyperparameters) into ws-server storage and broadcasts a `math1-input` pointer over the hub. +//! This module waits for that pointer, reads the input from storage, runs the `FedAvg` kernel -- +//! rounds of local full-batch gradient-descent epochs per client merged with a sample-count-weighted +//! average, only `+ - * /` on f64 so the result is bit-identical on every IEEE-754 runtime -- and +//! stores the global model to `math1-output.json` in its own bucket, where the test harness reads +//! and verifies it. The math1 twins in the other guest languages run the same protocol and kernel. + +#![expect( + clippy::float_arithmetic, + clippy::future_not_send, + clippy::single_call_fn, + reason = "browser WASM module: JsFuture is !Send; the FedAvg kernel is float math; helpers are single-use" +)] + +use std::cell::RefCell; +use std::rc::Rc; + +use et_web::JsResultExt as _; +use et_ws_wasm_agent::{WsClient, WsClientConfig, append_to_textarea}; +use futures_util::StreamExt as _; +use js_sys::{Promise, Reflect}; +use serde::Deserialize; +use tracing::info; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +/// The canonical input: per-client (feature, target) samples plus the training hyperparameters. +#[derive(Deserialize)] +struct Math1Input { + clients: Vec>, + rounds: u32, + epochs: u32, + learning_rate: f64, +} + +/// The broadcast pointer naming the storage bucket + filename the input JSON was injected at. +#[derive(Clone, Deserialize)] +struct InputPointer { + bucket: String, + filename: String, +} + +#[wasm_bindgen(start)] +pub fn init() { + tracing_wasm::set_as_global_default(); + info!("math1 FedAvg module initialized"); +} + +/// Sample count as f64, accumulated additively to avoid an integer-to-float cast. +fn sample_count(samples: &[(f64, f64)]) -> f64 { + samples.iter().fold(0.0_f64, |count, _| count + 1.0) +} + +/// Runs the `FedAvg` simulation on `input` and returns the final global (weight, bias). +fn fed_avg(input: &Math1Input) -> (f64, f64) { + let mut weight = 0.0_f64; + let mut bias = 0.0_f64; + let total_samples: f64 = input + .clients + .iter() + .fold(0.0_f64, |acc, samples| acc + sample_count(samples)); + for _ in 0_u32..input.rounds { + let mut merged_weight = 0.0_f64; + let mut merged_bias = 0.0_f64; + for samples in &input.clients { + let count = sample_count(samples); + let mut client_weight = weight; + let mut client_bias = bias; + for _ in 0_u32..input.epochs { + let mut grad_weight = 0.0_f64; + let mut grad_bias = 0.0_f64; + for &(feature, target) in samples { + let residual = client_weight * feature + client_bias - target; + grad_weight += residual * feature; + grad_bias += residual; + } + client_weight -= input.learning_rate * (2.0 * grad_weight / count); + client_bias -= input.learning_rate * (2.0 * grad_bias / count); + } + merged_weight += client_weight * count; + merged_bias += client_bias * count; + } + weight = merged_weight / total_samples; + bias = merged_bias / total_samples; + } + (weight, bias) +} + +#[wasm_bindgen] +pub async fn run() -> Result<(), JsValue> { + let msg = "math1: entered run()"; + log(msg); + set_module_status(msg)?; + + let ws_url = websocket_url()?; + let mut client = WsClient::new(WsClientConfig::new(ws_url)); + + // Capture the math1-input pointer broadcast; every other frame is ignored. + let pointer_slot: Rc>> = Rc::new(RefCell::new(None)); + #[expect( + clippy::as_conversions, + reason = "wasm-bindgen's Closure::wrap takes a `Box`; the cast is required to unsize the Box" + )] + let on_message = Closure::wrap(Box::new({ + let pointer_slot = Rc::clone(&pointer_slot); + move |value: JsValue| { + let Some(data) = value.as_string() else { + return; + }; + let Ok(json) = serde_json::from_str::(&data) else { + return; + }; + if json.get("type").and_then(serde_json::Value::as_str) == Some("math1-input") + && let Ok(pointer) = serde_json::from_value::(json) + { + *pointer_slot.borrow_mut() = Some(pointer); + } + } + }) as Box); + client.set_on_message(on_message.as_ref().clone()); + + client.connect()?; + wait_for_connected(&client).await?; + let agent_id = wait_for_agent_id(&client).await?; + let msg = format!("math1: connected as {agent_id}"); + log(&msg); + set_module_status(&msg)?; + + let msg = "math1: waiting for the math1-input pointer broadcast"; + log(msg); + set_module_status(msg)?; + let pointer = wait_for_pointer(&pointer_slot).await?; + + let msg = format!( + "math1: reading input from /storage/{}/{}", + pointer.bucket, pointer.filename + ); + log(&msg); + set_module_status(&msg)?; + // The typed REST client runs against the page origin -- every browser module is served from the + // same ws-server that owns its storage, so an empty base URL (relative paths) is what we want. + let rest = et_rest_client::Client::new(""); + let response = rest + .get_file(&pointer.bucket, &pointer.filename) + .await + .js_context("input GET failed")?; + let input_bytes = collect_stream(response.into_inner()).await?; + let input: Math1Input = serde_json::from_slice(&input_bytes).js_context("input JSON parse failed")?; + + let msg = format!( + "math1: running FedAvg - {} clients x {} rounds x {} local epochs", + input.clients.len(), + input.rounds, + input.epochs + ); + log(&msg); + set_module_status(&msg)?; + let (weight, bias) = fed_avg(&input); + let msg = format!("math1: global model weight={weight} bias={bias}"); + log(&msg); + set_module_status(&msg)?; + + let output = serde_json::json!({ "module": "math1", "weight": weight, "bias": bias }).to_string(); + let _put_response = rest + .put_file(&agent_id, "math1-output.json", output) + .await + .js_context("output PUT failed")?; + let msg = format!("math1: stored the global model to /storage/{agent_id}/math1-output.json"); + log(&msg); + set_module_status(&msg)?; + + sleep_ms(2000).await?; + client.disconnect(); + let msg = "math1: workflow complete"; + log(msg); + set_module_status(msg)?; + Ok(()) +} + +async fn collect_stream(mut stream: et_rest_client::ByteStream) -> Result, JsValue> { + let mut buf = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.js_context("stream chunk")?; + buf.extend_from_slice(&chunk); + } + Ok(buf) +} + +fn log(message: &str) { + let line = format!("[math1] {message}"); + web_sys::console::log_1(&JsValue::from_str(&line)); +} + +fn set_module_status(message: &str) -> Result<(), JsValue> { + append_to_textarea("module-output", message) +} + +async fn wait_for_connected(client: &WsClient) -> Result<(), JsValue> { + for _ in 0_u32..100 { + if client.get_state() == "connected" { + return Ok(()); + } + sleep_ms(100).await?; + } + Err(JsValue::from_str("Timed out waiting for websocket connection")) +} + +async fn wait_for_agent_id(client: &WsClient) -> Result { + for _ in 0_u32..100 { + let agent_id = client.get_agent_id(); + if !agent_id.is_empty() { + return Ok(agent_id); + } + sleep_ms(100).await?; + } + Err(JsValue::from_str("Timed out waiting for assigned agent_id")) +} + +async fn wait_for_pointer(slot: &Rc>>) -> Result { + for _ in 0_u32..100 { + if let Some(pointer) = slot.borrow().clone() { + return Ok(pointer); + } + sleep_ms(100).await?; + } + Err(JsValue::from_str("Timed out waiting for the math1-input pointer")) +} + +async fn sleep_ms(duration_ms: i32) -> Result<(), JsValue> { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let promise = Promise::new(&mut |resolve, _reject| { + let callback = Closure::once_into_js(move || { + et_web::ignore(resolve.call0(&JsValue::NULL)); + }); + let _id: Result = + window.set_timeout_with_callback_and_timeout_and_arguments_0(callback.unchecked_ref(), duration_ms); + }); + JsFuture::from(promise).await.map(|_| ()) +} + +fn websocket_url() -> Result { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let location = Reflect::get(window.as_ref(), &JsValue::from_str("location"))?; + let protocol = Reflect::get(&location, &JsValue::from_str("protocol"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.protocol is unavailable"))?; + let host = Reflect::get(&location, &JsValue::from_str("host"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.host is unavailable"))?; + let ws_protocol = if protocol == "https:" { "wss:" } else { "ws:" }; + Ok(format!("{ws_protocol}//{host}/ws")) +} diff --git a/services/ws-modules/pymath1/pkg/.gitignore b/services/ws-modules/pymath1/pkg/.gitignore new file mode 100644 index 00000000..d415af97 --- /dev/null +++ b/services/ws-modules/pymath1/pkg/.gitignore @@ -0,0 +1,2 @@ +*.whl +package.json diff --git a/services/ws-modules/pymath1/pkg/et_ws_pymath1.js b/services/ws-modules/pymath1/pkg/et_ws_pymath1.js new file mode 100644 index 00000000..bad9f7f6 --- /dev/null +++ b/services/ws-modules/pymath1/pkg/et_ws_pymath1.js @@ -0,0 +1,110 @@ +// et_ws_pymath1.js — Pyodide-based Python module shim +// Interface: default() (init), run() +// +// Storage-driven FedAvg: the shim owns the browser I/O (WebSocket, the math1-input pointer +// broadcast, storage GET/PUT); the wheel owns the kernel. The shim fetches the input JSON the +// pointer names, hands the raw text to Python, and stores the returned output JSON in this +// agent's bucket for the test harness to verify. + +const PYODIDE_BASE_PATH = "/modules/pyodide/"; + +let pyodide = null; +let pyMod = null; + +function loadPyodideScript() { + return new Promise((resolve, reject) => { + if (globalThis.loadPyodide) return resolve(); + const s = document.createElement("script"); + s.src = `${PYODIDE_BASE_PATH}pyodide.js`; + s.onload = resolve; + s.onerror = reject; + document.head.appendChild(s); + }); +} + +export default async function init() { + await loadPyodideScript(); + // The full Pyodide distribution is served at /modules/pyodide/, so the runtime resolves from this + // same origin — no CDN dependency. pymath1 has no PyPI deps: its FedAvg kernel is stdlib-only, so + // the only wheel to load is its own, served next to this shim. + pyodide = await globalThis.loadPyodide({ indexURL: PYODIDE_BASE_PATH }); + + const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); + const wheelName = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; + const bytes = new Uint8Array(await fetch(new URL(wheelName, import.meta.url)).then((r) => r.arrayBuffer())); + pyodide.FS.writeFile(`/tmp/${wheelName}`, bytes); + pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${wheelName}")`); + + // Start Pyodide coverage before importing so import-time lines count (no-op unless the runner set the gate). + if (globalThis.__etPyCov) await globalThis.__etPyCov.start(pyodide, "pymath1"); + + const pymath1 = pyodide.pyimport("pymath1"); + pyMod = { + run: pymath1.run, + }; +} + +export async function run() { + if (!pyMod) throw new Error("pymath1: not initialized"); + + const loc = typeof location !== "undefined" ? location : null; + const wsProto = loc?.protocol === "https:" ? "wss:" : "ws:"; + const wsHost = loc?.host ?? "localhost:8080"; + const wsUrl = globalThis.__ET_WS_URL || `${wsProto}//${wsHost}/ws`; + + const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + await wasmAgent.default(); + const { WsClient, WsClientConfig } = wasmAgent; + const client = new WsClient(new WsClientConfig(wsUrl)); + + let pointer = null; + client.set_on_message((frame) => { + if (typeof frame !== "string") return; + try { + const msg = JSON.parse(frame); + if (msg.type === "math1-input" && msg.bucket && msg.filename) pointer = msg; + } catch {} + }); + + client.connect(); + + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + const waitFor = async (what, ready) => { + for (let i = 0; i < 100; i++) { + const value = ready(); + if (value) return value; + await sleep(100); + } + throw new Error(`Timeout waiting for ${what}`); + }; + + const log = (msg) => { + console.log(msg); + const el = document.getElementById("module-output"); + if (el) el.value = (el.value ? el.value + "\n" : "") + msg; + }; + + try { + await waitFor("WebSocket connection", () => client.get_state() === "connected"); + const agentId = await waitFor("agent_id", () => client.get_agent_id()); + + log(`[pymath1] waiting for the math1-input pointer broadcast`); + const inputPtr = await waitFor("math1-input pointer", () => pointer); + log(`[pymath1] reading input from /storage/${inputPtr.bucket}/${inputPtr.filename}`); + const inputResponse = await fetch(`/storage/${inputPtr.bucket}/${inputPtr.filename}`); + if (!inputResponse.ok) throw new Error(`input GET failed: ${inputResponse.status}`); + const inputJson = await inputResponse.text(); + + const output = pyMod.run(agentId, inputJson, pyodide.toPy(log)); + const putResponse = await fetch(`/storage/${agentId}/math1-output.json`, { method: "PUT", body: output }); + if (!putResponse.ok) throw new Error(`output PUT failed: ${putResponse.status}`); + log(`[pymath1] stored the global model to /storage/${agentId}/math1-output.json`); + await sleep(2000); + } catch (err) { + log(`pymath1 run failed: ${String(err)}`); + throw err; + } finally { + if (globalThis.__etPyCov) await globalThis.__etPyCov.stop(pyodide, "pymath1"); + client.disconnect(); + } +} diff --git a/services/ws-modules/pymath1/pymath1/__init__.py b/services/ws-modules/pymath1/pymath1/__init__.py new file mode 100644 index 00000000..d198742a --- /dev/null +++ b/services/ws-modules/pymath1/pymath1/__init__.py @@ -0,0 +1,58 @@ +"""math1 twin in Python (Pyodide): a storage-driven FedAvg simulation. + +The JS shim hands this module the raw input JSON it fetched from ws-server storage (client +datasets + hyperparameters, injected by the test harness's fake agent). fed_avg() runs rounds of +local full-batch gradient-descent epochs per client and merges the local models with a +sample-count-weighted average. Only + - * / on floats (no math-module calls), so the result is +bit-identical to the other math1 language twins. run() returns the output JSON the shim stores to +this agent's bucket for the harness to verify. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable + + +def fed_avg(clients: list[list[list[float]]], rounds: int, epochs: int, learning_rate: float) -> tuple[float, float]: + """Run the FedAvg simulation and return the final global (weight, bias).""" + weight = 0.0 + bias = 0.0 + total_samples = 0.0 + for samples in clients: + total_samples += float(len(samples)) + for _ in range(rounds): + merged_weight = 0.0 + merged_bias = 0.0 + for samples in clients: + count = float(len(samples)) + client_weight = weight + client_bias = bias + for _ in range(epochs): + grad_weight = 0.0 + grad_bias = 0.0 + for sample in samples: + residual = client_weight * sample[0] + client_bias - sample[1] + grad_weight += residual * sample[0] + grad_bias += residual + client_weight -= learning_rate * (2.0 * grad_weight / count) + client_bias -= learning_rate * (2.0 * grad_bias / count) + merged_weight += client_weight * count + merged_bias += client_bias * count + weight = merged_weight / total_samples + bias = merged_bias / total_samples + return weight, bias + + +def run(agent_id: str, input_json: str, log: Callable[[str], None]) -> str: + """Run FedAvg on the fetched input and return the output JSON for the shim to store.""" + log(f"[pymath1] connected as {agent_id}") + params = json.loads(input_json) + clients = params["clients"] + rounds = params["rounds"] + epochs = params["epochs"] + learning_rate = params["learning_rate"] + log(f"[pymath1] running FedAvg - {len(clients)} clients x {rounds} rounds x {epochs} local epochs") + weight, bias = fed_avg(clients, rounds, epochs, learning_rate) + log(f"[pymath1] global model weight={weight!r} bias={bias!r}") + return json.dumps({"module": "pymath1", "weight": weight, "bias": bias}) diff --git a/services/ws-modules/pymath1/pyproject.toml b/services/ws-modules/pymath1/pyproject.toml new file mode 100644 index 00000000..708609ab --- /dev/null +++ b/services/ws-modules/pymath1/pyproject.toml @@ -0,0 +1,19 @@ +[project] +dependencies = [] +description = "Python math 1" +license = "Apache-2.0 OR MIT" +name = "et-ws-pymath1" +requires-python = ">=3.10" +version = "0.1.0" + +[build-system] +build-backend = "uv_build" +requires = ["uv_build==0.11.8"] + +[tool.uv.build-backend] +module-name = "pymath1" +module-root = "" + +# Browser-side module dependency: the Pyodide runtime served at /modules/pyodide/. +[tool.ws-module.dependencies] +pyodide = "*" diff --git a/services/ws-modules/rmath1/pkg/et_ws_rmath1.js b/services/ws-modules/rmath1/pkg/et_ws_rmath1.js new file mode 100644 index 00000000..bce02f13 --- /dev/null +++ b/services/ws-modules/rmath1/pkg/et_ws_rmath1.js @@ -0,0 +1,59 @@ +// et_ws_rmath1.js -- bootstrap only. The control loop is module.R's run(); this shim boots webR, sets up the +// agent WebSocket transport that the R code drives via webr::eval_js, and hands control to run(). +// +// webR cannot open the agent WebSocket itself, so the transport (the shared et-ws-wasm-agent WsClient) lives +// here and is exposed on globalThis.__etAgent for R to drive; the broadcast math1-input pointer is captured +// onto __etAgent.input for R to poll. Everything else -- sequencing, the storage reads/writes (httr2 over the +// /websockify relay), the FedAvg kernel -- happens in module.R. webR is vendored under pkg/webr/ (see +// build-ws-rmath1-module) and served at the path below. + +const WEBR_BASE_URL = "/modules/et-ws-rmath1/webr/"; +const R_SOURCE_URL = "/modules/et-ws-rmath1/module.R"; + +let webR = null; + +export default async function init() { + const { WebR } = await import(`${WEBR_BASE_URL}webr.mjs`); + webR = new WebR({ baseUrl: WEBR_BASE_URL }); + await webR.init(); + // Cache-bust so edits to module.R are picked up on reload. + const rSource = await (await fetch(`${R_SOURCE_URL}?v=${Date.now()}`)).text(); + await webR.evalRVoid(rSource); +} + +export async function run() { + if (!webR) throw new Error("rmath1: not initialized"); + await setupAgent(); + // Hand control to R -- run() is the control loop. + await webR.evalRVoid("run()"); +} + +// Expose the agent WebSocket to R on globalThis.__etAgent. R drives it (connect state, agent_id, disconnect) +// via webr::eval_js; the shim only creates and connects it, and captures the math1-input pointer broadcast. +async function setupAgent() { + const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + await wasmAgent.default(); + const { WsClient, WsClientConfig } = wasmAgent; + const loc = typeof location !== "undefined" ? location : null; + const wsProto = loc?.protocol === "https:" ? "wss:" : "ws:"; + const wsHost = loc?.host ?? "localhost:8080"; + const wsUrl = globalThis.__ET_WS_URL || `${wsProto}//${wsHost}/ws`; + const client = new WsClient(new WsClientConfig(wsUrl)); + globalThis.__etAgent = { + client, + input: null, + log(msg) { + console.log(`[rmath1] ${msg}`); + const el = typeof document !== "undefined" ? document.getElementById("module-output") : null; + if (el) el.value = (el.value ? `${el.value}\n` : "") + msg; + }, + }; + client.set_on_message((frame) => { + if (typeof frame !== "string") return; + try { + const msg = JSON.parse(frame); + if (msg.type === "math1-input" && msg.bucket && msg.filename) globalThis.__etAgent.input = msg; + } catch {} + }); + client.connect(); +} diff --git a/services/ws-modules/rmath1/pkg/module.R b/services/ws-modules/rmath1/pkg/module.R new file mode 100644 index 00000000..547ea97c --- /dev/null +++ b/services/ws-modules/rmath1/pkg/module.R @@ -0,0 +1,148 @@ +# module.R -- the rmath1 FedAvg module. run() IS the control loop; the JS shim only boots webR, sets up the +# agent WebSocket transport, and hands control here. +# +# Storage-driven: run() waits for the broadcast math1-input pointer (captured by the shim onto +# globalThis.__etAgent.input), reads the input JSON (client datasets + hyperparameters) from ws-server +# storage with httr2 (tunnelled via the /websockify SOCKS5 relay), runs the FedAvg kernel -- only + - * / +# on doubles, bit-identical to the other math1 twins -- and PUTs the global model to math1-output.json in +# its own bucket, where the test harness reads and verifies it. httr2/jsonlite install at runtime from the +# webR package repo. + +# --- transport: drive the shim's WsClient + browser globals via webr::eval_js (await = TRUE -> main thread) --- +js <- function(code) { + webr::eval_js(code, await = TRUE) +} +agent_state <- function() as.character(js("globalThis.__etAgent.client.get_state()")) +agent_id <- function() as.character(js("globalThis.__etAgent.client.get_agent_id()")) +agent_disconnect <- function() js("globalThis.__etAgent.client.disconnect(); true") +agent_log <- function(msg) { + js(sprintf("globalThis.__etAgent.log(%s); true", encodeString(msg, quote = "\""))) +} +agent_input_pointer <- function() { + code <- paste0( + "globalThis.__etAgent.input", + " ? globalThis.__etAgent.input.bucket + '\\n' + globalThis.__etAgent.input.filename : ''" + ) + as.character(js(code)) +} +sleep_ms <- function(ms) { + js(sprintf("new Promise(function(resolve){setTimeout(resolve, %d);})", as.integer(ms))) +} + +# Point webR's Emscripten sockets at the /websockify relay so httr2/curl reach the server through it. This runs +# in the webR worker (no await = TRUE), where the SOCKFS filesystem lives. +rmath1_configure_socket <- function(relay_url) { + code <- paste0( + "SOCKFS.websocketArgs = SOCKFS.websocketArgs || {};", + " SOCKFS.websocketArgs.url = ", encodeString(relay_url, quote = "\""), ";", + " SOCKFS.websocketArgs.subprotocol = 'binary'; 0" + ) + webr::eval_js(code) +} + +# Run the FedAvg simulation on the parsed input and return the final global c(weight =, bias =). +# params$clients is a list of n x 2 matrices (feature, target); only + - * / on doubles in a fixed +# evaluation order, so the result is bit-identical to the other math1 language twins. +rmath1_fed_avg <- function(params) { + clients <- params$clients + rounds <- as.integer(params$rounds) + epochs <- as.integer(params$epochs) + learning_rate <- as.numeric(params$learning_rate) + weight <- 0.0 + bias <- 0.0 + total_samples <- 0.0 + for (samples in clients) total_samples <- total_samples + as.numeric(nrow(samples)) + for (round in seq_len(rounds)) { + merged_weight <- 0.0 + merged_bias <- 0.0 + for (samples in clients) { + count <- as.numeric(nrow(samples)) + client_weight <- weight + client_bias <- bias + for (epoch in seq_len(epochs)) { + grad_weight <- 0.0 + grad_bias <- 0.0 + for (i in seq_len(nrow(samples))) { + feature <- samples[i, 1] + target <- samples[i, 2] + residual <- client_weight * feature + client_bias - target + grad_weight <- grad_weight + residual * feature + grad_bias <- grad_bias + residual + } + client_weight <- client_weight - learning_rate * (2.0 * grad_weight / count) + client_bias <- client_bias - learning_rate * (2.0 * grad_bias / count) + } + merged_weight <- merged_weight + client_weight * count + merged_bias <- merged_bias + client_bias * count + } + weight <- merged_weight / total_samples + bias <- merged_bias / total_samples + } + c(weight = weight, bias = bias) +} + +# The control loop the JS shim hands control to. +run <- function() { + webr::install("httr2") + webr::install("jsonlite") + # Route curl through the /websockify relay's SOCKS5 front end (see the relay service). The proxy host is + # nominal -- SOCKFS sends every socket to relay_url -- but the scheme must be socks5h so curl speaks SOCKS5. + Sys.setenv(ALL_PROXY = "socks5h://127.0.0.1:8080") + agent_log("rmath1: entered run()") + + # Point webR's sockets at this origin's /websockify relay (http(s) origin -> ws(s) relay URL). + origin <- as.character(js("location.protocol + '//' + location.host")) + relay_url <- sub("^http", "ws", paste0(origin, "/websockify")) + rmath1_configure_socket(relay_url) + + # Wait for the agent WebSocket to connect and hand us the agent_id (this bucket). + repeat { + if (identical(agent_state(), "connected")) break + sleep_ms(100) + } + bucket <- "" + repeat { + bucket <- agent_id() + if (nzchar(bucket)) break + sleep_ms(100) + } + agent_log(sprintf("rmath1: registered as %s", bucket)) + + # Wait for the broadcast math1-input pointer ("bucket\nfilename" once captured by the shim). + agent_log("rmath1: waiting for the math1-input pointer broadcast") + pointer <- "" + repeat { + pointer <- agent_input_pointer() + if (nzchar(pointer)) break + sleep_ms(100) + } + parts <- strsplit(pointer, "\n", fixed = TRUE)[[1]] + input_url <- sprintf("http://127.0.0.1:8080/storage/%s/%s", parts[[1]], parts[[2]]) + agent_log(sprintf("rmath1: reading input from %s", input_url)) + input_text <- httr2::request(input_url) |> + httr2::req_perform() |> + httr2::resp_body_string() + params <- jsonlite::fromJSON(input_text, simplifyDataFrame = FALSE) + + agent_log(sprintf( + "rmath1: running FedAvg - %d clients x %d rounds x %d local epochs", + length(params$clients), as.integer(params$rounds), as.integer(params$epochs) + )) + model <- rmath1_fed_avg(params) + agent_log(sprintf("rmath1: global model weight=%.17g bias=%.17g", model[["weight"]], model[["bias"]])) + + # %.17g preserves the exact f64 across the JSON round-trip the harness parses. + output <- sprintf( + "{\"module\":\"rmath1\",\"weight\":%.17g,\"bias\":%.17g}", + model[["weight"]], model[["bias"]] + ) + output_url <- sprintf("http://127.0.0.1:8080/storage/%s/math1-output.json", bucket) + httr2::request(output_url) |> + httr2::req_method("PUT") |> + httr2::req_body_raw(output) |> + httr2::req_perform() + agent_log(sprintf("rmath1: stored the global model to %s", output_url)) + + agent_disconnect() + agent_log("rmath1: workflow complete") +} diff --git a/services/ws-modules/rmath1/pkg/package.json b/services/ws-modules/rmath1/pkg/package.json new file mode 100644 index 00000000..e1ca0831 --- /dev/null +++ b/services/ws-modules/rmath1/pkg/package.json @@ -0,0 +1,8 @@ +{ + "description": "R math 1", + "license": "Apache-2.0 OR MIT", + "main": "et_ws_rmath1.js", + "name": "et-ws-rmath1", + "type": "module", + "version": "0.1.0" +} diff --git a/services/ws-modules/wasi-math1/.gitignore b/services/ws-modules/wasi-math1/.gitignore new file mode 100644 index 00000000..af5726a8 --- /dev/null +++ b/services/ws-modules/wasi-math1/.gitignore @@ -0,0 +1,4 @@ +/pkg/*.wasm +/pkg/package.json +et_ws_wasi_math1.wasm +package.json diff --git a/services/ws-modules/wasi-math1/Cargo.toml b/services/ws-modules/wasi-math1/Cargo.toml new file mode 100644 index 00000000..e27644f4 --- /dev/null +++ b/services/ws-modules/wasi-math1/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "et-ws-wasi-math1" +description = "WASI Preview 2 math 1" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +crate-type = ["cdylib"] +doctest = false +test = false + +# Scope the wit-bindgen deps to wasi targets so a host `cargo check --workspace` skips them. +# The lib body is gated the same way (`#![cfg(target_os = "wasi")]` at the top of src/lib.rs), +# so on the host target the crate compiles to an empty cdylib. +[target.'cfg(target_os = "wasi")'.dependencies] +fs-err = { workspace = true, optional = true } +minicov = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +wit-bindgen.workspace = true + +# Coverage instrumentation, off by default and enabled only by the coverage build. +# Pulls minicov, whose capture_coverage the guest dumps to the runner's /cov preopen at the end of run(). +[features] +coverage = ["dep:fs-err", "dep:minicov"] + +# Build script (runs on the host) locates the repo root to emit ET_WIT_DIR. +[build-dependencies] +et-path.workspace = true + +[lints] +workspace = true diff --git a/services/ws-modules/wasi-math1/build.rs b/services/ws-modules/wasi-math1/build.rs new file mode 100644 index 00000000..17f2602c --- /dev/null +++ b/services/ws-modules/wasi-math1/build.rs @@ -0,0 +1,10 @@ +//! Emit `ET_WIT_DIR` (absolute path to the shared WIT directory) so the +//! `wit_bindgen::generate!` invocation in `src/lib.rs` locates it via `env!`, +//! instead of a `..`-relative path that hardcodes this crate's depth below the +//! repository root. + +fn main() { + let wit_dir = et_path::find_project_root_from_manifest().join("generated/specs/wit"); + println!("cargo:rustc-env=ET_WIT_DIR={}", wit_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/services/ws-modules/wasi-math1/src/coverage.rs b/services/ws-modules/wasi-math1/src/coverage.rs new file mode 100644 index 00000000..0258237c --- /dev/null +++ b/services/ws-modules/wasi-math1/src/coverage.rs @@ -0,0 +1,16 @@ +//! Coverage dump for the instrumented build, isolated into its own file. +//! +//! minicov's `capture_coverage` is an `unsafe fn` (it reads the raw instrumented counter buffers), which +//! Codacy flags for audit. Codacy can only exclude whole paths in-repo, not suppress per line, so this +//! one-function file is the single thing excluded from Codacy (see .codacy.yaml) while the rest of the guest +//! stays analyzed. The unsafe is still covered by the repo's own clippy (the crate expects `unsafe_code`) and +//! by DeepSource. Called from `run()` at the end; writes the profile to the runner's `/cov` preopen. + +pub fn dump() { + let mut coverage = Vec::new(); + // SAFETY: single-threaded guest; capture_coverage reads the instrumented counters once at run() end. + unsafe { + minicov::capture_coverage(&mut coverage).unwrap(); + } + fs_err::write("/cov/et_ws_wasi_math1.profraw", coverage).unwrap(); +} diff --git a/services/ws-modules/wasi-math1/src/lib.rs b/services/ws-modules/wasi-math1/src/lib.rs new file mode 100644 index 00000000..567f7224 --- /dev/null +++ b/services/ws-modules/wasi-math1/src/lib.rs @@ -0,0 +1,215 @@ +//! Rust WASI Preview 2 twin of the math1 `FedAvg` module. +//! +//! Storage-driven, under wasmtime -- the family's only native (non-browser) executor: waits for the +//! broadcast `math1-input` pointer (relayed to the guest through `ws.recv`), reads the input JSON +//! (client datasets + hyperparameters) from storage via `wasi:keyvalue/store`, runs the kernel -- +//! only `+ - * /` on f64 in a fixed evaluation order, bit-identical to the browser twins -- and +//! stores the global model to `math1-output.json` in its own bucket, where the test harness reads +//! and verifies it. +//! +//! Crate-level cfg gate: the wit-bindgen-generated extern declarations +//! reference WASI imports that only resolve on `wasm32-wasip2`. Gating the +//! whole module on `target_os = "wasi"` lets the crate sit in the parent +//! workspace -- `cargo check --workspace` from the repo root produces an +//! empty cdylib for the host target without linker errors. + +#![cfg(target_os = "wasi")] +// unsafe_code: wit_bindgen::generate! emits `unsafe fn` and `#[export_name]` items; +// `export!(Component)` does the same. Both trip workspace `unsafe_code = "deny"`; expect it at +// crate scope because outer `#[expect]` on the macro invocations themselves doesn't propagate to +// the items they expand into. +#![expect( + clippy::float_arithmetic, + unsafe_code, + reason = "wit-bindgen macro expansions are unsafe by construction; the FedAvg kernel is float math by design" +)] + +wit_bindgen::generate!({ + // ET_WIT_DIR is the absolute path to generated/specs/wit, emitted by build.rs. + path: env!("ET_WIT_DIR"), + world: "module", + generate_all, +}); + +use et::ws_messages::messages::ServerMessage; +use et::ws_wasi::ws::WsError; +use exports::et::ws_wasi::entry::{EntryError, Guest}; +use serde::Deserialize; +use wasi::keyvalue::store; +use wasi::logging::logging::{self, Level}; + +// Coverage dump lives in its own module so Codacy can exclude just that file (its minicov call is unsafe). +#[cfg(feature = "coverage")] +mod coverage; + +const LOG_CONTEXT: &str = env!("CARGO_PKG_NAME"); + +/// The canonical input: per-client (feature, target) samples plus the training hyperparameters. +#[derive(Deserialize)] +struct Math1Input { + clients: Vec>, + rounds: u32, + epochs: u32, + learning_rate: f64, +} + +/// The broadcast pointer naming the storage bucket + filename the input JSON was injected at. +#[derive(Deserialize)] +struct InputPointer { + bucket: String, + filename: String, +} + +fn info(message: &str) { + logging::log(Level::Info, LOG_CONTEXT, message); +} + +// Lets `?` lift a `ws-error` into `entry-error.ws(...)` so the body of `run` +// stays free of explicit `.map_err`s (which the workspace's no-map-err +// ast-grep rule bans outside listed error.rs files anyway). +impl From for EntryError { + fn from(err: WsError) -> Self { + Self::Ws(err) + } +} + +// Same idea for `wasi:keyvalue/store.error` -- the upstream type is a +// value variant (no resources involved), so `entry-error.store(...)` +// carries it through unchanged and guests propagate via `?`. +impl From for EntryError { + fn from(err: store::Error) -> Self { + Self::Store(err) + } +} + +/// Sample count as f64, accumulated additively to avoid an integer-to-float cast. +fn sample_count(samples: &[(f64, f64)]) -> f64 { + samples.iter().fold(0.0_f64, |count, _| count + 1.0) +} + +/// Runs the `FedAvg` simulation on `input` and returns the final global (weight, bias). +#[expect( + clippy::single_call_fn, + reason = "the kernel is a distinct step, kept separate from the ws workflow" +)] +fn fed_avg(input: &Math1Input) -> (f64, f64) { + let mut weight = 0.0_f64; + let mut bias = 0.0_f64; + let total_samples: f64 = input + .clients + .iter() + .fold(0.0_f64, |acc, samples| acc + sample_count(samples)); + for _ in 0_u32..input.rounds { + let mut merged_weight = 0.0_f64; + let mut merged_bias = 0.0_f64; + for samples in &input.clients { + let count = sample_count(samples); + let mut client_weight = weight; + let mut client_bias = bias; + for _ in 0_u32..input.epochs { + let mut grad_weight = 0.0_f64; + let mut grad_bias = 0.0_f64; + for &(feature, target) in samples { + let residual = client_weight * feature + client_bias - target; + grad_weight += residual * feature; + grad_bias += residual; + } + client_weight -= input.learning_rate * (2.0 * grad_weight / count); + client_bias -= input.learning_rate * (2.0 * grad_bias / count); + } + merged_weight += client_weight * count; + merged_bias += client_bias * count; + } + weight = merged_weight / total_samples; + bias = merged_bias / total_samples; + } + (weight, bias) +} + +struct Component; + +impl Guest for Component { + async fn run() -> Result<(), EntryError> { + info("entered run()"); + + et::ws_wasi::ws::connect()?; + let agent_id = + wait_for_agent_id().ok_or_else(|| EntryError::Runtime("did not receive agent_id".to_string()))?; + info(&format!("websocket connected with agent_id={agent_id}")); + + info("waiting for the math1-input pointer broadcast"); + let pointer = wait_for_pointer() + .ok_or_else(|| EntryError::Runtime("did not receive the math1-input pointer".to_string()))?; + + info(&format!( + "reading input from bucket={} key={}", + pointer.bucket, pointer.filename + )); + let input_bucket = store::open(&pointer.bucket)?; + let input_bytes = input_bucket + .get(&pointer.filename)? + .ok_or_else(|| EntryError::Runtime(format!("input {} not found", pointer.filename)))?; + let input: Math1Input = match serde_json::from_slice(&input_bytes) { + Ok(input) => input, + Err(err) => return Err(EntryError::Runtime(format!("input JSON parse failed: {err}"))), + }; + + info(&format!( + "running FedAvg - {} clients x {} rounds x {} local epochs", + input.clients.len(), + input.rounds, + input.epochs + )); + let (weight, bias) = fed_avg(&input); + info(&format!("global model weight={weight} bias={bias}")); + + let own_bucket = store::open(&agent_id)?; + let output = serde_json::json!({ "module": "wasi-math1", "weight": weight, "bias": bias }).to_string(); + own_bucket.set("math1-output.json", output.as_bytes())?; + info("stored the global model to math1-output.json"); + + et::ws_wasi::ws::disconnect(); + info("workflow complete"); + #[cfg(feature = "coverage")] + coverage::dump(); + Ok(()) + } +} + +/// Poll `agent_id` until the server's `ConnectAck` has landed. +/// `ws.connect` waits briefly for that message, but the host returns once its wait expires regardless, so +/// polling is what keeps this safe under load. +fn wait_for_agent_id() -> Option { + for _ in 0..100 { + let id = et::ws_wasi::ws::agent_id(); + if !id.is_empty() { + return Some(id); + } + sleep_ms(50); + } + None +} + +/// Drain the recv inbox until the relayed `math1-input` pointer broadcast arrives. +/// +/// The fake agent re-broadcasts the pointer until the output lands, so each 100ms recv window only +/// has to catch one of them; foreign frames arrive as `relay-text` envelopes. +fn wait_for_pointer() -> Option { + for _ in 0..100 { + if let Ok(Some(ServerMessage::RelayText(payload))) = et::ws_wasi::ws::recv(100) + && let Ok(json) = serde_json::from_str::(&payload.content) + && json.get("type").and_then(serde_json::Value::as_str) == Some("math1-input") + && let Ok(pointer) = serde_json::from_value::(json) + { + return Some(pointer); + } + } + None +} + +fn sleep_ms(ms: u64) { + let pollable = wasi::clocks::monotonic_clock::subscribe_duration(ms * 1_000_000); + let _ready = wasi::io::poll::poll(&[&pollable]); +} + +export!(Component); diff --git a/services/ws-modules/zig-math1/build.zig b/services/ws-modules/zig-math1/build.zig new file mode 100644 index 00000000..c389b8d6 --- /dev/null +++ b/services/ws-modules/zig-math1/build.zig @@ -0,0 +1,50 @@ +const std = @import("std"); +const zon = @import("build.zig.zon"); + +const npm_name = blk: { + const s = @tagName(zon.name); + var buf: [s.len]u8 = s[0..s.len].*; + for (&buf) |*c| if (c.* == '_') { + c.* = '-'; + }; + break :blk buf; +}; + +const name = @tagName(zon.name); +const wasm_install_path = "../pkg/" ++ name ++ ".wasm"; + +pub fn build(b: *std.Build) void { + const target = b.resolveTargetQuery(.{ + .cpu_arch = .wasm32, + .os_tag = .freestanding, + }); + const optimize = b.standardOptimizeOption(.{}); + + const root_module = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + }); + + const lib = b.addExecutable(.{ + .name = name, + .root_module = root_module, + }); + lib.entry = .disabled; + lib.rdynamic = true; + + const install = b.addInstallFile(lib.getEmittedBin(), wasm_install_path); + b.getInstallStep().dependOn(&install.step); + + const pkg_json = std.json.Stringify.valueAlloc(b.allocator, .{ + .name = &npm_name, + .type = "module", + .description = zon.description, + .version = zon.version, + .license = zon.license, + .main = zon.main, + }, .{ .whitespace = .indent_2 }) catch unreachable; + const wf = b.addWriteFile("package.json", pkg_json); + const install_pkg_json = b.addInstallFile(wf.getDirectory().path(b, "package.json"), "../pkg/package.json"); + b.getInstallStep().dependOn(&install_pkg_json.step); +} diff --git a/services/ws-modules/zig-math1/build.zig.zon b/services/ws-modules/zig-math1/build.zig.zon new file mode 100644 index 00000000..5c6d9b9c --- /dev/null +++ b/services/ws-modules/zig-math1/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .et_ws_zig_math1, + .version = "0.1.0", + .description = "Zig math 1", + .license = "Apache-2.0 or MIT", + .main = "et_ws_zig_math1.js", + .fingerprint = 0x50849fc82a52afc5, + .paths = .{""}, +} diff --git a/services/ws-modules/zig-math1/pkg/.gitignore b/services/ws-modules/zig-math1/pkg/.gitignore new file mode 100644 index 00000000..b67918f1 --- /dev/null +++ b/services/ws-modules/zig-math1/pkg/.gitignore @@ -0,0 +1,2 @@ +*.wasm +package.json diff --git a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js new file mode 100644 index 00000000..809fc3b1 --- /dev/null +++ b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js @@ -0,0 +1,178 @@ +// et_ws_zig_math1.js — zig-math1 WASM module +// Runs WASM in a Web Worker; main thread proxies WebSocket + REST calls via +// SharedArrayBuffer. Shared memory layout (Int32 offsets): +// [0] signal: 0=idle, 1=request-pending +// [1] request type: 0=sleep, 1=ws_connect, 2=ws_get_state, 3=ws_get_agent_id, +// 4=ws_get_input, 6=ws_disconnect, 7=log, 8=set_status, +// 9=get_ws_url, 11=rest_request +// [2] payload length (also response length; -1 on rest_request failure) +// [3] aux length (binary request body for rest_request) +// Data area starts at byte offset 16. + +export default async function init() {} + +export async function run() { + const DATA_OFFSET = 16; + const sab = new SharedArrayBuffer(64 * 1024); + const ctrl = new Int32Array(sab, 0, 4); + const data = new Uint8Array(sab, DATA_OFFSET); + const enc = new TextEncoder(); + const dec = new TextDecoder(); + + const workerUrl = new URL("et_ws_zig_math1_worker.js", import.meta.url).href; + + const respond = (str = "") => { + if (str) { + const b = enc.encode(str); + data.set(b); + Atomics.store(ctrl, 2, b.length); + } else Atomics.store(ctrl, 2, 0); + Atomics.store(ctrl, 0, 0); + Atomics.notify(ctrl, 0); + }; + + const respondBytes = (bytes) => { + data.set(bytes); + Atomics.store(ctrl, 2, bytes.length); + Atomics.store(ctrl, 0, 0); + Atomics.notify(ctrl, 0); + }; + + const respondError = () => { + Atomics.store(ctrl, 2, -1); + Atomics.store(ctrl, 0, 0); + Atomics.notify(ctrl, 0); + }; + + return new Promise((resolve, reject) => { + let ws = null, + wsState = "disconnected", + agentId = "", + inputPointer = null; + + const poll = () => { + if (Atomics.load(ctrl, 0) !== 1) { + setTimeout(poll, 0); + return; + } + + const type = Atomics.load(ctrl, 1); + const plen = Atomics.load(ctrl, 2); + const alen = Atomics.load(ctrl, 3); + const payload = dec.decode(Uint8Array.from(data.subarray(0, plen))); + + switch (type) { + case 0: + setTimeout( + () => { + respond(); + poll(); + }, + parseInt(payload) || 0, + ); + return; + case 1: + ws = new WebSocket(payload); + wsState = "connecting"; + ws.onopen = () => { + wsState = "connected"; + ws.send(JSON.stringify({ type: "et-connect" })); + }; + ws.onmessage = (e) => { + try { + const msg = JSON.parse(e.data); + if (msg.type === "et-connect-ack" && msg.agent_id) agentId = msg.agent_id; + if (msg.type === "math1-input" && msg.bucket && msg.filename) inputPointer = msg; + } catch {} + }; + ws.onclose = ws.onerror = () => { + wsState = "disconnected"; + }; + respond(); + break; + case 2: + respond(wsState); + break; + case 3: + respond(agentId); + break; + case 4: + respond(inputPointer ? `${inputPointer.bucket}\n${inputPointer.filename}` : ""); + break; + case 6: + ws?.close(); + wsState = "disconnected"; + respond(); + break; + case 7: + console.log(payload); + appendOutput(payload); + respond(); + break; + case 8: + appendOutput(payload); + respond(); + break; + case 9: { + const p = location.protocol === "https:" ? "wss:" : "ws:"; + respond(`${p}//${location.host}/ws`); + break; + } + case 11: { + // payload = "METHOD url", aux = binary body. Response is the raw + // body bytes; signal failures with respondError() so the Zig + // extern returns -1. + const spaceIdx = payload.indexOf(" "); + const method = payload.substring(0, spaceIdx); + const url = payload.substring(spaceIdx + 1); + const opts = { method }; + if (alen > 0) { + opts.body = new Uint8Array(data.subarray(plen, plen + alen)).slice(); + } + fetch(url, opts) + .then((r) => { + if (!r.ok) throw new Error(`HTTP ${r.status}`); + return r.arrayBuffer(); + }) + .then((buf) => { + respondBytes(new Uint8Array(buf)); + poll(); + }) + .catch(() => { + respondError(); + poll(); + }); + return; + } + default: + respond(); + break; + } + setTimeout(poll, 0); + }; + + const worker = new Worker(workerUrl, { type: "module" }); + worker.onmessage = (e) => { + if (e.data.done) { + worker.terminate(); + if (e.data.ret === 0) { + resolve(); + } else { + reject(new Error("zig-math1: run() returned " + e.data.ret)); + } + } + }; + worker.onerror = (e) => { + worker.terminate(); + reject(e); + }; + // The worker resolves its own wasm URL from import.meta.url; only the shared buffer crosses the boundary. + worker.postMessage({ sab }); + poll(); + }); +} + +function appendOutput(msg) { + const el = document.getElementById("module-output"); + if (el) el.value = (el.value ? el.value + "\n" : "") + msg; +} diff --git a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js new file mode 100644 index 00000000..452954b1 --- /dev/null +++ b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js @@ -0,0 +1,85 @@ +// et_ws_zig_math1_worker.js — Web Worker for zig-math1 WASM module +const DATA_OFFSET = 16; +let ctrl, data, wasmMemory; +const enc = new TextEncoder(), + dec = new TextDecoder(); +const readStr = (ptr, len) => dec.decode(new Uint8Array(wasmMemory.buffer, ptr, len)); + +// String payload, response is a UTF-8 string. +function call(type, payload = "") { + const pb = enc.encode(payload); + data.set(pb); + Atomics.store(ctrl, 3, 0); + Atomics.store(ctrl, 2, pb.length); + Atomics.store(ctrl, 1, type); + Atomics.store(ctrl, 0, 1); + Atomics.notify(ctrl, 0); + Atomics.wait(ctrl, 0, 1); // block until main thread responds + const rlen = Atomics.load(ctrl, 2); + return dec.decode(Uint8Array.from(data.subarray(0, rlen))); +} + +// "METHOD url" payload plus a binary body; response is raw bytes, or null when +// the main thread signalled failure (negative response length). +function callRest(method, url, body) { + const pb = enc.encode(`${method} ${url}`); + const ab = body || new Uint8Array(0); + data.set(pb); + if (ab.length) data.set(ab, pb.length); + Atomics.store(ctrl, 3, ab.length); + Atomics.store(ctrl, 2, pb.length); + Atomics.store(ctrl, 1, 11); + Atomics.store(ctrl, 0, 1); + Atomics.notify(ctrl, 0); + Atomics.wait(ctrl, 0, 1); + const rlen = Atomics.load(ctrl, 2); + if (rlen < 0) return null; + return Uint8Array.from(data.subarray(0, rlen)); +} + +const writeBack = (r, buf, max) => { + const b = enc.encode(r); + const n = Math.min(b.length, max); + new Uint8Array(wasmMemory.buffer, buf, n).set(b.subarray(0, n)); + return n; +}; + +const imports = { + env: { + js_log: (p, l) => call(7, readStr(p, l)), + js_set_status: (p, l) => call(8, readStr(p, l)), + js_ws_connect: (p, l) => call(1, readStr(p, l)), + js_ws_disconnect: () => call(6), + js_ws_get_state: (buf, max) => writeBack(call(2), buf, max), + js_ws_get_agent_id: (buf, max) => writeBack(call(3), buf, max), + js_ws_get_input: (buf, max) => writeBack(call(4), buf, max), + js_rest_request: (mp, ml, up, ul, bp, bl, buf, max) => { + const method = readStr(mp, ml); + const url = readStr(up, ul); + const body = bl > 0 ? new Uint8Array(wasmMemory.buffer, bp, bl).slice() : null; + const response = callRest(method, url, body); + if (response === null) return -1; + const n = Math.min(response.length, max); + new Uint8Array(wasmMemory.buffer, buf, n).set(response.subarray(0, n)); + return n; + }, + js_sleep_ms: (ms) => call(0, String(ms)), + js_get_ws_url: (buf, max) => writeBack(call(9), buf, max), + }, +}; + +self.onmessage = async (e) => { + // Dedicated worker: messages only originate from the same-origin context that created it. Reject any + // cross-origin message defensively (the browser already guarantees this, but make the check explicit). + if (e.origin && e.origin !== self.location.origin) return; + const { sab } = e.data; + ctrl = new Int32Array(sab, 0, 4); + data = new Uint8Array(sab, DATA_OFFSET); + // Resolve the module wasm from this worker's own location (self.location), never from a postMessage value, + // so the fetch URL cannot depend on message data. The wasm is a fixed-name sibling of this worker script. + const wasmUrl = new URL("et_ws_zig_math1.wasm", self.location.href); + const { instance } = await WebAssembly.instantiateStreaming(fetch(wasmUrl), imports); + wasmMemory = instance.exports.memory; + const ret = instance.exports.run(); + self.postMessage({ done: true, ret }); +}; diff --git a/services/ws-modules/zig-math1/src/main.zig b/services/ws-modules/zig-math1/src/main.zig new file mode 100644 index 00000000..b69c0883 --- /dev/null +++ b/services/ws-modules/zig-math1/src/main.zig @@ -0,0 +1,212 @@ +// zig-math1: storage-driven federated-averaging (FedAvg) demo in a Zig wasm module. +// Waits for the broadcast math1-input pointer, reads the input JSON (client datasets + +// hyperparameters) from ws-server storage through the worker shim's REST relay, runs the kernel -- +// only + - * / on f64 in a fixed evaluation order, bit-identical to the other math1 twins -- and +// stores the global model to math1-output.json in its own bucket, where the test harness reads and +// verifies it. All browser I/O is provided by JS imports, mirroring zig-data1's worker shim. + +const std = @import("std"); + +extern fn js_log(ptr: [*]const u8, len: usize) void; +extern fn js_set_status(ptr: [*]const u8, len: usize) void; +extern fn js_ws_connect(url_ptr: [*]const u8, url_len: usize) void; +extern fn js_ws_disconnect() void; +extern fn js_ws_get_state(buf: [*]u8, max: usize) usize; +extern fn js_ws_get_agent_id(buf: [*]u8, max: usize) usize; +extern fn js_ws_get_input(buf: [*]u8, max: usize) usize; +extern fn js_sleep_ms(ms: u32) void; +extern fn js_get_ws_url(buf: [*]u8, max: usize) usize; +extern fn js_rest_request( + method_ptr: [*]const u8, + method_len: usize, + url_ptr: [*]const u8, + url_len: usize, + body_ptr: [*]const u8, + body_len: usize, + buf: [*]u8, + max: usize, +) i32; + +var heap: [64 * 1024]u8 = undefined; +var fba = std.heap.FixedBufferAllocator.init(&heap); +const alloc = fba.allocator(); + +const Math1Input = struct { + clients: [][][2]f64, + rounds: u32, + epochs: u32, + learning_rate: f64, +}; + +const Model = struct { weight: f64, bias: f64 }; + +fn log(comptime fmt: []const u8, args: anytype) void { + const msg = std.fmt.allocPrint(alloc, "[zig-math1] " ++ fmt, args) catch return; + defer alloc.free(msg); + js_log(msg.ptr, msg.len); +} + +fn set_status(comptime fmt: []const u8, args: anytype) void { + const msg = std.fmt.allocPrint(alloc, fmt, args) catch return; + defer alloc.free(msg); + js_set_status(msg.ptr, msg.len); +} + +fn wait_state(want: []const u8) bool { + var buf: [32]u8 = undefined; + var i: u32 = 0; + while (i < 100) : (i += 1) { + const n = js_ws_get_state(&buf, buf.len); + if (std.mem.eql(u8, buf[0..n], want)) return true; + js_sleep_ms(100); + } + return false; +} + +fn wait_agent_id(buf: []u8) usize { + var i: u32 = 0; + while (i < 100) : (i += 1) { + const n = js_ws_get_agent_id(buf.ptr, buf.len); + if (n > 0) return n; + js_sleep_ms(100); + } + return 0; +} + +// The main-thread shim serialises the captured math1-input pointer as "bucket\nfilename". +fn wait_input_pointer(buf: []u8) usize { + var i: u32 = 0; + while (i < 100) : (i += 1) { + const n = js_ws_get_input(buf.ptr, buf.len); + if (n > 0) return n; + js_sleep_ms(100); + } + return 0; +} + +// Runs the FedAvg simulation on the fetched input and returns the final global model. +fn fed_avg(input: Math1Input) Model { + var weight: f64 = 0.0; + var bias: f64 = 0.0; + var total_samples: f64 = 0.0; + for (input.clients) |samples| { + total_samples += @as(f64, @floatFromInt(samples.len)); + } + var round: u32 = 0; + while (round < input.rounds) : (round += 1) { + var merged_weight: f64 = 0.0; + var merged_bias: f64 = 0.0; + for (input.clients) |samples| { + const count: f64 = @floatFromInt(samples.len); + var client_weight = weight; + var client_bias = bias; + var epoch: u32 = 0; + while (epoch < input.epochs) : (epoch += 1) { + var grad_weight: f64 = 0.0; + var grad_bias: f64 = 0.0; + for (samples) |sample| { + const residual = client_weight * sample[0] + client_bias - sample[1]; + grad_weight += residual * sample[0]; + grad_bias += residual; + } + client_weight -= input.learning_rate * (2.0 * grad_weight / count); + client_bias -= input.learning_rate * (2.0 * grad_bias / count); + } + merged_weight += client_weight * count; + merged_bias += client_bias * count; + } + weight = merged_weight / total_samples; + bias = merged_bias / total_samples; + } + return .{ .weight = weight, .bias = bias }; +} + +export fn run() i32 { + var url_buf: [256]u8 = undefined; + const url_len = js_get_ws_url(&url_buf, url_buf.len); + const ws_url = url_buf[0..url_len]; + + log("entered run()", .{}); + set_status("zig-math1: entered run()", .{}); + + js_ws_connect(ws_url.ptr, ws_url.len); + + if (!wait_state("connected")) { + log("timed out waiting for connection", .{}); + return -1; + } + + var agent_buf: [128]u8 = undefined; + const agent_len = wait_agent_id(&agent_buf); + if (agent_len == 0) { + log("timed out waiting for agent_id", .{}); + return -1; + } + const agent_id = agent_buf[0..agent_len]; + log("connected as {s}", .{agent_id}); + set_status("zig-math1: connected as {s}", .{agent_id}); + + set_status("zig-math1: waiting for the math1-input pointer broadcast", .{}); + var pointer_buf: [512]u8 = undefined; + const pointer_len = wait_input_pointer(&pointer_buf); + if (pointer_len == 0) { + log("timed out waiting for the math1-input pointer", .{}); + return -1; + } + const pointer = pointer_buf[0..pointer_len]; + const newline = std.mem.indexOfScalar(u8, pointer, '\n') orelse { + log("malformed input pointer: {s}", .{pointer}); + return -1; + }; + const bucket = pointer[0..newline]; + const filename = pointer[newline + 1 ..]; + + const input_url = std.fmt.allocPrint(alloc, "/storage/{s}/{s}", .{ bucket, filename }) catch return -1; + defer alloc.free(input_url); + set_status("zig-math1: reading input from {s}", .{input_url}); + var input_buf: [4096]u8 = undefined; + const input_len = js_rest_request("GET", 3, input_url.ptr, input_url.len, "", 0, &input_buf, input_buf.len); + if (input_len < 0) { + log("input GET failed", .{}); + return -1; + } + const input_bytes = input_buf[0..@intCast(input_len)]; + + const parsed = std.json.parseFromSlice(Math1Input, alloc, input_bytes, .{}) catch { + log("input JSON parse failed", .{}); + return -1; + }; + defer parsed.deinit(); + const input = parsed.value; + + set_status( + "zig-math1: running FedAvg - {d} clients x {d} rounds x {d} local epochs", + .{ input.clients.len, input.rounds, input.epochs }, + ); + const model = fed_avg(input); + log("global model weight={d} bias={d}", .{ model.weight, model.bias }); + set_status("zig-math1: global model weight={d} bias={d}", .{ model.weight, model.bias }); + + const output = std.fmt.allocPrint( + alloc, + "{{\"module\":\"zig-math1\",\"weight\":{d},\"bias\":{d}}}", + .{ model.weight, model.bias }, + ) catch return -1; + defer alloc.free(output); + const output_url = std.fmt.allocPrint(alloc, "/storage/{s}/math1-output.json", .{agent_id}) catch return -1; + defer alloc.free(output_url); + var put_buf: [256]u8 = undefined; + const put_len = + js_rest_request("PUT", 3, output_url.ptr, output_url.len, output.ptr, output.len, &put_buf, put_buf.len); + if (put_len < 0) { + log("output PUT failed", .{}); + return -1; + } + set_status("zig-math1: stored the global model to {s}", .{output_url}); + + js_sleep_ms(2000); + js_ws_disconnect(); + log("workflow complete", .{}); + set_status("zig-math1: workflow complete", .{}); + return 0; +} diff --git a/services/ws-pyo3-runner/python/math1.py b/services/ws-pyo3-runner/python/math1.py new file mode 100644 index 00000000..ef489d36 --- /dev/null +++ b/services/ws-pyo3-runner/python/math1.py @@ -0,0 +1,82 @@ +"""FedAvg math1 twin for `et-ws-pyo3-runner`: storage-driven, on native CPython. + +A fake agent injects the canonical input JSON (client datasets + hyperparameters) into ws-server +storage and broadcasts a `math1-input` pointer, which arrives here as an unrecognised text frame. +This module reads the input through the runner's storage handle, runs the FedAvg kernel -- only ++ - * / on floats, so the result is bit-identical to the other math1 twins -- and stores the global +model to math1-output.json in its own bucket, where the test harness reads and verifies it. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +_logger = logging.getLogger(__name__) + +_storage: Any = None + + +def fed_avg(clients: list, rounds: int, epochs: int, learning_rate: float) -> tuple[float, float]: + """Run the FedAvg simulation and return the final global (weight, bias).""" + weight = 0.0 + bias = 0.0 + total_samples = 0.0 + for samples in clients: + total_samples += float(len(samples)) + for _ in range(rounds): + merged_weight = 0.0 + merged_bias = 0.0 + for samples in clients: + count = float(len(samples)) + client_weight = weight + client_bias = bias + for _ in range(epochs): + grad_weight = 0.0 + grad_bias = 0.0 + for sample in samples: + residual = client_weight * sample[0] + client_bias - sample[1] + grad_weight += residual * sample[0] + grad_bias += residual + client_weight -= learning_rate * (2.0 * grad_weight / count) + client_bias -= learning_rate * (2.0 * grad_bias / count) + merged_weight += client_weight * count + merged_bias += client_bias * count + weight = merged_weight / total_samples + bias = merged_bias / total_samples + return weight, bias + + +def init(_send, storage) -> None: + """Stash the WsStorage handle for the exchange.""" + global _storage + _storage = storage + + +def on_text_frame(text: str) -> None: + """On the math1-input pointer broadcast: read the input, compute, and store the output. + + The pointer is re-broadcast until the harness sees the output, so duplicates just recompute + and re-store the same bytes -- idempotent by construction. + """ + try: + msg = json.loads(text) + except ValueError: + return + if not (isinstance(msg, dict) and msg.get("type") == "math1-input"): + return + input_bytes = _storage.get(msg["bucket"], msg["filename"]) + if input_bytes is None: + raise RuntimeError(f"input {msg['filename']} not found in bucket {msg['bucket']}") + params = json.loads(bytes(input_bytes).decode("utf-8")) + _logger.info( + "running FedAvg - %d clients x %d rounds x %d local epochs", + len(params["clients"]), + params["rounds"], + params["epochs"], + ) + weight, bias = fed_avg(params["clients"], params["rounds"], params["epochs"], params["learning_rate"]) + _logger.info("global model weight=%r bias=%r", weight, bias) + output = json.dumps({"module": "math1", "weight": weight, "bias": bias}) + _storage.put("math1-output.json", output.encode("utf-8")) diff --git a/services/ws-pyo3-runner/tests/modules.rs b/services/ws-pyo3-runner/tests/modules.rs index 6fb85176..0f4a6f5e 100644 --- a/services/ws-pyo3-runner/tests/modules.rs +++ b/services/ws-pyo3-runner/tests/modules.rs @@ -270,6 +270,26 @@ async fn run_exchange(control: &mut ControlSocket, self_id: &str, exchange: &Exc Ok(()) } +/// math1's storage-driven exchange: inject the canonical input, then verify the stored model. +/// +/// The fake-agent side lives in `et_ws_test_server::math1`; the module reads the input through the +/// runner's storage handle, computes, and stores its global model, which is verified against the +/// expected weights for the canonical input. The runner is long-lived, so it is killed once the +/// exchange resolves (mirroring `module_behaves`). +#[tokio::test(flavor = "current_thread")] +async fn math1_stores_verified_model() -> Result<(), Box> { + let server = et_ws_test_server::start(); + let mut runner = spawn_runner("math1", &server.ws_url); + let outcome = + et_ws_test_server::math1::drive_math1_exchange(&server.ws_url, server.storage_dir.path(), EXCHANGE_BUDGET) + .await; + runner.kill().unwrap(); + let _status = runner.wait().unwrap(); + let (weight, bias) = outcome?; + et_ws_test_server::math1::verify_math1_model(weight, bias)?; + Ok(()) +} + /// Torch case checker: matmul + tiny-classifier summary from `torch_inference.py`. fn check_torch(value: &serde_json::Value) -> Result<(), Box> { if value.get("framework").and_then(serde_json::Value::as_str) != Some("torch") { diff --git a/services/ws-test-server/Cargo.toml b/services/ws-test-server/Cargo.toml index f46d8a16..d87c7b27 100644 --- a/services/ws-test-server/Cargo.toml +++ b/services/ws-test-server/Cargo.toml @@ -22,6 +22,7 @@ fs-err.workspace = true futures-util.workspace = true serde_json.workspace = true tempfile.workspace = true +thiserror.workspace = true tokio = { workspace = true, features = ["macros", "net", "rt", "time"] } tokio-tungstenite = { workspace = true, features = ["connect"] } # Same TracingLogger setup as the real ws-server. diff --git a/services/ws-test-server/data/math1-input.json b/services/ws-test-server/data/math1-input.json new file mode 100644 index 00000000..cdca13f0 --- /dev/null +++ b/services/ws-test-server/data/math1-input.json @@ -0,0 +1,25 @@ +{ + "clients": [ + [ + [0.0, 1.1], + [1.0, 2.9], + [2.0, 5.2], + [3.0, 6.8] + ], + [ + [0.5, 2.2], + [1.5, 4.1], + [2.5, 5.9] + ], + [ + [1.0, 3.1], + [2.0, 4.9], + [3.0, 7.2], + [4.0, 9.1], + [5.0, 10.8] + ] + ], + "rounds": 10, + "epochs": 5, + "learning_rate": 0.02 +} diff --git a/services/ws-test-server/src/lib.rs b/services/ws-test-server/src/lib.rs index 120cf109..f98a7dfe 100644 --- a/services/ws-test-server/src/lib.rs +++ b/services/ws-test-server/src/lib.rs @@ -20,6 +20,8 @@ use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; use tracing_actix_web::TracingLogger; +pub mod math1; + /// A running test server. The temporary storage directory is cleaned up on drop. #[non_exhaustive] pub struct TestServer { @@ -87,8 +89,10 @@ pub fn start_on(port: u16) -> TestServer { panic!("test ws-server did not start within 5 seconds on port {port}"); } -/// Open a ws connection to `ws_url`, send `et-connect`, and return `(stream, agent_id)` once the -/// `et-connect-ack` has been observed. Lets a test drive the hub as a websocket client. +/// Open a ws connection to `ws_url` and drive `et-connect` through its ack. +/// +/// Returns `(stream, agent_id)` once the `et-connect-ack` has been observed. Lets a test drive the +/// hub as a websocket client. pub async fn connect_agent( ws_url: &str, ) -> ( @@ -117,8 +121,10 @@ pub async fn connect_agent( panic!("never received et-connect-ack within 5s"); } -/// Pull the next frame from `stream`, skipping known protocol acks (`et-connect-ack`, -/// `et-message-status`, `et-response`) so callers see the next "real" payload. +/// Pull the next non-ack frame from `stream`. +/// +/// Skips known protocol acks (`et-connect-ack`, `et-message-status`, `et-response`) so callers see +/// the next "real" payload. pub async fn next_payload( stream: &mut tokio_tungstenite::WebSocketStream>, ) -> Message { diff --git a/services/ws-test-server/src/math1.rs b/services/ws-test-server/src/math1.rs new file mode 100644 index 00000000..3ab4ffb7 --- /dev/null +++ b/services/ws-test-server/src/math1.rs @@ -0,0 +1,163 @@ +//! The fake-agent side of the math1 storage exchange, shared by every runner's math1 test. +//! +//! The math1 family's protocol: a fake agent injects the canonical input JSON (committed at +//! `data/math1-input.json`) into the ws-server's storage, then broadcasts a pointer frame +//! `{"type":"math1-input","bucket":...,"filename":...}` over the hub (an unrecognised frame the +//! server relays verbatim to every other agent). The math1 module under test reads the input from +//! storage, runs the `FedAvg` kernel with the file's parameters, and writes its global model to +//! `math1-output.json` in its own bucket, where [`drive_math1_exchange`] picks it up and +//! [`verify_math1_model`] checks it against the expected weights for the canonical input. The +//! pointer is re-broadcast until the output lands, so a module that connects after the first +//! broadcast still hears it. + +#![expect( + clippy::float_arithmetic, + reason = "verifying FedAvg model floats against the expected constants is float math by design" +)] + +use std::path::Path; +use std::time::Duration; + +use edge_toolkit::ws::{ClientMessage, ServerMessage}; +use futures_util::{SinkExt as _, StreamExt as _}; +use tokio_tungstenite::connect_async; +use tokio_tungstenite::tungstenite::Message; + +/// Storage object name of the canonical input, inside the fake agent's bucket. +pub const MATH1_INPUT_FILENAME: &str = "math1-input.json"; +/// Storage object name each math1 module writes its global model to, inside its own bucket. +pub const MATH1_OUTPUT_FILENAME: &str = "math1-output.json"; +/// The canonical input dataset + hyperparameters, committed so every module computes the same run. +pub const MATH1_INPUT_JSON: &str = include_str!("../data/math1-input.json"); +/// Expected global model for the canonical input; the one hard-coded verification point. +pub const MATH1_EXPECTED_WEIGHT: f64 = 2.027_406_278_700_665; +/// Expected bias for the canonical input; see [`MATH1_EXPECTED_WEIGHT`]. +pub const MATH1_EXPECTED_BIAS: f64 = 0.914_969_140_718_165_6; +/// Comparison tolerance: effectively exact for f64s that survived a JSON round-trip. +pub const MATH1_TOLERANCE: f64 = 1e-12; + +/// How often the pointer is re-broadcast and the output file re-polled. +const POLL_INTERVAL: Duration = Duration::from_millis(250); + +/// Failure of the math1 exchange, either in the fake agent's transport or in the module's output. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Math1Error { + /// Websocket transport failure on the fake agent's connection. + /// + /// Boxed: tungstenite's error is large, and `result_large_err` fires on every `Result` + /// carrying it inline. + #[error(transparent)] + Transport(Box), + /// The module's stored output was not the expected JSON shape. + #[error(transparent)] + Json(#[from] serde_json::Error), + /// Protocol-level failure: a timeout, a missing field, or a wrong model value. + #[error("{0}")] + Protocol(String), +} + +impl From for Math1Error { + fn from(err: tokio_tungstenite::tungstenite::Error) -> Self { + Self::Transport(Box::new(err)) + } +} + +/// Connect the fake agent, inject the input, broadcast the pointer, and await the module's output. +/// +/// Returns the (weight, bias) parsed from the module's `math1-output.json`. The input file is +/// written straight into `storage_dir` under the fake agent's bucket (the disk layout the local +/// storage backend serves), and the module's output is read back the same way -- the module itself +/// exercises the real REST/keyvalue path in both directions. +pub async fn drive_math1_exchange( + ws_url: &str, + storage_dir: &Path, + budget: Duration, +) -> Result<(f64, f64), Math1Error> { + let (mut socket, _response) = connect_async(ws_url).await?; + let connect = serde_json::to_string(&ClientMessage::Connect { agent_id: None })?; + socket.send(Message::Text(connect)).await?; + + let deadline = tokio::time::Instant::now() + budget; + let mut fake_id = String::new(); + let mut peers: Vec = Vec::new(); + let mut pointer = String::new(); + + loop { + if tokio::time::Instant::now() >= deadline { + return Err(Math1Error::Protocol(format!( + "math1 exchange timed out: fake_id={fake_id:?} peers={peers:?} (no {MATH1_OUTPUT_FILENAME} yet)" + ))); + } + + // Drain inbound frames for one poll interval, tracking the ack and the agent roster. + let drain_until = tokio::time::Instant::now() + POLL_INTERVAL; + while tokio::time::Instant::now() < drain_until { + let remaining = drain_until - tokio::time::Instant::now(); + match tokio::time::timeout(remaining, socket.next()).await { + Ok(Some(Ok(Message::Text(text)))) => match serde_json::from_str::(&text) { + Ok(ServerMessage::ConnectAck { agent_id, .. }) => { + fake_id = agent_id; + let bucket_dir = storage_dir.join(&fake_id); + fs_err::create_dir_all(&bucket_dir).unwrap(); + fs_err::write(bucket_dir.join(MATH1_INPUT_FILENAME), MATH1_INPUT_JSON).unwrap(); + pointer = format!( + r#"{{"type":"math1-input","bucket":"{fake_id}","filename":"{MATH1_INPUT_FILENAME}"}}"# + ); + } + Ok(ServerMessage::ListAgentsResponse { agents }) => { + peers = agents + .into_iter() + .map(|summary| summary.agent_id) + .filter(|agent_id| *agent_id != fake_id) + .collect(); + } + _ => {} + }, + Ok(Some(Ok(_))) => {} + Ok(Some(Err(err))) => return Err(err.into()), + Ok(None) => return Err(Math1Error::Protocol("fake agent socket closed".to_string())), + Err(_elapsed) => break, + } + } + + // The module writes its output into its own bucket; poll every known peer's bucket on disk. + for peer in &peers { + let output_path = storage_dir.join(peer).join(MATH1_OUTPUT_FILENAME); + if let Ok(bytes) = fs_err::read(&output_path) { + let value: serde_json::Value = serde_json::from_slice(&bytes)?; + let weight = value + .get("weight") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| Math1Error::Protocol("output missing `weight`".to_string()))?; + let bias = value + .get("bias") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| Math1Error::Protocol("output missing `bias`".to_string()))?; + return Ok((weight, bias)); + } + } + + // Ask for the roster and re-broadcast the pointer; both are safe to repeat. + let list = serde_json::to_string(&ClientMessage::ListAgents)?; + socket.send(Message::Text(list)).await?; + if !pointer.is_empty() { + socket.send(Message::Text(pointer.clone())).await?; + } + } +} + +/// Check a module's global model against the expected weights for the canonical input. +pub fn verify_math1_model(weight: f64, bias: f64) -> Result<(), Math1Error> { + if (weight - MATH1_EXPECTED_WEIGHT).abs() > MATH1_TOLERANCE { + return Err(Math1Error::Protocol(format!( + "weight {weight} != expected {MATH1_EXPECTED_WEIGHT}" + ))); + } + if (bias - MATH1_EXPECTED_BIAS).abs() > MATH1_TOLERANCE { + return Err(Math1Error::Protocol(format!( + "bias {bias} != expected {MATH1_EXPECTED_BIAS}" + ))); + } + Ok(()) +} diff --git a/services/ws-wasi-runner/tests/modules.rs b/services/ws-wasi-runner/tests/modules.rs index 1cf3ac2e..74edffae 100644 --- a/services/ws-wasi-runner/tests/modules.rs +++ b/services/ws-wasi-runner/tests/modules.rs @@ -40,3 +40,63 @@ fn module_runs_successfully(#[case] module: &str, #[case] language: Language) { .status_checked() .unwrap_or_else(|error| panic!("{module} runner failed: {error}")); } + +/// Run wasi-math1 through the storage-driven exchange and verify its stored model. +/// +/// The fake-agent side lives in `et_ws_test_server::math1`: it injects the canonical input JSON +/// into storage, broadcasts the `math1-input` pointer, and reads back the component's +/// `math1-output.json`, which is verified against the expected weights for that input. The +/// runner's own exit status is asserted too. +#[tokio::test(flavor = "current_thread")] +#[cfg_attr( + windows, + ignore = "pkg/package.json 404 on Windows -- see module_runs_successfully's comment" +)] +async fn wasi_math1_stores_verified_model() { + if !mise_env_includes(Language::Rust) { + return; + } + let server = et_ws_test_server::start(); + let bin = env!("CARGO_BIN_EXE_et-ws-wasi-runner"); + let mut runner = std::process::Command::new(bin) + .env("RUNNER_MODULE", "et-ws-wasi-math1") + .env("WS_SERVER_URL", &server.ws_url) + .env("ET_TEST_WS_WASI_RUNNER_FAST_EXIT", "1") + .spawn() + .unwrap(); + let outcome = et_ws_test_server::math1::drive_math1_exchange( + &server.ws_url, + server.storage_dir.path(), + std::time::Duration::from_secs(90), + ) + .await; + // Reap the runner regardless of the exchange outcome; it exits on its own once the component + // completes. + let status = wait_for_runner_exit(&mut runner); + let (weight, bias) = outcome.unwrap_or_else(|err| panic!("wasi-math1: {err}")); + et_ws_test_server::math1::verify_math1_model(weight, bias).unwrap_or_else(|err| panic!("wasi-math1: {err}")); + assert!(status.success(), "wasi-math1 runner exited {status:?}"); +} + +/// Poll the spawned runner until it exits, killing it if it overstays the bound. +/// +/// Blocking here is fine: this runs after the exchange future has already resolved, so nothing +/// else is pending on the current-thread runtime. +#[expect( + clippy::arithmetic_side_effects, + clippy::single_call_fn, + reason = "distinct reap step; the deadline addition cannot overflow within a test's lifetime" +)] +fn wait_for_runner_exit(runner: &mut std::process::Child) -> std::process::ExitStatus { + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(2); + loop { + if let Some(status) = runner.try_wait().unwrap() { + return status; + } + if std::time::Instant::now() >= deadline { + runner.kill().unwrap(); + return runner.wait().unwrap(); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } +} diff --git a/services/ws-web-runner/Cargo.toml b/services/ws-web-runner/Cargo.toml index 139a0259..82a5d25a 100644 --- a/services/ws-web-runner/Cargo.toml +++ b/services/ws-web-runner/Cargo.toml @@ -58,6 +58,7 @@ fs-err.workspace = true et-ws-test-server.workspace = true fs-err.workspace = true rstest.workspace = true +tokio = { workspace = true, features = ["macros", "rt", "time"] } # `coverage` (off by default): compiles the browser-module coverage-capture code into the runner. # It adds the pycov + ws-agent-id shims, the __ET_TEST_COVERAGE prelude, and the wrapper's __et_capture_coverage diff --git a/services/ws-web-runner/tests/modules.rs b/services/ws-web-runner/tests/modules.rs index cccf3306..4b981221 100644 --- a/services/ws-web-runner/tests/modules.rs +++ b/services/ws-web-runner/tests/modules.rs @@ -104,6 +104,86 @@ fn module_runs_successfully(#[case] module: &str, #[case] language: Language) { collect_module_coverage(&server); } +/// Run each math1 module through the storage-driven exchange and verify its stored model. +/// +/// The fake-agent side lives in `et_ws_test_server::math1`: it injects the canonical input JSON +/// into storage, broadcasts the `math1-input` pointer, and reads back the module's +/// `math1-output.json`, which is verified against the expected weights for that input. The +/// runner's own exit status is asserted too, so a module that errors after storing its output +/// still fails the case. +#[rstest] +#[case::math1("et-ws-math1", Language::Rust)] +#[case::dart_math1("et-ws-dart-math1", Language::Dart)] +#[case::dotnet_math1("et-ws-dotnet-math1", Language::Dotnet)] +#[case::java_math1("et-ws-java-math1", Language::Java)] +#[case::js_math1("et-ws-js-math1", Language::Js)] +#[case::kotlin_math1("et-ws-kotlin-math1", Language::Kotlin)] +#[case::pymath1("et-ws-pymath1", Language::Python)] +#[case::zig_math1("et-ws-zig-math1", Language::Zig)] +#[tokio::test(flavor = "current_thread")] +async fn math1_module_stores_verified_model(#[case] module: &str, #[case] language: Language) { + if !mise_env_includes(language) { + println!( + "skipping {module}: requires the `{}` mise env, not loaded", + language.as_str() + ); + return; + } + if module == "et-ws-dotnet-math1" && !dotnet_math1_pkg_built() { + println!("skipping {module}: pkg/ not built (build-ws-dotnet-math1-module has not run on this host)"); + return; + } + if module == "et-ws-kotlin-math1" && !kotlin_math1_pkg_built() { + println!("skipping {module}: pkg/ not built (build-ws-kotlin-math1-module has not run on this host)"); + return; + } + let server = et_ws_test_server::start(); + let bin = env!("CARGO_BIN_EXE_et-ws-web-runner"); + let mut runner = std::process::Command::new(bin) + .env("RUNNER_MODULE", module) + .env("WS_SERVER_URL", &server.ws_url) + .env("RUNNER_TIMEOUT", "90s") + .spawn() + .unwrap(); + let outcome = et_ws_test_server::math1::drive_math1_exchange( + &server.ws_url, + server.storage_dir.path(), + std::time::Duration::from_secs(90), + ) + .await; + // Reap the runner regardless of the exchange outcome; it exits on its own once the module + // completes (RUNNER_TIMEOUT bounds a hung module). + let status = wait_for_runner_exit(&mut runner); + let (weight, bias) = outcome.unwrap_or_else(|err| panic!("{module}: {err}")); + et_ws_test_server::math1::verify_math1_model(weight, bias).unwrap_or_else(|err| panic!("{module}: {err}")); + assert!(status.success(), "{module} runner exited {status:?}"); + #[cfg(feature = "coverage")] + collect_module_coverage(&server); +} + +/// Poll the spawned runner until it exits, killing it if it overstays the runner-timeout bound. +/// +/// Blocking here is fine: this runs after the exchange future has already resolved, so nothing +/// else is pending on the current-thread runtime. +#[expect( + clippy::arithmetic_side_effects, + clippy::single_call_fn, + reason = "distinct reap step; the deadline addition cannot overflow within a test's lifetime" +)] +fn wait_for_runner_exit(runner: &mut std::process::Child) -> std::process::ExitStatus { + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(2); + loop { + if let Some(status) = runner.try_wait().unwrap() { + return status; + } + if std::time::Instant::now() >= deadline { + runner.kill().unwrap(); + return runner.wait().unwrap(); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } +} + /// Probe for dotnet-data1's built `pkg/` wasm artifacts, logging a skip instead of failing when absent. /// /// dotnet-data1's `pkg/` wasm artifacts only exist after `build-ws-dotnet-data1-module` has run on this @@ -120,6 +200,20 @@ fn dotnet_data1_pkg_built() -> bool { .exists() } +/// Probe for dotnet-math1's built `pkg/` wasm artifacts, logging a skip instead of failing when absent. +/// +/// Same shape as [`dotnet_data1_pkg_built`]: probe the stably-named `dotnet.js` that +/// `build-ws-dotnet-math1-module` copies into `pkg/`, and skip rather than 404 on the fetch. +#[expect( + clippy::single_call_fn, + reason = "distinct probe step; kept named for the skip-trace log line" +)] +fn dotnet_math1_pkg_built() -> bool { + edge_toolkit::config::get_project_root() + .join("services/ws-modules/dotnet-math1/pkg/dotnet.js") + .exists() +} + /// Probe for js-data1's esbuild bundle, logging a skip instead of failing when absent. /// /// The AWS SDK v3 bundle `pkg/et_ws_js_data1.js` is generated by `build-ws-js-data1-module` and gitignored, so @@ -151,6 +245,20 @@ fn kotlin_data1_pkg_built() -> bool { .exists() } +/// Probe for kotlin-math1's Kotlin/Wasm build output, logging a skip instead of failing when absent. +/// +/// Same shape as [`kotlin_data1_pkg_built`]: the `WasmGC` module and loader glue are generated by +/// `build-ws-kotlin-math1-module` and gitignored, so probe the loader and skip rather than 404 on the fetch. +#[expect( + clippy::single_call_fn, + reason = "distinct probe step; kept named for the skip-trace log line" +)] +fn kotlin_math1_pkg_built() -> bool { + edge_toolkit::config::get_project_root() + .join("services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1_compiled.mjs") + .exists() +} + /// Spawn two runners against one ws-server and assert both finish ok. /// Used by communication modules that need to discover at least one peer /// via `et-list-agents` before they can complete (comm1, dart-comm1). @@ -259,6 +367,7 @@ fn hardware_module_load_fails(#[case] module: &str, #[case] language: Language) #[rstest] #[case::rdata1("et-ws-rdata1", Language::R)] #[case::rcomm1("et-ws-rcomm1", Language::R)] +#[case::rmath1("et-ws-rmath1", Language::R)] fn r_module_load_fails(#[case] module: &str, #[case] language: Language) { if !mise_env_includes(language) { println!( From ca7f8a5079561e77290ea1d2d1023bd5eaabb973 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 17 Aug 2026 05:31:06 +0800 Subject: [PATCH 2/8] lint fixes --- .../rules/doc-summary-ends-with-period.yaml | 1 + .../dotnet-math1/pkg/et_ws_dotnet_math1.js | 12 +- .../java-math1/pkg/et_ws_java_math1.js | 12 +- .../src/main/java/au/edu/curtin/et/Math1.java | 4 +- .../kotlin-math1/pkg/et_ws_kotlin_math1.js | 12 +- .../ws-test-server/tests/math1_exchange.rs | 133 ++++++++++++++++++ 6 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 services/ws-test-server/tests/math1_exchange.rs diff --git a/config/ast-grep/rules/doc-summary-ends-with-period.yaml b/config/ast-grep/rules/doc-summary-ends-with-period.yaml index 125c6522..4a16353f 100644 --- a/config/ast-grep/rules/doc-summary-ends-with-period.yaml +++ b/config/ast-grep/rules/doc-summary-ends-with-period.yaml @@ -55,6 +55,7 @@ files: - services/ws-pyo3-runner/tests/modules.rs - services/ws-test-server/src/lib.rs - services/ws-test-server/src/math1.rs + - services/ws-test-server/tests/math1_exchange.rs - services/ws-server/src/config.rs - services/ws-server/src/lib.rs - services/ws-server/src/main.rs diff --git a/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js index b2d6c4d2..8d2e04e2 100644 --- a/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js +++ b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js @@ -17,6 +17,10 @@ export default async function init() { agentId = "", inputPointer = null; + // Ack/broadcast values feed the /storage/ fetch URLs below; accept only single, traversal-free + // path segments so a hostile frame cannot steer those requests. + const SAFE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + setModuleImports("dotnet-math1", { wsConnect: (url) => { ws = new WebSocket(url); @@ -28,8 +32,12 @@ export default async function init() { ws.onmessage = (e) => { try { const msg = JSON.parse(e.data); - if (msg.type === "et-connect-ack" && msg.agent_id) agentId = msg.agent_id; - if (msg.type === "math1-input" && msg.bucket && msg.filename) inputPointer = msg; + if (msg.type === "et-connect-ack" && SAFE_SEGMENT.test(msg.agent_id)) agentId = msg.agent_id; + const bucket = msg.bucket; + const filename = msg.filename; + if (msg.type === "math1-input" && SAFE_SEGMENT.test(bucket) && SAFE_SEGMENT.test(filename)) { + inputPointer = { bucket, filename }; + } } catch {} }; ws.onclose = ws.onerror = () => { diff --git a/services/ws-modules/java-math1/pkg/et_ws_java_math1.js b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js index 9e6105a8..870ac5ce 100644 --- a/services/ws-modules/java-math1/pkg/et_ws_java_math1.js +++ b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js @@ -15,6 +15,10 @@ export default async function init() { inputPointer = null, input = null; + // Ack/broadcast values feed the /storage/ fetch URLs below; accept only single, traversal-free + // path segments so a hostile frame cannot steer those requests. + const SAFE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + // TeaVM @JSBody calls reference `host` as a global globalThis.host = { wsConnect: (url) => { @@ -27,8 +31,12 @@ export default async function init() { ws.onmessage = (e) => { try { const msg = JSON.parse(e.data); - if (msg.type === "et-connect-ack" && msg.agent_id) agentId = msg.agent_id; - if (msg.type === "math1-input" && msg.bucket && msg.filename) inputPointer = msg; + if (msg.type === "et-connect-ack" && SAFE_SEGMENT.test(msg.agent_id)) agentId = msg.agent_id; + const bucket = msg.bucket; + const filename = msg.filename; + if (msg.type === "math1-input" && SAFE_SEGMENT.test(bucket) && SAFE_SEGMENT.test(filename)) { + inputPointer = { bucket, filename }; + } } catch {} }; ws.onclose = ws.onerror = () => { diff --git a/services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java b/services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java index d0847d71..45b013d1 100644 --- a/services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java +++ b/services/ws-modules/java-math1/src/main/java/au/edu/curtin/et/Math1.java @@ -126,7 +126,7 @@ private static void waitForInput(int attempt, JSConsumer resolve, JSCo } if (hasInput()) { loadInput().then(v -> { - computeAndStore(resolve, reject); + computeAndStore(resolve); return null; }); return; @@ -185,7 +185,7 @@ private static double[] fedAvg() { return new double[] {weight, bias}; } - private static void computeAndStore(JSConsumer resolve, JSConsumer reject) { + private static void computeAndStore(JSConsumer resolve) { status("running FedAvg - " + inputDescribe()); double[] model = fedAvg(); double weight = model[0]; diff --git a/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js b/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js index 1e36f0e0..67a33c8d 100644 --- a/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js +++ b/services/ws-modules/kotlin-math1/pkg/et_ws_kotlin_math1.js @@ -14,6 +14,10 @@ export default async function init() { inputPointer = null, input = null; + // Ack/broadcast values feed the /storage/ fetch URLs below; accept only single, traversal-free + // path segments so a hostile frame cannot steer those requests. + const SAFE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + // The Kotlin js() interop bridges reference `host` as a global globalThis.host = { wsConnect: (url) => { @@ -26,8 +30,12 @@ export default async function init() { ws.onmessage = (e) => { try { const msg = JSON.parse(e.data); - if (msg.type === "et-connect-ack" && msg.agent_id) agentId = msg.agent_id; - if (msg.type === "math1-input" && msg.bucket && msg.filename) inputPointer = msg; + if (msg.type === "et-connect-ack" && SAFE_SEGMENT.test(msg.agent_id)) agentId = msg.agent_id; + const bucket = msg.bucket; + const filename = msg.filename; + if (msg.type === "math1-input" && SAFE_SEGMENT.test(bucket) && SAFE_SEGMENT.test(filename)) { + inputPointer = { bucket, filename }; + } } catch {} }; ws.onclose = ws.onerror = () => { diff --git a/services/ws-test-server/tests/math1_exchange.rs b/services/ws-test-server/tests/math1_exchange.rs new file mode 100644 index 00000000..e7293ed5 --- /dev/null +++ b/services/ws-test-server/tests/math1_exchange.rs @@ -0,0 +1,133 @@ +//! Exercise the math1 fake-agent driver's success and error paths against an in-process server. +//! +//! The runner integration suites only ever see the happy path (a real module answers), so this +//! file drives [`et_ws_test_server::math1`] directly: a second ws client plays the module's role +//! by writing `math1-output.json` shapes straight into its own storage bucket, and the driver's +//! transport, timeout, parse, and verification branches are asserted one by one. + +#![cfg(test)] + +use std::time::Duration; + +use et_ws_test_server::math1::{ + MATH1_EXPECTED_BIAS, MATH1_EXPECTED_WEIGHT, MATH1_OUTPUT_FILENAME, Math1Error, drive_math1_exchange, + verify_math1_model, +}; +use futures_util::SinkExt as _; +use tokio_tungstenite::tungstenite::Message; + +/// Budget generous enough for the driver to see the peer and its pre-written output. +const EXCHANGE_BUDGET: Duration = Duration::from_secs(30); + +#[test] +fn verify_accepts_the_expected_model() { + verify_math1_model(MATH1_EXPECTED_WEIGHT, MATH1_EXPECTED_BIAS).unwrap(); +} + +#[test] +fn verify_rejects_a_wrong_weight() { + let err = verify_math1_model(MATH1_EXPECTED_WEIGHT + 1.0, MATH1_EXPECTED_BIAS).unwrap_err(); + assert!(err.to_string().contains("weight"), "unexpected error: {err}"); +} + +#[test] +fn verify_rejects_a_wrong_bias() { + let err = verify_math1_model(MATH1_EXPECTED_WEIGHT, MATH1_EXPECTED_BIAS + 1.0).unwrap_err(); + assert!(err.to_string().contains("bias"), "unexpected error: {err}"); +} + +/// An unreachable server surfaces as the boxed transport variant (covers the `From` impl). +#[tokio::test(flavor = "current_thread")] +async fn unreachable_server_is_a_transport_error() { + let err = drive_math1_exchange("ws://127.0.0.1:9/ws", std::env::temp_dir().as_path(), EXCHANGE_BUDGET) + .await + .unwrap_err(); + assert!(matches!(err, Math1Error::Transport(_)), "unexpected error: {err}"); +} + +/// With no module ever answering, the driver keeps re-broadcasting until the budget expires. +#[tokio::test(flavor = "current_thread")] +async fn times_out_when_no_module_answers() { + let server = et_ws_test_server::start(); + let err = drive_math1_exchange(&server.ws_url, server.storage_dir.path(), Duration::from_millis(600)) + .await + .unwrap_err(); + assert!( + err.to_string().contains("timed out"), + "expected the timeout protocol error, got: {err}" + ); +} + +/// Happy path: a peer's stored output is found, parsed, and passes verification. +/// +/// The peer also relays noise (a foreign text frame and a binary frame) mid-exchange, so the +/// driver's ignore-arms for non-protocol traffic are exercised alongside the success path. +#[tokio::test(flavor = "current_thread")] +async fn reads_and_verifies_a_peer_output() { + let server = et_ws_test_server::start(); + // The peer plays the module: it registers on the hub and its bucket carries a valid output. + let (mut peer, peer_id) = et_ws_test_server::connect_agent(&server.ws_url).await; + let storage_dir = server.storage_dir.path().to_path_buf(); + let output = format!(r#"{{"module":"t","weight":{MATH1_EXPECTED_WEIGHT},"bias":{MATH1_EXPECTED_BIAS}}}"#); + let noise_then_output = tokio::spawn(async move { + // Let the fake agent connect and start draining, then relay noise before the output lands. + tokio::time::sleep(Duration::from_millis(600)).await; + peer.send(Message::Text(r#"{"type":"noise"}"#.to_string())) + .await + .unwrap(); + peer.send(Message::Binary(vec![1, 2, 3])).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let bucket = storage_dir.join(&peer_id); + fs_err::create_dir_all(&bucket).unwrap(); + fs_err::write(bucket.join(MATH1_OUTPUT_FILENAME), output).unwrap(); + peer // keep the peer registered until the exchange resolves + }); + let (weight, bias) = drive_math1_exchange(&server.ws_url, server.storage_dir.path(), EXCHANGE_BUDGET) + .await + .unwrap(); + verify_math1_model(weight, bias).unwrap(); + let _peer = noise_then_output.await.unwrap(); +} + +/// A non-JSON output file surfaces as the JSON variant. +#[tokio::test(flavor = "current_thread")] +async fn malformed_output_is_a_json_error() { + let server = et_ws_test_server::start(); + let (_peer, peer_id) = et_ws_test_server::connect_agent(&server.ws_url).await; + write_peer_output(&server, &peer_id, "not json"); + let err = drive_math1_exchange(&server.ws_url, server.storage_dir.path(), EXCHANGE_BUDGET) + .await + .unwrap_err(); + assert!(matches!(err, Math1Error::Json(_)), "unexpected error: {err}"); +} + +/// Output JSON without the model fields names the first missing field. +#[tokio::test(flavor = "current_thread")] +async fn output_without_weight_is_a_protocol_error() { + let server = et_ws_test_server::start(); + let (_peer, peer_id) = et_ws_test_server::connect_agent(&server.ws_url).await; + write_peer_output(&server, &peer_id, r#"{"module":"t"}"#); + let err = drive_math1_exchange(&server.ws_url, server.storage_dir.path(), EXCHANGE_BUDGET) + .await + .unwrap_err(); + assert!(err.to_string().contains("weight"), "unexpected error: {err}"); +} + +/// Output JSON with a weight but no bias trips the second field check. +#[tokio::test(flavor = "current_thread")] +async fn output_without_bias_is_a_protocol_error() { + let server = et_ws_test_server::start(); + let (_peer, peer_id) = et_ws_test_server::connect_agent(&server.ws_url).await; + write_peer_output(&server, &peer_id, r#"{"module":"t","weight":1.0}"#); + let err = drive_math1_exchange(&server.ws_url, server.storage_dir.path(), EXCHANGE_BUDGET) + .await + .unwrap_err(); + assert!(err.to_string().contains("bias"), "unexpected error: {err}"); +} + +/// Drop `content` into the peer's storage bucket as its `math1-output.json`. +fn write_peer_output(server: &et_ws_test_server::TestServer, peer_id: &str, content: &str) { + let bucket = server.storage_dir.path().join(peer_id); + fs_err::create_dir_all(&bucket).unwrap(); + fs_err::write(bucket.join(MATH1_OUTPUT_FILENAME), content).unwrap(); +} From d98a203ff69dbb9a5ab914d4ce39d24a2f7163a7 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 17 Aug 2026 07:31:26 +0800 Subject: [PATCH 3/8] README and lints --- README.md | 11 ++++++++++ .../rules/doc-summary-ends-with-period.yaml | 6 ++++++ config/ast-grep/rules/no-string-new.yaml | 12 +++++++++++ libs/otlp-mock/src/lib.rs | 18 +++++++++-------- libs/test-helpers/src/lib.rs | 4 ++-- services/storage/src/tty_image.rs | 2 +- services/ws-modules/comm1/src/lib.rs | 2 +- services/ws-modules/face-detection/src/lib.rs | 2 +- services/ws-modules/har1/src/lib.rs | 11 +++++----- services/ws-modules/sensor1/src/lib.rs | 4 ++-- services/ws-test-server/src/math1.rs | 4 ++-- services/ws-wasi-runner/src/host/ws.rs | 20 ++++++++++--------- 12 files changed, 65 insertions(+), 31 deletions(-) create mode 100644 config/ast-grep/rules/no-string-new.yaml diff --git a/README.md b/README.md index 4dc24bc3..0669e8cf 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,17 @@ Modules built as WASI Preview 2 components and run under wasmtime: Native CPython modules linked via [PyO3](https://pyo3.rs) -- used for workloads that need a real CPython runtime (e.g. PyTorch inference). +### The math1 family + +The `math1` modules are the same federated-learning demo implemented once per guest language (Rust, JavaScript, +Dart, Python, Kotlin, C#, Java, R, and Zig in the browser runner, plus WASI-component and native-CPython twins for +the other two runners). A test-harness "fake agent" uploads a canonical input file +([math1-input.json](services/ws-test-server/data/math1-input.json)) into the server's storage and broadcasts a +pointer to it over the hub; each module reads the input, runs the same FedAvg simulation -- rounds of local +gradient-descent epochs per simulated client, merged with a sample-count-weighted average, using only `+ - * /` on +IEEE-754 doubles -- and stores its resulting global model back to storage, where the test harness verifies that +every language produced bit-identical weights. + ## Root module The default UX in the web-browser is also a loadable module located in diff --git a/config/ast-grep/rules/doc-summary-ends-with-period.yaml b/config/ast-grep/rules/doc-summary-ends-with-period.yaml index 4a16353f..316e2764 100644 --- a/config/ast-grep/rules/doc-summary-ends-with-period.yaml +++ b/config/ast-grep/rules/doc-summary-ends-with-period.yaml @@ -28,6 +28,7 @@ files: - libs/edge-toolkit/tests/no_mise.rs - libs/edge-toolkit/tests/npm_mod.rs - libs/otlp-emit/src/lib.rs + - libs/otlp-mock/src/lib.rs - libs/test-helpers/src/lib.rs - libs/ws-runner-common/tests/config.rs - services/modules/tests/api_modules.rs @@ -42,12 +43,16 @@ files: - services/storage/tests/s3_backend.rs - services/websockify/src/lib.rs - services/websockify/tests/relay.rs + - services/ws-modules/comm1/src/lib.rs - services/ws-modules/except1/src/lib.rs + - services/ws-modules/face-detection/src/lib.rs + - services/ws-modules/har1/src/lib.rs - services/ws-modules/math1/src/lib.rs - services/ws-modules/pic-viewer/build.rs - services/ws-modules/pic-viewer/src/lib.rs - services/ws-modules/pic-viewer/tests/parse.rs - services/ws-modules/pic-viewer/tests/show_image.rs + - services/ws-modules/sensor1/src/lib.rs - services/ws-modules/wasi-comm1/src/lib.rs - services/ws-modules/wasi-data1/src/lib.rs - services/ws-modules/wasi-math1/src/coverage.rs @@ -67,6 +72,7 @@ files: - services/ws-wasi-runner/src/bindings.rs - services/ws-wasi-runner/src/host/error.rs - services/ws-wasi-runner/src/host/mod.rs + - services/ws-wasi-runner/src/host/ws.rs - services/ws-wasi-runner/src/lib.rs - services/ws-wasi-runner/tests/modules.rs - services/ws-wasi-runner/tests/o2_ingest_search.rs diff --git a/config/ast-grep/rules/no-string-new.yaml b/config/ast-grep/rules/no-string-new.yaml new file mode 100644 index 00000000..600b1ec6 --- /dev/null +++ b/config/ast-grep/rules/no-string-new.yaml @@ -0,0 +1,12 @@ +id: no-string-new +language: Rust +severity: error +message: | + `String::new()` is banned (DeepSource RS-W1079): construct empty strings with `String::default()` instead, so + the zero value routes through the `Default` impl like every other default-constructed value in the codebase. +rule: + pattern: String::new() +# Mechanical autofix: the two constructors are documented to produce the identical empty string. +fix: String::default() +ignores: + - generated/** diff --git a/libs/otlp-mock/src/lib.rs b/libs/otlp-mock/src/lib.rs index ad4829b0..969c2d34 100644 --- a/libs/otlp-mock/src/lib.rs +++ b/libs/otlp-mock/src/lib.rs @@ -54,9 +54,10 @@ pub struct OtlpMock { } impl OtlpMock { - /// Pass this to `OTLP_COLLECTOR_URL` in env so OTLP exporters target - /// the mock. Trace endpoint is `/traces`; logs is - /// `/logs` -- matches `et_otlp::init`'s URL convention. + /// Pass this to `OTLP_COLLECTOR_URL` in env so OTLP exporters target the mock. + /// + /// Trace endpoint is `/traces`; logs is `/logs` -- matches + /// `et_otlp::init`'s URL convention. #[must_use] pub fn collector_url(&self) -> &str { &self.collector_url @@ -68,10 +69,11 @@ impl OtlpMock { self.captured.logs.lock().unwrap().clone() } - /// Walk every span across every captured request, returning each span with - /// its parent `Resource`'s `service.name` attribute (so the test can group - /// spans by service). Trace/span ids are lowercase-hex-encoded from the - /// decoded bytes -- compare against [`to_hex`] of the raw ids you sent. + /// Walk every span across every captured request, pairing each with its service name. + /// + /// The name is the parent `Resource`'s `service.name` attribute (so the test can group spans + /// by service). Trace/span ids are lowercase-hex-encoded from the decoded bytes -- compare + /// against [`to_hex`] of the raw ids you sent. #[must_use] pub fn flatten_spans(&self) -> Vec { let mut out = Vec::new(); @@ -171,7 +173,7 @@ fn sum_number_points(points: &[opentelemetry_proto::tonic::metrics::v1::NumberDa #[must_use] pub fn to_hex(bytes: &[u8]) -> String { use std::fmt::Write as _; - let mut out = String::new(); + let mut out = String::default(); for byte in bytes { // Writing to a String is infallible; unwrap is allowed crate-wide. write!(out, "{byte:02x}").unwrap(); diff --git a/libs/test-helpers/src/lib.rs b/libs/test-helpers/src/lib.rs index d1a61c79..faa767e5 100644 --- a/libs/test-helpers/src/lib.rs +++ b/libs/test-helpers/src/lib.rs @@ -112,10 +112,10 @@ fn drain_pipe(pipe: Pipe) -> Arc> where Pipe: std::io::Read + Send + 'static, { - let log = Arc::new(Mutex::new(String::new())); + let log = Arc::new(Mutex::new(String::default())); let sink = Arc::clone(&log); let _drainer = std::thread::spawn(move || { - let mut buffer = String::new(); + let mut buffer = String::default(); let _read = std::io::BufReader::new(pipe).read_to_string(&mut buffer); *sink.lock().unwrap() = buffer; }); diff --git a/services/storage/src/tty_image.rs b/services/storage/src/tty_image.rs index c65ce66b..bd10e17a 100644 --- a/services/storage/src/tty_image.rs +++ b/services/storage/src/tty_image.rs @@ -59,7 +59,7 @@ pub fn render_bytes(bytes: &[u8]) -> image::ImageResult<()> { .resize_exact(TARGET_COLUMNS, sample_height, image::imageops::FilterType::Triangle) .to_rgba8(); - let mut out = String::new(); + let mut out = String::default(); for row in 0..rows { let top_y = row.saturating_mul(2); let bottom_y = top_y.saturating_add(1); diff --git a/services/ws-modules/comm1/src/lib.rs b/services/ws-modules/comm1/src/lib.rs index 2e5771d8..33f1797b 100644 --- a/services/ws-modules/comm1/src/lib.rs +++ b/services/ws-modules/comm1/src/lib.rs @@ -32,7 +32,7 @@ pub async fn run() -> Result<(), JsValue> { log(&format!("comm1: resolved websocket URL: {ws_url}")); let mut client = WsClient::new(WsClientConfig::new(ws_url)); - let self_agent_id = Rc::new(RefCell::new(String::new())); + let self_agent_id = Rc::new(RefCell::new(String::default())); let other_connected_agents: Rc>> = Rc::new(RefCell::new(Vec::new())); let on_message_boxed: Box = Box::new({ diff --git a/services/ws-modules/face-detection/src/lib.rs b/services/ws-modules/face-detection/src/lib.rs index 29a6823c..01ff51d6 100644 --- a/services/ws-modules/face-detection/src/lib.rs +++ b/services/ws-modules/face-detection/src/lib.rs @@ -420,7 +420,7 @@ fn update_face_status(input_name: &str, output_names: &[String], summary: &Detec ]; if let Some(best) = summary.detections.first() { - lines.push(String::new()); + lines.push(String::default()); lines.push(format!( "best box: {:.1}, {:.1}, {:.1}, {:.1}", best.box_coords[0], best.box_coords[1], best.box_coords[2], best.box_coords[3] diff --git a/services/ws-modules/har1/src/lib.rs b/services/ws-modules/har1/src/lib.rs index 0f848a9c..2b890a9c 100644 --- a/services/ws-modules/har1/src/lib.rs +++ b/services/ws-modules/har1/src/lib.rs @@ -517,7 +517,7 @@ fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { "updated: {}", String::from(js_sys::Date::new_0().to_locale_time_string("en-US")) ), - String::new(), + String::default(), String::from("orientation"), ]; @@ -530,7 +530,7 @@ fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { lines.push(String::from("waiting for orientation event...")); } - lines.push(String::new()); + lines.push(String::default()); lines.push(String::from("motion")); if let Some(motion) = motion { lines.push(format!( @@ -791,9 +791,10 @@ fn create_feat_tensor(values: &[f32]) -> Result { Reflect::construct(&tensor_ctor, &args) } -/// Compute 36 hand-crafted features from the sample buffer: -/// 8 channels x 4 stats (mean, std, min, max) = 32, plus 4 stats on the -/// per-sample vector magnitude (mean, std, min, max) = 36 total. +/// Compute 36 hand-crafted features from the sample buffer. +/// +/// 8 channels x 4 stats (mean, std, min, max) = 32, plus 4 stats on the per-sample vector +/// magnitude (mean, std, min, max) = 36 total. fn compute_feat_input(sample_buffer: &VecDeque<[f32; HAR_FEATURE_COUNT]>) -> [f32; HAR_FEAT_INPUT_SIZE] { let sample_count = sample_buffer.len() as f32; let mut out = [0.0f32; HAR_FEAT_INPUT_SIZE]; diff --git a/services/ws-modules/sensor1/src/lib.rs b/services/ws-modules/sensor1/src/lib.rs index f4526e1a..c2f09c1f 100644 --- a/services/ws-modules/sensor1/src/lib.rs +++ b/services/ws-modules/sensor1/src/lib.rs @@ -422,7 +422,7 @@ fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { "updated: {}", String::from(js_sys::Date::new_0().to_locale_time_string("en-US")) ), - String::new(), + String::default(), String::from("orientation"), ]; @@ -435,7 +435,7 @@ fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { lines.push(String::from("waiting for orientation event...")); } - lines.push(String::new()); + lines.push(String::default()); lines.push(String::from("motion")); if let Some(motion) = motion { lines.push(format!( diff --git a/services/ws-test-server/src/math1.rs b/services/ws-test-server/src/math1.rs index 3ab4ffb7..ec1c95d0 100644 --- a/services/ws-test-server/src/math1.rs +++ b/services/ws-test-server/src/math1.rs @@ -79,9 +79,9 @@ pub async fn drive_math1_exchange( socket.send(Message::Text(connect)).await?; let deadline = tokio::time::Instant::now() + budget; - let mut fake_id = String::new(); + let mut fake_id = String::default(); let mut peers: Vec = Vec::new(); - let mut pointer = String::new(); + let mut pointer = String::default(); loop { if tokio::time::Instant::now() >= deadline { diff --git a/services/ws-wasi-runner/src/host/ws.rs b/services/ws-wasi-runner/src/host/ws.rs index b803a404..7d43ae51 100644 --- a/services/ws-wasi-runner/src/host/ws.rs +++ b/services/ws-wasi-runner/src/host/ws.rs @@ -43,8 +43,9 @@ use crate::bindings::et::ws_wasi::ws::WsError; use crate::host::error::WsDecodeErrExt as _; use crate::host::error::WsTransportErrExt as _; -/// Live state for an open websocket connection. Owned by `HostState` behind a -/// `Mutex`; replaced on disconnect. +/// Live state for an open websocket connection. +/// +/// Owned by `HostState` behind a `Mutex`; replaced on disconnect. pub struct WsBackend { sink: Arc>, inbox: Arc>>, @@ -186,7 +187,7 @@ impl Host for HostState { let slot = self.ws.lock().await; match slot.as_ref() { Some(bridge) => bridge.current_agent_id().await, - None => String::new(), + None => String::default(), } } @@ -251,9 +252,10 @@ impl Host for HostState { } } -/// Convert a guest-emitted WIT `client-message` into the canonical Rust -/// `ClientMessage`. Opaque JSON fields (sent as `string` over WIT) are parsed -/// here so the host always works with `serde_json::Value` payloads. +/// Convert a guest-emitted WIT `client-message` into the canonical Rust `ClientMessage`. +/// +/// Opaque JSON fields (sent as `string` over WIT) are parsed here so the host always works with +/// `serde_json::Value` payloads. #[expect( clippy::single_call_fn, reason = "named converter; used once by ::send" @@ -295,9 +297,9 @@ fn wit_to_client_message(msg: WitClientMessage) -> Result Date: Mon, 17 Aug 2026 08:52:49 +0800 Subject: [PATCH 4/8] coverage --- .github/workflows/test.yaml | 8 ++++ .../rules/doc-summary-ends-with-period.yaml | 1 + services/storage/src/lib.rs | 2 +- services/storage/src/tty_image.rs | 4 -- services/storage/tests/tty_render.rs | 22 +++++++++ .../ws-test-server/tests/math1_exchange.rs | 47 ++++++++++++++++++- 6 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 services/storage/tests/tty_render.rs diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2b52326d..d00090a0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,6 +18,10 @@ name: test - macos-26-intel - windows-latest - windows-11-arm + windows-overrides: + description: "Also run the msvc + mingw override lanes" + type: boolean + default: false permissions: contents: read @@ -157,6 +161,10 @@ jobs: # `mingw` links that msvc archive into an x86_64-pc-windows-gnu binary with winlibs GCC. Separate from # the `default` job because they override the gnullvm default target rather than exercising it. override: + # Always on for pull requests; opt-in on manual dispatches. + # The windows-overrides checkbox gates these lanes so a dispatch aimed at one OS doesn't also spawn two + # long Windows jobs. + if: github.event_name != 'workflow_dispatch' || inputs.windows-overrides # Read-only pull of the maintainer-published ghcr.io mise-tools store (install-mise-tools restore step). # Never write: the stores are published only from a maintainer's machine via `mise run push-mise-tools`. permissions: diff --git a/config/ast-grep/rules/doc-summary-ends-with-period.yaml b/config/ast-grep/rules/doc-summary-ends-with-period.yaml index 316e2764..4fdf508a 100644 --- a/config/ast-grep/rules/doc-summary-ends-with-period.yaml +++ b/config/ast-grep/rules/doc-summary-ends-with-period.yaml @@ -41,6 +41,7 @@ files: - services/storage/tests/image_logging.rs - services/storage/tests/put.rs - services/storage/tests/s3_backend.rs + - services/storage/tests/tty_render.rs - services/websockify/src/lib.rs - services/websockify/tests/relay.rs - services/ws-modules/comm1/src/lib.rs diff --git a/services/storage/src/lib.rs b/services/storage/src/lib.rs index d266fd1e..87055ba5 100644 --- a/services/storage/src/lib.rs +++ b/services/storage/src/lib.rs @@ -18,7 +18,7 @@ use serde_default::DefaultFromSerde; use thiserror::Error; pub mod routes; -mod tty_image; +pub mod tty_image; pub use self::routes::{get_file, head_file, put_file}; diff --git a/services/storage/src/tty_image.rs b/services/storage/src/tty_image.rs index bd10e17a..ff81842e 100644 --- a/services/storage/src/tty_image.rs +++ b/services/storage/src/tty_image.rs @@ -26,10 +26,6 @@ const TARGET_COLUMNS: u32 = 48; /// /// Returns the underlying `image` decode error on failure; the caller decides how to report it (this module /// stays IO-boundary-agnostic rather than picking a logging mechanism itself). -#[expect( - clippy::single_call_fn, - reason = "distinct step of show_image_on_tty; kept separate for readability and testing" -)] pub fn render_bytes(bytes: &[u8]) -> image::ImageResult<()> { let source = image::load_from_memory(bytes)?; if source.width() == 0 || source.height() == 0 { diff --git a/services/storage/tests/tty_render.rs b/services/storage/tests/tty_render.rs new file mode 100644 index 00000000..5793abb7 --- /dev/null +++ b/services/storage/tests/tty_render.rs @@ -0,0 +1,22 @@ +//! Exercise `tty_image::render_bytes`: the ANSI half-block render loop and its decode-error path. + +#![cfg(test)] + +use et_storage_service::tty_image::render_bytes; + +#[test] +fn renders_a_small_image_to_ansi() { + // Encode a tiny RGBA image in memory; the renderer decodes it back and walks the full + // half-block loop (aspect-scaled resize, per-cell truecolor SGR emit, single stdout write). + let img = image::RgbaImage::from_fn(3, 2, |_x, _y| image::Rgba([200, 100, 50, 255])); + let mut png = Vec::new(); + image::DynamicImage::ImageRgba8(img) + .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) + .unwrap(); + render_bytes(&png).unwrap(); +} + +#[test] +fn rejects_undecodable_bytes() { + assert!(render_bytes(b"not an image").is_err()); +} diff --git a/services/ws-test-server/tests/math1_exchange.rs b/services/ws-test-server/tests/math1_exchange.rs index e7293ed5..9fc3260c 100644 --- a/services/ws-test-server/tests/math1_exchange.rs +++ b/services/ws-test-server/tests/math1_exchange.rs @@ -13,7 +13,7 @@ use et_ws_test_server::math1::{ MATH1_EXPECTED_BIAS, MATH1_EXPECTED_WEIGHT, MATH1_OUTPUT_FILENAME, Math1Error, drive_math1_exchange, verify_math1_model, }; -use futures_util::SinkExt as _; +use futures_util::{SinkExt as _, StreamExt as _}; use tokio_tungstenite::tungstenite::Message; /// Budget generous enough for the driver to see the peer and its pre-written output. @@ -45,6 +45,51 @@ async fn unreachable_server_is_a_transport_error() { assert!(matches!(err, Math1Error::Transport(_)), "unexpected error: {err}"); } +/// A server that closes the websocket cleanly mid-exchange surfaces as the socket-closed error. +#[tokio::test(flavor = "current_thread")] +async fn server_close_is_a_protocol_error() { + let ws_url = accept_one_connection_then(true).await; + let err = drive_math1_exchange(&ws_url, std::env::temp_dir().as_path(), EXCHANGE_BUDGET) + .await + .unwrap_err(); + assert!( + err.to_string().contains("socket closed"), + "expected the socket-closed protocol error, got: {err}" + ); +} + +/// A server that drops the TCP stream without a close handshake surfaces as a transport error. +#[tokio::test(flavor = "current_thread")] +async fn abrupt_server_drop_is_a_transport_error() { + let ws_url = accept_one_connection_then(false).await; + let err = drive_math1_exchange(&ws_url, std::env::temp_dir().as_path(), EXCHANGE_BUDGET) + .await + .unwrap_err(); + assert!(matches!(err, Math1Error::Transport(_)), "unexpected error: {err}"); +} + +/// Accept exactly one ws connection, consume its first frame, then close it (gracefully or not). +/// +/// Returns the `ws://` URL to hand to the driver. The close style picks which driver arm trips: +/// a clean close handshake ends the stream (`None`), an abrupt TCP drop yields a protocol `Err`. +async fn accept_one_connection_then(close_gracefully: bool) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let _server = tokio::spawn(async move { + let (stream, _peer) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + // Consume the driver's et-connect frame so its send completes before the teardown. + let _frame = socket.next().await; + if close_gracefully { + socket.close(None).await.unwrap(); + // Hold the stream open long enough for the close handshake to reach the driver. + tokio::time::sleep(Duration::from_millis(200)).await; + } + // Dropping the socket here tears the TCP stream down without a close handshake. + }); + format!("ws://{addr}/ws") +} + /// With no module ever answering, the driver keeps re-broadcasting until the budget expires. #[tokio::test(flavor = "current_thread")] async fn times_out_when_no_module_answers() { From 760a51a66063a59e2867b8e6365757792f2ccb1c Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 17 Aug 2026 10:35:15 +0800 Subject: [PATCH 5/8] A few fixes --- .mise/config.maint.toml | 18 ++++++++++++++++-- .mise/config.toml | 9 ++++++++- README.md | 13 +++++++++++++ config/conftest/policy/mise/mise.rego | 5 +++++ config/semgrep/no-non-ascii.yaml | 9 +++++---- .../mise-cargo-backend-allowlist.schema.json | 1 + .../dart-comm1/pkg/et_ws_dart_comm1.js | 2 +- .../dotnet-data1/pkg/et_ws_dotnet_data1.js | 2 +- .../dotnet-math1/pkg/et_ws_dotnet_math1.js | 2 +- .../java-data1/pkg/et_ws_java_data1.js | 2 +- .../java-math1/pkg/et_ws_java_math1.js | 2 +- .../ws-modules/pydata1/pkg/et_ws_pydata1.js | 6 +++--- .../ws-modules/pydemo1/pkg/et_ws_pydemo1.js | 14 +++++++------- .../ws-modules/pyface1/pkg/et_ws_pyface1.js | 2 +- .../ws-modules/pymath1/pkg/et_ws_pymath1.js | 4 ++-- .../pyspeech1/pkg/et_ws_pyspeech1.js | 2 +- .../zig-data1/pkg/et_ws_zig_data1.js | 2 +- .../zig-data1/pkg/et_ws_zig_data1_worker.js | 4 ++-- .../zig-except1/pkg/et_ws_zig_except1.js | 2 +- .../pkg/et_ws_zig_except1_worker.js | 2 +- .../zig-math1/pkg/et_ws_zig_math1.js | 2 +- .../zig-math1/pkg/et_ws_zig_math1_worker.js | 2 +- 22 files changed, 74 insertions(+), 33 deletions(-) diff --git a/.mise/config.maint.toml b/.mise/config.maint.toml index 3112bd53..4fde6aed 100644 --- a/.mise/config.maint.toml +++ b/.mise/config.maint.toml @@ -514,10 +514,24 @@ shell = "bash -euo pipefail -c" # packages:write, so a compromised workflow cannot poison what every job then executes. `mise oci build` # emits one content-addressed layer per tool; crane does the upload, skipping blobs the registry already # holds, so re-publishing after a config bump uploads just the changed tools. Each store carries -# host-native binaries: run this on a machine of the platform being published. Needs a one-time -# `docker login ghcr.io` (or `crane auth login ghcr.io`) with a packages:write PAT, and installs the full +# host-native binaries: run this on a machine of the platform being published; it installs the full # language set first so the image matches what CI expects. # +# One-time auth setup (the push needs a GitHub token that can write packages): +# 1. Add the `write:packages` scope to the gh CLI's existing token: +# gh auth refresh -s write:packages +# 2. Log the local credential store into ghcr.io with that token, via docker: +# gh auth token | docker login ghcr.io -u "$(gh api user --jq .login)" --password-stdin +# or, on a machine without docker, via the crane this repo already installs: +# mise exec -E maint -- crane auth login ghcr.io -u "$(gh api user --jq .login)" -p "$(gh auth token)" +# The `-u` username must be NON-EMPTY (ghcr accepts any value alongside a token). A login without it +# stores an auth entry whose username half is blank, which docker/cli then refuses to parse back, and +# every later crane/docker call on that machine dies with +# +# Error: parsing config file (~/.docker/config.json): invalid auth configuration file +# +# (observed on a linux maintainer host). +# # crane pushes the layout rather than `mise oci push` because mise 2026.8.0's built-in registry client # trips GHCR's chunked blob-upload range rules on large layers and dies mid-push with # diff --git a/.mise/config.toml b/.mise/config.toml index 7119a279..124b20ae 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -61,7 +61,9 @@ action-validator = { version = "latest", os = ["linux", "macos"] } "aqua:koalaman/shellcheck" = "latest" "aqua:rhysd/actionlint" = "latest" "aqua:rustwasm/wasm-pack" = "latest" -"aqua:vectordotdev/vector" = "0.56.0" +# Vector ships no darwin/amd64 prebuilt, so second-tier macos/x64 builds the same pin from source below. +# The aqua error was: `unsupported env: darwin/amd64 (supported: ["linux", "darwin/arm64", "windows/amd64"])`. +"aqua:vectordotdev/vector" = { version = "0.56.0", os = ["linux", "macos/arm64", "windows"] } ast-grep = "latest" cargo-binstall = "latest" "cargo:cargo-expand" = { version = "latest", os = ["linux", "macos"] } @@ -106,6 +108,11 @@ ripgrep = "latest" "github:benhoyt/goawk" = "latest" "github:caldempsey/parfit" = "latest" "github:grok-rs/waitup" = "latest" +# The macos/x64 half of vector (see the aqua:vectordotdev/vector note above). +# A git `tag:` spec rather than the crates.io crate so this platform builds the exact source the other +# platforms' 0.56.0 prebuilts are cut from -- bump it with the aqua entry. A source build is acceptable here +# because macos/x64 is second-tier. +"cargo:vectordotdev/vector" = { version = "tag:v0.56.0", os = ["macos/x64"] } # The macos/x64 half of rustfs, the one platform upstream ships no prebuilt for. # A git spec rather than a bare `cargo:rustfs`: the crates.io crate is a stale 0.0.2 placeholder while real # releases are 1.0.0-beta.x on GitHub only, so installing the crate would give this platform different software diff --git a/README.md b/README.md index 0669e8cf..b23196c3 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,19 @@ mise install-all The `preinstall` task will advise if there are any required dependencies are are missing, such as Xcode Command Line Tools on MacOS. +### Optionally pre-populating tools from the OCI stores + +Most of the tool tree can be restored from this project's per-platform OCI packages on ghcr.io instead of +downloaded tool-by-tool from each upstream: + +```bash +mise run pull-mise-tools +``` + +This pre-populates the local mise data dir from `ghcr.io/edge-toolkit/core/mise-tools/-`, so the +`mise install` that follows only fills the gaps (the store carries relocatable tools only) and regenerates shims. +This is the same store CI restores from; it needs no authentication. + ### Install failures `mise install` runs tool installs in parallel. If they fail intermittently -- a download race, or a `cargo:` source diff --git a/config/conftest/policy/mise/mise.rego b/config/conftest/policy/mise/mise.rego index fe210216..d0af0f51 100644 --- a/config/conftest/policy/mise/mise.rego +++ b/config/conftest/policy/mise/mise.rego @@ -163,6 +163,11 @@ allowed_os_scoped_tool := { # windows_exporter is a Windows-only host/GPU Prometheus exporter, so the o2-winmetrics task scopes it to Windows. "github:prometheus-community/windows_exporter", "cargo:findutils", + # vector is the OTLP store-and-forward relay, covered by two per-platform entries. + # The aqua prebuilt covers every platform upstream ships (no darwin/amd64 asset exists); a cargo source + # build from the same git tag covers macos/x64. Between the two every platform is covered. + "aqua:vectordotdev/vector", + "cargo:vectordotdev/vector", # rustfs is the S3 server the storage backend test runs against, covered by two per-platform entries. # http: names an upstream asset for every platform that ships one; a cargo source build from the same git # tag covers macos/x64, which has no prebuilt at all. Between the two every platform is covered. diff --git a/config/semgrep/no-non-ascii.yaml b/config/semgrep/no-non-ascii.yaml index 4ec0fedc..8db5238f 100644 --- a/config/semgrep/no-non-ascii.yaml +++ b/config/semgrep/no-non-ascii.yaml @@ -4,13 +4,14 @@ rules: paths: exclude: # Generated / build-output trees regenerate non-ASCII from their tools. - # The source of truth there is the generator, not a hand edit: clap's HELP.md carries a jump glyph; wasm-pack - # pkg/ bundles and the AsyncAPI/OpenAPI/WIT output under generated/ plus the regen-verification output under - # verification/ all reintroduce it. + # The source of truth there is the generator, not a hand edit: clap's HELP.md carries a jump glyph and the + # AsyncAPI/OpenAPI/WIT output under generated/ plus the regen-verification output under verification/ all + # reintroduce it. Module pkg/ dirs are NOT excluded: their generated contents (wasm-pack bundles, wheels, + # compiled JS) are gitignored and semgrep scans only tracked files, while the committed shims in pkg/ are + # hand-written source the ban must cover -- a blanket pkg/ exclude once hid shim em-dashes for months. - "/generated/**" - "/verification/**" - "**/HELP.md" - - "**/pkg/**" # License texts carry non-ASCII legal glyphs and are not ours to edit. - "LICENSE*" # Binary assets. diff --git a/config/taplo/mise-cargo-backend-allowlist.schema.json b/config/taplo/mise-cargo-backend-allowlist.schema.json index 21696781..4047bc1d 100644 --- a/config/taplo/mise-cargo-backend-allowlist.schema.json +++ b/config/taplo/mise-cargo-backend-allowlist.schema.json @@ -18,6 +18,7 @@ "cargo:open", "cargo:rustfs/rustfs", "cargo:ryl", + "cargo:vectordotdev/vector", "cargo:wasm-opt" ] } diff --git a/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js b/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js index 9d32d5ee..26192504 100644 --- a/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js +++ b/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js @@ -1,4 +1,4 @@ -// et_ws_dart_comm1.js — ES module shim for dart-comm1 +// et_ws_dart_comm1.js -- ES module shim for dart-comm1 export default async function init() { await new Promise((resolve, reject) => { diff --git a/services/ws-modules/dotnet-data1/pkg/et_ws_dotnet_data1.js b/services/ws-modules/dotnet-data1/pkg/et_ws_dotnet_data1.js index 527c1d24..07494161 100644 --- a/services/ws-modules/dotnet-data1/pkg/et_ws_dotnet_data1.js +++ b/services/ws-modules/dotnet-data1/pkg/et_ws_dotnet_data1.js @@ -1,4 +1,4 @@ -// et_ws_dotnet_data1.js — .NET WASM shim for dotnet-data1 +// et_ws_dotnet_data1.js -- .NET WASM shim for dotnet-data1 // Interface: default(), run() let exports = null; diff --git a/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js index 8d2e04e2..c24aebc9 100644 --- a/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js +++ b/services/ws-modules/dotnet-math1/pkg/et_ws_dotnet_math1.js @@ -1,4 +1,4 @@ -// et_ws_dotnet_math1.js — .NET WASM shim for dotnet-math1 +// et_ws_dotnet_math1.js -- .NET WASM shim for dotnet-math1 // Interface: default(), run() // // Storage-driven FedAvg: the shim owns the browser I/O -- the WebSocket (including capturing the diff --git a/services/ws-modules/java-data1/pkg/et_ws_java_data1.js b/services/ws-modules/java-data1/pkg/et_ws_java_data1.js index 040cadd4..1b22aa0d 100644 --- a/services/ws-modules/java-data1/pkg/et_ws_java_data1.js +++ b/services/ws-modules/java-data1/pkg/et_ws_java_data1.js @@ -1,4 +1,4 @@ -// et_ws_java_data1.js — TeaVM JS shim for java-data1 +// et_ws_java_data1.js -- TeaVM JS shim for java-data1 // Interface: default(), run() let javaRun = null; diff --git a/services/ws-modules/java-math1/pkg/et_ws_java_math1.js b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js index 870ac5ce..b38f741d 100644 --- a/services/ws-modules/java-math1/pkg/et_ws_java_math1.js +++ b/services/ws-modules/java-math1/pkg/et_ws_java_math1.js @@ -1,4 +1,4 @@ -// et_ws_java_math1.js — TeaVM JS shim for java-math1 +// et_ws_java_math1.js -- TeaVM JS shim for java-math1 // Interface: default(), run() // // Storage-driven FedAvg: the shim owns the browser I/O -- the WebSocket (including capturing the diff --git a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js index 3160b8b9..da54a161 100644 --- a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js +++ b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js @@ -1,4 +1,4 @@ -// et_ws_pydata1.js — Pyodide-based Python module shim +// et_ws_pydata1.js -- Pyodide-based Python module shim // Interface: default(wasmUrl), metadata(), run() const PYODIDE_BASE_PATH = "/modules/pyodide/"; @@ -40,13 +40,13 @@ export default async function init() { await loadPyodideScript(); // `mise install pyodide` extracts the full GitHub-release distribution // (~200 MB of pinned wheels) at /modules/pyodide/, so both the runtime - // and `micropip.install("httpx")` resolve from this same origin — no CDN + // and `micropip.install("httpx")` resolve from this same origin -- no CDN // dependency at runtime. pyodide = await globalThis.loadPyodide({ indexURL: PYODIDE_BASE_PATH }); // pydata1's runtime stack: PyPI deps via micropip (httpx + attrs power // the generated client; pyodide-http rewires httpx to use the browser's - // fetch()), plus two local wheels — pydata1 itself (next to this shim) + // fetch()), plus two local wheels -- pydata1 itself (next to this shim) // and the generated et-rest-client wheel served by its own ws-module // mount at /modules/et-rest-client/. Going through micropip for the // local wheels would make it look up "et-rest-client" on PyPI, which we diff --git a/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js b/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js index 18ccfa21..7fd6a1f7 100644 --- a/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js +++ b/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js @@ -133,7 +133,7 @@ async function enterDemo(state) { demo.exitButton.addEventListener("click", () => window.history.back()); try { - setLoadingMessage(demo, py ? "Local Python runtime ready" : "Preparing the local Python runtime…"); + setLoadingMessage(demo, py ? "Local Python runtime ready" : "Preparing the local Python runtime..."); await loadPythonRuntime(); if (demo.stopped) return; await py.run(platformFor(state, demo)); @@ -265,7 +265,7 @@ function createDemoView(captureSeconds, uploadConsent) { "border:0", ]); const speechStatus = document.createElement("span"); - speechStatus.textContent = "Preparing microphone…"; + speechStatus.textContent = "Preparing microphone..."; speechStatus.style.cssText = "position:absolute;left:22px;bottom:16px;color:#adc8c9;font-size:15px"; speechPanel.append(speechCanvas, speechBadge, countdown, speechStatus); @@ -441,7 +441,7 @@ function createLoadingScreen() { title.textContent = "Preparing local AI"; title.style.cssText = "display:block;color:#effffb;font-size:17px"; const message = document.createElement("span"); - message.textContent = "Starting the demo…"; + message.textContent = "Starting the demo..."; message.style.cssText = "display:block;margin-top:6px;color:#94bab8;font-size:13px"; const style = document.createElement("style"); style.textContent = "@keyframes pydemo1-spin{to{transform:rotate(360deg)}}"; @@ -503,7 +503,7 @@ function platformFor(state, demo) { } async function loadModels(demo) { - setLoadingMessage(demo, "Loading eye and speech detection models…"); + setLoadingMessage(demo, "Loading eye and speech detection models..."); [demo.landmarker, demo.speechSession] = await initializeModels(demo); } @@ -512,8 +512,8 @@ async function initializeModels(demo) { let eyeReady = false; let speechReady = false; const reportModelProgress = () => { - if (eyeReady && !speechReady) setLoadingMessage(demo, "Eye model ready; finishing the speech model…"); - if (speechReady && !eyeReady) setLoadingMessage(demo, "Speech model ready; finishing the eye model…"); + if (eyeReady && !speechReady) setLoadingMessage(demo, "Eye model ready; finishing the speech model..."); + if (speechReady && !eyeReady) setLoadingMessage(demo, "Speech model ready; finishing the eye model..."); }; const eyeModel = (async () => { const vision = await import(cfg.eye.bundle_path); @@ -587,7 +587,7 @@ async function runSpeechDetection(state, demo) { demo.phase = "COMPLETE"; demo.countdown.textContent = "Complete"; demo.speechStatus.textContent = result.speech_detected - ? `Speech detected · ${(result.confidence * 100).toFixed(1)}% peak confidence` + ? `Speech detected - ${(result.confidence * 100).toFixed(1)}% peak confidence` : "No speech detected"; demo.speechStatus.hidden = result.speech_detected; state.client?.send?.(result.event_json); diff --git a/services/ws-modules/pyface1/pkg/et_ws_pyface1.js b/services/ws-modules/pyface1/pkg/et_ws_pyface1.js index 55ca49d2..eab6a487 100644 --- a/services/ws-modules/pyface1/pkg/et_ws_pyface1.js +++ b/services/ws-modules/pyface1/pkg/et_ws_pyface1.js @@ -51,7 +51,7 @@ export default async function init() { await installLocalWheel(pyfaceWheel); // The generated et-ws Pydantic-models wheel is its own ws-module mounted // at /modules/et-ws/. We declare it in [tool.ws-module.dependencies] and - // delegate wheel install to its shim — version lives in its own + // delegate wheel install to its shim -- version lives in its own // package.json so a bump there doesn't require touching this file. const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); await installEtWs(pyodide); diff --git a/services/ws-modules/pymath1/pkg/et_ws_pymath1.js b/services/ws-modules/pymath1/pkg/et_ws_pymath1.js index bad9f7f6..5fa7340e 100644 --- a/services/ws-modules/pymath1/pkg/et_ws_pymath1.js +++ b/services/ws-modules/pymath1/pkg/et_ws_pymath1.js @@ -1,4 +1,4 @@ -// et_ws_pymath1.js — Pyodide-based Python module shim +// et_ws_pymath1.js -- Pyodide-based Python module shim // Interface: default() (init), run() // // Storage-driven FedAvg: the shim owns the browser I/O (WebSocket, the math1-input pointer @@ -25,7 +25,7 @@ function loadPyodideScript() { export default async function init() { await loadPyodideScript(); // The full Pyodide distribution is served at /modules/pyodide/, so the runtime resolves from this - // same origin — no CDN dependency. pymath1 has no PyPI deps: its FedAvg kernel is stdlib-only, so + // same origin -- no CDN dependency. pymath1 has no PyPI deps: its FedAvg kernel is stdlib-only, so // the only wheel to load is its own, served next to this shim. pyodide = await globalThis.loadPyodide({ indexURL: PYODIDE_BASE_PATH }); diff --git a/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js b/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js index 9e3cfca6..7b4d8170 100644 --- a/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js +++ b/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js @@ -145,7 +145,7 @@ async function recordAndDetect(state) { button.style.cursor = "wait"; button.style.opacity = "0.72"; button.dataset.state = "recording"; - button.querySelector("span:last-child").textContent = "Recording…"; + button.querySelector("span:last-child").textContent = "Recording..."; setStatus(py.starting_status()); try { diff --git a/services/ws-modules/zig-data1/pkg/et_ws_zig_data1.js b/services/ws-modules/zig-data1/pkg/et_ws_zig_data1.js index 5a1f6514..f49c19ca 100644 --- a/services/ws-modules/zig-data1/pkg/et_ws_zig_data1.js +++ b/services/ws-modules/zig-data1/pkg/et_ws_zig_data1.js @@ -1,4 +1,4 @@ -// et_ws_zig_data1.js — zig-data1 WASM module +// et_ws_zig_data1.js -- zig-data1 WASM module // Runs WASM in a Web Worker; main thread proxies WebSocket + fetch via // SharedArrayBuffer. Shared memory layout (Int32 offsets): // [0] signal: 0=idle, 1=request-pending diff --git a/services/ws-modules/zig-data1/pkg/et_ws_zig_data1_worker.js b/services/ws-modules/zig-data1/pkg/et_ws_zig_data1_worker.js index 728a9063..0d935392 100644 --- a/services/ws-modules/zig-data1/pkg/et_ws_zig_data1_worker.js +++ b/services/ws-modules/zig-data1/pkg/et_ws_zig_data1_worker.js @@ -1,4 +1,4 @@ -// et_ws_zig_data1_worker.js — Web Worker for zig-data1 WASM module +// et_ws_zig_data1_worker.js -- Web Worker for zig-data1 WASM module const DATA_OFFSET = 16; let ctrl, data, wasmMemory; const enc = new TextEncoder(), @@ -37,7 +37,7 @@ function callRest(method, url, body) { Atomics.notify(ctrl, 0); Atomics.wait(ctrl, 0, 1); const rlen = Atomics.load(ctrl, 2); - // Negative response length is the error sentinel — the main-thread + // Negative response length is the error sentinel -- the main-thread // dispatch encodes (max int32 + 1 - n) to signal failure. if (rlen < 0) return null; // Slice copies the bytes out of the SAB region so the wasm caller can diff --git a/services/ws-modules/zig-except1/pkg/et_ws_zig_except1.js b/services/ws-modules/zig-except1/pkg/et_ws_zig_except1.js index ee559f7b..eaa0c1b6 100644 --- a/services/ws-modules/zig-except1/pkg/et_ws_zig_except1.js +++ b/services/ws-modules/zig-except1/pkg/et_ws_zig_except1.js @@ -1,4 +1,4 @@ -// et_ws_zig_except1.js — zig-except1 WASM module +// et_ws_zig_except1.js -- zig-except1 WASM module // Runs WASM in a Web Worker; main thread proxies WebSocket calls via // SharedArrayBuffer. Shared memory layout (Int32 offsets): // [0] signal: 0=idle, 1=request-pending diff --git a/services/ws-modules/zig-except1/pkg/et_ws_zig_except1_worker.js b/services/ws-modules/zig-except1/pkg/et_ws_zig_except1_worker.js index 7e425c4a..5093cdfa 100644 --- a/services/ws-modules/zig-except1/pkg/et_ws_zig_except1_worker.js +++ b/services/ws-modules/zig-except1/pkg/et_ws_zig_except1_worker.js @@ -1,4 +1,4 @@ -// et_ws_zig_except1_worker.js — Web Worker for zig-except1 WASM module +// et_ws_zig_except1_worker.js -- Web Worker for zig-except1 WASM module const DATA_OFFSET = 16; let ctrl, data, wasmMemory; const enc = new TextEncoder(), diff --git a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js index 809fc3b1..3a6ef28a 100644 --- a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js +++ b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1.js @@ -1,4 +1,4 @@ -// et_ws_zig_math1.js — zig-math1 WASM module +// et_ws_zig_math1.js -- zig-math1 WASM module // Runs WASM in a Web Worker; main thread proxies WebSocket + REST calls via // SharedArrayBuffer. Shared memory layout (Int32 offsets): // [0] signal: 0=idle, 1=request-pending diff --git a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js index 452954b1..805f9739 100644 --- a/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js +++ b/services/ws-modules/zig-math1/pkg/et_ws_zig_math1_worker.js @@ -1,4 +1,4 @@ -// et_ws_zig_math1_worker.js — Web Worker for zig-math1 WASM module +// et_ws_zig_math1_worker.js -- Web Worker for zig-math1 WASM module const DATA_OFFSET = 16; let ctrl, data, wasmMemory; const enc = new TextEncoder(), From dbfb3cc501bd11d58535aa3b51c5fc1dcea11359 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 17 Aug 2026 11:29:47 +0800 Subject: [PATCH 6/8] tidy --- CLAUDE.md | 9 --------- config/semgrep/no-non-ascii.yaml | 2 +- services/ws-modules/rdata1/pkg/module.R | 2 ++ services/ws-modules/rmath1/pkg/module.R | 2 ++ 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d63e088a..0a494ccd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -374,15 +374,6 @@ Languages: ws-web-runner). Their JS shims are linted by the `js` env; `MISE_ENV=r` supplies webR + the vendoring build tasks. - **Zig -> WASM**: zig-data1, zig-except1 (C++ wasm-exception-handling demo), zig-math1 -The math1 modules are one family, driven by a storage exchange: a fake agent (the shared helper in -`et-ws-test-server`, embedding the canonical committed input at its `data/math1-input.json`) injects the -input JSON into ws-server storage and broadcasts a `math1-input` pointer over the hub; each module reads the -input, runs the same FedAvg simulation (`+ - * /` on f64 only), and stores its global model to -`math1-output.json` in its own bucket, which the test harness reads and verifies against the expected -weights -- proving bit-identical float math across every guest runtime. Beyond the browser twins the family -also covers wasi-math1 (a WASI Preview 2 component under `et-ws-wasi-runner`'s wasmtime) and the `math1.py` -module under `et-ws-pyo3-runner` (native CPython) -- the two non-browser executors. - - **Python (componentize-py -> WASI Preview 2 component)**: wasi-graphics-info -- runs in `et-ws-wasi-runner` rather than the browser. The WIT world the component implements is at `services/ws-wasi-runner/wit/world.wit` and is mirrored under the module's own `wit/`. diff --git a/config/semgrep/no-non-ascii.yaml b/config/semgrep/no-non-ascii.yaml index 8db5238f..58f03c41 100644 --- a/config/semgrep/no-non-ascii.yaml +++ b/config/semgrep/no-non-ascii.yaml @@ -25,5 +25,5 @@ rules: Keep source, config, and docs ASCII-only: use `--` for an em-dash, `->` for an arrow, `...` for an ellipsis, `<=`/`!=`/`^2` for math glyphs, and straight quotes for curly quotes. ASCII keeps terminals, log scrapers, and `grep` reading the same bytes the editor shows. Generated trees (generated/, verification/, - HELP.md, pkg/) are exempt because their generator owns the output. + HELP.md) are exempt because their generator owns the output. severity: ERROR diff --git a/services/ws-modules/rdata1/pkg/module.R b/services/ws-modules/rdata1/pkg/module.R index a0fb1ec4..954105ef 100644 --- a/services/ws-modules/rdata1/pkg/module.R +++ b/services/ws-modules/rdata1/pkg/module.R @@ -95,6 +95,8 @@ run <- function() { agent_log(sprintf("rdata1: registered as %s", bucket)) # Compute in R, then PUT and GET the payload with httr2 (tunnelled to the server via the relay), and verify. + # The literal loopback host is required, NOT a stand-in for the page origin: the relay only honours loopback + # CONNECT targets (its SSRF guard) and always bridges to its own fixed server target, whatever host:port says. content <- rdata1_payload() url <- sprintf("http://127.0.0.1:8080/storage/%s/%s", bucket, rdata1_filename()) httr2::request(url) |> diff --git a/services/ws-modules/rmath1/pkg/module.R b/services/ws-modules/rmath1/pkg/module.R index 547ea97c..d8313168 100644 --- a/services/ws-modules/rmath1/pkg/module.R +++ b/services/ws-modules/rmath1/pkg/module.R @@ -117,6 +117,8 @@ run <- function() { sleep_ms(100) } parts <- strsplit(pointer, "\n", fixed = TRUE)[[1]] + # The literal loopback host is required, NOT a stand-in for the page origin: the relay only honours loopback + # CONNECT targets (its SSRF guard) and always bridges to its own fixed server target, whatever host:port says. input_url <- sprintf("http://127.0.0.1:8080/storage/%s/%s", parts[[1]], parts[[2]]) agent_log(sprintf("rmath1: reading input from %s", input_url)) input_text <- httr2::request(input_url) |> From db374f051f72f7c285fe8267f5acc80a52cfcfd9 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 17 Aug 2026 12:29:11 +0800 Subject: [PATCH 7/8] Add math1-sender --- .mise/config.rust.toml | 6 + CLAUDE.md | 2 +- Cargo.lock | 18 +++ Cargo.toml | 1 + README.md | 4 +- .../rules/doc-summary-ends-with-period.yaml | 4 + config/ls-lint.yaml | 1 + services/ws-modules/face-detection/src/lib.rs | 36 ++++- .../face-detection/tests/status_lines.rs | 32 ++++ services/ws-modules/har1/src/lib.rs | 15 +- .../ws-modules/har1/tests/status_lines.rs | 13 ++ services/ws-modules/math1-sender/Cargo.toml | 33 ++++ services/ws-modules/math1-sender/build.rs | 10 ++ services/ws-modules/math1-sender/src/lib.rs | 143 ++++++++++++++++++ services/ws-web-runner/Cargo.toml | 1 + services/ws-web-runner/tests/modules.rs | 49 +++++- 16 files changed, 354 insertions(+), 14 deletions(-) create mode 100644 services/ws-modules/face-detection/tests/status_lines.rs create mode 100644 services/ws-modules/har1/tests/status_lines.rs create mode 100644 services/ws-modules/math1-sender/Cargo.toml create mode 100644 services/ws-modules/math1-sender/build.rs create mode 100644 services/ws-modules/math1-sender/src/lib.rs diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 98420e5f..11a015b1 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -30,6 +30,12 @@ description = "Build the math1 FedAvg WASM module" dir = "services/ws-modules/math1" run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +[tasks.build-ws-math1-sender-module] +depends = ["build-wasm-cov-wrapper"] +description = "Build the math1-sender trigger WASM module" +dir = "services/ws-modules/math1-sender" +run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" + [tasks.build-ws-comm1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the comm1 workflow WASM module" diff --git a/CLAUDE.md b/CLAUDE.md index 0a494ccd..9b959fc0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -361,7 +361,7 @@ The server only serves them from disk. Languages: - **Rust -> WASM** (wasm-pack): audio1, bluetooth, comm1, data1, except1, face-detection, geolocation, graphics-info, - har1, math1, nfc, sensor1, speech-recognition, video1 + har1, math1, math1-sender, nfc, sensor1, speech-recognition, video1 - **JavaScript**: js-data1 (esbuild bundle of the AWS SDK v3 twin), js-math1 (dependency-free, committed as-is) - **Dart -> JS**: dart-comm1, dart-data1, dart-math1 - **Kotlin -> WASM (WasmGC)**: kotlin-data1, kotlin-math1 -- compiled by the Kotlin Gradle plugin's `wasmJs` target; diff --git a/Cargo.lock b/Cargo.lock index 4a798bce..a87b0acf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4572,6 +4572,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "et-ws-math1-sender" +version = "0.1.0" +dependencies = [ + "et-path", + "et-rest-client", + "et-web", + "et-ws-wasm-agent", + "js-sys", + "tracing", + "tracing-wasm", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", +] + [[package]] name = "et-ws-nfc" version = "0.1.0" @@ -4918,6 +4935,7 @@ dependencies = [ "rstest", "serde", "serde-env", + "serde_json", "sys_traits", "thiserror 2.0.19", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 8af8251b..b45fe9e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "services/ws-modules/graphics-info", "services/ws-modules/har1", "services/ws-modules/math1", + "services/ws-modules/math1-sender", "services/ws-modules/nfc", "services/ws-modules/pic-viewer", "services/ws-modules/sensor1", diff --git a/README.md b/README.md index b23196c3..55270e4c 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,9 @@ the other two runners). A test-harness "fake agent" uploads a canonical input fi pointer to it over the hub; each module reads the input, runs the same FedAvg simulation -- rounds of local gradient-descent epochs per simulated client, merged with a sample-count-weighted average, using only `+ - * /` on IEEE-754 doubles -- and stores its resulting global model back to storage, where the test harness verifies that -every language produced bit-identical weights. +every language produced bit-identical weights. To trigger the twins manually, run the `math1-sender` module in +another browser tab: it plays the fake-agent side itself, uploading the canonical input and broadcasting the +pointer once a second for a minute. ## Root module diff --git a/config/ast-grep/rules/doc-summary-ends-with-period.yaml b/config/ast-grep/rules/doc-summary-ends-with-period.yaml index 4fdf508a..a4bfaf1c 100644 --- a/config/ast-grep/rules/doc-summary-ends-with-period.yaml +++ b/config/ast-grep/rules/doc-summary-ends-with-period.yaml @@ -47,7 +47,11 @@ files: - services/ws-modules/comm1/src/lib.rs - services/ws-modules/except1/src/lib.rs - services/ws-modules/face-detection/src/lib.rs + - services/ws-modules/face-detection/tests/status_lines.rs - services/ws-modules/har1/src/lib.rs + - services/ws-modules/har1/tests/status_lines.rs + - services/ws-modules/math1-sender/build.rs + - services/ws-modules/math1-sender/src/lib.rs - services/ws-modules/math1/src/lib.rs - services/ws-modules/pic-viewer/build.rs - services/ws-modules/pic-viewer/src/lib.rs diff --git a/config/ls-lint.yaml b/config/ls-lint.yaml index f0949985..ce05984f 100644 --- a/config/ls-lint.yaml +++ b/config/ls-lint.yaml @@ -63,6 +63,7 @@ ignore: - services/ws-modules/js-math1/pkg - services/ws-modules/kotlin-data1/pkg - services/ws-modules/kotlin-math1/pkg + - services/ws-modules/math1-sender/pkg - services/ws-modules/math1/pkg - services/ws-modules/nfc/pkg - services/ws-modules/pic-viewer/pkg diff --git a/services/ws-modules/face-detection/src/lib.rs b/services/ws-modules/face-detection/src/lib.rs index 01ff51d6..a17d5345 100644 --- a/services/ws-modules/face-detection/src/lib.rs +++ b/services/ws-modules/face-detection/src/lib.rs @@ -409,25 +409,49 @@ async fn infer_once( } fn update_face_status(input_name: &str, output_names: &[String], summary: &DetectionSummary) { + let lines = face_status_lines( + input_name, + output_names, + summary.detections.len(), + summary.confidence, + &summary.processed_at, + summary.detections.first().map(|best| best.box_coords), + ); + face_set_status(&lines.join("\n")); +} + +/// Build the status-panel lines for a detection summary. +/// +/// Pure (no DOM/interop calls) and parameterised on plain values, so the host-side test can cover +/// the formatting -- including the blank separator ahead of the best-box section. +#[must_use] +pub fn face_status_lines( + input_name: &str, + output_names: &[String], + detection_count: usize, + confidence: f64, + processed_at: &str, + best_box: Option<[f64; 4]>, +) -> Vec { let mut lines = vec![ String::from("face detection demo"), format!("model file: {FACE_MODEL_PATH}"), format!("input: {input_name}"), format!("outputs: {}", output_names.join(", ")), - format!("detections: {}", summary.detections.len()), - format!("best confidence: {:.4}", summary.confidence), - format!("processed at: {}", summary.processed_at), + format!("detections: {detection_count}"), + format!("best confidence: {confidence:.4}"), + format!("processed at: {processed_at}"), ]; - if let Some(best) = summary.detections.first() { + if let Some(coords) = best_box { lines.push(String::default()); lines.push(format!( "best box: {:.1}, {:.1}, {:.1}, {:.1}", - best.box_coords[0], best.box_coords[1], best.box_coords[2], best.box_coords[3] + coords[0], coords[1], coords[2], coords[3] )); } - face_set_status(&lines.join("\n")); + lines } fn decode_retinaface_outputs( diff --git a/services/ws-modules/face-detection/tests/status_lines.rs b/services/ws-modules/face-detection/tests/status_lines.rs new file mode 100644 index 00000000..c887b58d --- /dev/null +++ b/services/ws-modules/face-detection/tests/status_lines.rs @@ -0,0 +1,32 @@ +//! Host-side check of the pure status-panel line builder (the wasm interop stays untested here). + +#![cfg(test)] + +use et_ws_face_detection::face_status_lines; + +#[test] +fn includes_a_separated_best_box_section_when_a_detection_exists() { + let lines = face_status_lines( + "input", + &[String::from("out0"), String::from("out1")], + 2, + 0.9876_f64, + "2026-08-17T00:00:00Z", + Some([1.0_f64, 2.0_f64, 3.0_f64, 4.0_f64]), + ); + assert!( + lines.contains(&String::default()), + "missing the blank separator: {lines:?}" + ); + assert_eq!(lines.last().unwrap(), "best box: 1.0, 2.0, 3.0, 4.0"); + assert!( + lines.contains(&String::from("detections: 2")), + "missing the count: {lines:?}" + ); +} + +#[test] +fn omits_the_best_box_section_without_detections() { + let lines = face_status_lines("input", &[], 0, 0.0_f64, "2026-08-17T00:00:00Z", None); + assert!(!lines.contains(&String::default()), "unexpected separator: {lines:?}"); +} diff --git a/services/ws-modules/har1/src/lib.rs b/services/ws-modules/har1/src/lib.rs index 2b890a9c..f69f0d40 100644 --- a/services/ws-modules/har1/src/lib.rs +++ b/services/ws-modules/har1/src/lib.rs @@ -499,6 +499,15 @@ async fn run_inner(client: &WsClient, sensors: &mut DeviceSensors) -> Result<(), Ok(()) } +/// Append a blank-line-separated section header to the status-panel lines. +/// +/// Pure (no DOM/interop calls), so the host-side test can cover the separator formatting the +/// sensor display builds its orientation/motion sections through. +pub fn push_section(lines: &mut Vec, title: &str) { + lines.push(String::default()); + lines.push(String::from(title)); +} + fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { let orientation = if sensors.has_orientation() { Some(sensors.orientation_snapshot()?) @@ -517,9 +526,8 @@ fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { "updated: {}", String::from(js_sys::Date::new_0().to_locale_time_string("en-US")) ), - String::default(), - String::from("orientation"), ]; + push_section(&mut lines, "orientation"); if let Some(orientation) = orientation { lines.push(format!("alpha: {}", format_number(orientation.alpha(), 3))); @@ -530,8 +538,7 @@ fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { lines.push(String::from("waiting for orientation event...")); } - lines.push(String::default()); - lines.push(String::from("motion")); + push_section(&mut lines, "motion"); if let Some(motion) = motion { lines.push(format!( "acceleration: x={} y={} z={}", diff --git a/services/ws-modules/har1/tests/status_lines.rs b/services/ws-modules/har1/tests/status_lines.rs new file mode 100644 index 00000000..d8d6e02d --- /dev/null +++ b/services/ws-modules/har1/tests/status_lines.rs @@ -0,0 +1,13 @@ +//! Host-side check of the pure status-panel section helper (the wasm interop stays untested here). + +#![cfg(test)] + +use et_ws_har1::push_section; + +#[test] +fn appends_a_blank_separator_then_the_title() { + let mut lines = vec![String::from("header")]; + push_section(&mut lines, "orientation"); + push_section(&mut lines, "motion"); + assert_eq!(lines, ["header", "", "orientation", "", "motion"]); +} diff --git a/services/ws-modules/math1-sender/Cargo.toml b/services/ws-modules/math1-sender/Cargo.toml new file mode 100644 index 00000000..c57bae13 --- /dev/null +++ b/services/ws-modules/math1-sender/Cargo.toml @@ -0,0 +1,33 @@ +[package] +description = "math 1 sender" +name = "et-ws-math1-sender" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +crate-type = ["cdylib", "rlib"] +doctest = false +test = false + +[dependencies] +et-rest-client.workspace = true +et-web.workspace = true +et-ws-wasm-agent.workspace = true +js-sys.workspace = true +tracing.workspace = true +tracing-wasm.workspace = true +wasm-bindgen.workspace = true +wasm-bindgen-futures.workspace = true +web-sys = { workspace = true, features = ["Window", "console"] } + +# Build script (runs on the host) locates the repo root to emit ET_MATH1_INPUT_PATH. +[build-dependencies] +et-path.workspace = true + +[dev-dependencies] +wasm-bindgen-test.workspace = true + +[lints] +workspace = true diff --git a/services/ws-modules/math1-sender/build.rs b/services/ws-modules/math1-sender/build.rs new file mode 100644 index 00000000..9000ad23 --- /dev/null +++ b/services/ws-modules/math1-sender/build.rs @@ -0,0 +1,10 @@ +//! Emit `ET_MATH1_INPUT_PATH` (absolute path to the canonical math1 input JSON) so `include_str!` +//! in `src/lib.rs` embeds the same bytes every math1 test harness injects, without a `..`-relative +//! path that hardcodes this crate's depth below the repository root. + +fn main() { + let input = et_path::find_project_root_from_manifest().join("services/ws-test-server/data/math1-input.json"); + println!("cargo:rustc-env=ET_MATH1_INPUT_PATH={}", input.display()); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed={}", input.display()); +} diff --git a/services/ws-modules/math1-sender/src/lib.rs b/services/ws-modules/math1-sender/src/lib.rs new file mode 100644 index 00000000..ccf04a4c --- /dev/null +++ b/services/ws-modules/math1-sender/src/lib.rs @@ -0,0 +1,143 @@ +//! math1-sender: the manual trigger for the math1 family. +//! +//! Plays the fake-agent side of the math1 storage exchange from a browser: uploads the canonical +//! input JSON (embedded at build time from ws-test-server's `data/math1-input.json`, the same bytes +//! every test harness injects) into this agent's own storage bucket, then broadcasts the +//! `math1-input` pointer once a second for a minute so math1 twins started in other tabs (or on +//! other machines connected to the same hub) pick it up, compute, and store their models. This +//! module only sends; each twin's stored `math1-output.json` is the operator's to inspect. + +#![expect( + clippy::future_not_send, + clippy::single_call_fn, + reason = "browser WASM module: JsFuture is !Send; module-local helpers are single-use by design" +)] + +use et_web::JsResultExt as _; +use et_ws_wasm_agent::{WsClient, WsClientConfig, append_to_textarea}; +use js_sys::{Promise, Reflect}; +use tracing::info; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +/// The canonical input bytes, embedded from the committed file so there is one source of truth. +const MATH1_INPUT_JSON: &str = include_str!(env!("ET_MATH1_INPUT_PATH")); +/// Storage object name the twins' broadcast pointer names, matching the test harnesses. +const INPUT_FILENAME: &str = "math1-input.json"; +/// How many one-second-spaced pointer broadcasts to send before completing. +const BROADCASTS: u32 = 60; + +#[wasm_bindgen(start)] +pub fn init() { + tracing_wasm::set_as_global_default(); + info!("math1-sender module initialized"); +} + +#[wasm_bindgen] +pub async fn run() -> Result<(), JsValue> { + let msg = "math1-sender: entered run()"; + log(msg); + set_module_status(msg)?; + + let ws_url = websocket_url()?; + let mut config = WsClientConfig::new(ws_url); + // The wasm-agent retains its server-issued id in localStorage, which is shared across every tab + // of this origin -- reusing it here would steal the identity of a math1 twin running in another + // tab, and the hub would then never relay this module's broadcasts to it. A fresh id keeps the + // sender a distinct peer. + config.set_use_retained_agent_id(false); + let mut client = WsClient::new(config); + + client.connect()?; + wait_for_connected(&client).await?; + let agent_id = wait_for_agent_id(&client).await?; + let msg = format!("math1-sender: connected as {agent_id}"); + log(&msg); + set_module_status(&msg)?; + + // The typed REST client runs against the page origin -- every browser module is served from the + // same ws-server that owns its storage, so an empty base URL (relative paths) is what we want. + let rest = et_rest_client::Client::new(""); + let _put_response = rest + .put_file(&agent_id, INPUT_FILENAME, MATH1_INPUT_JSON.to_string()) + .await + .js_context("input PUT failed")?; + let msg = format!("math1-sender: injected the canonical input to /storage/{agent_id}/{INPUT_FILENAME}"); + log(&msg); + set_module_status(&msg)?; + + let pointer = format!(r#"{{"type":"math1-input","bucket":"{agent_id}","filename":"{INPUT_FILENAME}"}}"#); + let msg = format!("math1-sender: broadcasting the math1-input pointer every second, {BROADCASTS} times"); + log(&msg); + set_module_status(&msg)?; + for round in 1_u32..=BROADCASTS { + client.send(&pointer)?; + if round == 1 || round.is_multiple_of(10) { + let msg = format!("math1-sender: broadcast {round}/{BROADCASTS}"); + log(&msg); + set_module_status(&msg)?; + } + sleep_ms(1000).await?; + } + + client.disconnect(); + let msg = "math1-sender: workflow complete"; + log(msg); + set_module_status(msg)?; + Ok(()) +} + +fn log(message: &str) { + let line = format!("[math1-sender] {message}"); + web_sys::console::log_1(&JsValue::from_str(&line)); +} + +fn set_module_status(message: &str) -> Result<(), JsValue> { + append_to_textarea("module-output", message) +} + +async fn wait_for_connected(client: &WsClient) -> Result<(), JsValue> { + for _ in 0_u32..100 { + if client.get_state() == "connected" { + return Ok(()); + } + sleep_ms(100).await?; + } + Err(JsValue::from_str("Timed out waiting for websocket connection")) +} + +async fn wait_for_agent_id(client: &WsClient) -> Result { + for _ in 0_u32..100 { + let agent_id = client.get_agent_id(); + if !agent_id.is_empty() { + return Ok(agent_id); + } + sleep_ms(100).await?; + } + Err(JsValue::from_str("Timed out waiting for assigned agent_id")) +} + +async fn sleep_ms(duration_ms: i32) -> Result<(), JsValue> { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let promise = Promise::new(&mut |resolve, _reject| { + let callback = Closure::once_into_js(move || { + et_web::ignore(resolve.call0(&JsValue::NULL)); + }); + let _id: Result = + window.set_timeout_with_callback_and_timeout_and_arguments_0(callback.unchecked_ref(), duration_ms); + }); + JsFuture::from(promise).await.map(|_| ()) +} + +fn websocket_url() -> Result { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let location = Reflect::get(window.as_ref(), &JsValue::from_str("location"))?; + let protocol = Reflect::get(&location, &JsValue::from_str("protocol"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.protocol is unavailable"))?; + let host = Reflect::get(&location, &JsValue::from_str("host"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.host is unavailable"))?; + let ws_protocol = if protocol == "https:" { "wss:" } else { "ws:" }; + Ok(format!("{ws_protocol}//{host}/ws")) +} diff --git a/services/ws-web-runner/Cargo.toml b/services/ws-web-runner/Cargo.toml index 82a5d25a..f9cccef7 100644 --- a/services/ws-web-runner/Cargo.toml +++ b/services/ws-web-runner/Cargo.toml @@ -58,6 +58,7 @@ fs-err.workspace = true et-ws-test-server.workspace = true fs-err.workspace = true rstest.workspace = true +serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt", "time"] } # `coverage` (off by default): compiles the browser-module coverage-capture code into the runner. diff --git a/services/ws-web-runner/tests/modules.rs b/services/ws-web-runner/tests/modules.rs index 4b981221..8881c503 100644 --- a/services/ws-web-runner/tests/modules.rs +++ b/services/ws-web-runner/tests/modules.rs @@ -161,14 +161,59 @@ async fn math1_module_stores_verified_model(#[case] module: &str, #[case] langua collect_module_coverage(&server); } +/// The manual-trigger pair: math1-sender plays the fake agent while the Rust math1 twin computes. +/// +/// Both modules spawn against one server. The sender uploads the embedded canonical input, +/// broadcasts the pointer (for a minute -- its manual-use window, which this test's wall clock +/// includes), and math1 stores its model, which is then verified against the expected weights. +/// This is the end-to-end proof of the browser-only trigger path, with no harness fake agent. +#[test] +fn math1_sender_triggers_math1() { + if !mise_env_includes(Language::Rust) { + println!("skipping et-ws-math1-sender: requires the `rust` mise env, not loaded"); + return; + } + let server = et_ws_test_server::start(); + let bin = env!("CARGO_BIN_EXE_et-ws-web-runner"); + let spawn = |module: &str| { + std::process::Command::new(bin) + .env("RUNNER_MODULE", module) + .env("WS_SERVER_URL", &server.ws_url) + .env("RUNNER_TIMEOUT", "110s") + .spawn() + .unwrap() + }; + let mut sender = spawn("et-ws-math1-sender"); + let mut math1 = spawn("et-ws-math1"); + let math1_status = wait_for_runner_exit(&mut math1); + let sender_status = wait_for_runner_exit(&mut sender); + assert!(math1_status.success(), "math1 runner exited {math1_status:?}"); + assert!(sender_status.success(), "math1-sender runner exited {sender_status:?}"); + + // The twin stored its model in its own bucket; find it and verify the weights. + let mut verified = false; + for bucket in fs_err::read_dir(server.storage_dir.path()).unwrap().flatten() { + let output_path = bucket.path().join("math1-output.json"); + if let Ok(bytes) = fs_err::read(&output_path) { + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let weight = value.get("weight").and_then(serde_json::Value::as_f64).unwrap(); + let bias = value.get("bias").and_then(serde_json::Value::as_f64).unwrap(); + et_ws_test_server::math1::verify_math1_model(weight, bias).unwrap(); + verified = true; + } + } + assert!(verified, "no math1-output.json found in any storage bucket"); + #[cfg(feature = "coverage")] + collect_module_coverage(&server); +} + /// Poll the spawned runner until it exits, killing it if it overstays the runner-timeout bound. /// /// Blocking here is fine: this runs after the exchange future has already resolved, so nothing /// else is pending on the current-thread runtime. #[expect( clippy::arithmetic_side_effects, - clippy::single_call_fn, - reason = "distinct reap step; the deadline addition cannot overflow within a test's lifetime" + reason = "the deadline addition cannot overflow within a test's lifetime" )] fn wait_for_runner_exit(runner: &mut std::process::Child) -> std::process::ExitStatus { let deadline = std::time::Instant::now() + std::time::Duration::from_mins(2); From 3a37e5310541eae2dd59cfae290e647c44537c22 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 17 Aug 2026 13:15:33 +0800 Subject: [PATCH 8/8] fix coverage tracking bug --- .mise/config.coverage.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 05946ead..821bb9b1 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -154,9 +154,17 @@ for profraw in "$covdir"/*.profraw; do # nothing and every module aborted the task with `wasi-cov: no .ll found for et_ws_audio1`. Observed on commit # 3fe98398 at https://github.com/edge-toolkit/core/actions/runs/30789818011/job/91610793550. Both globs stay so # the task works whichever layout the toolchain produces; drop the deps fallback once no supported cargo emits it. - ll="$(find target -path '*/release/build/*' -name "$name*.ll" 2>/dev/null | coreutils head -n 1)" + # `-print -quit` (not `| coreutils head -n 1`) and an exact-or-hash-suffixed name pattern, both load-bearing: + # (1) `$name*.ll` also matched prefix-sharing sibling crates once math1-sender landed next to math1, risking a + # profraw paired with the wrong module's covmap; (2) with 2+ matches, head exits after the first line and the + # uutils find dies on the broken pipe (Rust ignores SIGPIPE) -- a panic on the Linux runners, exiting 101 with + # its message discarded by the 2>/dev/null, so `pipefail` killed this task with no output at all. Observed on + # commit db374f051f72f7c285fe8267f5acc80a52cfcfd9 at + # https://github.com/edge-toolkit/core/actions/runs/31994728610/job/95284251850; reproduced locally on macOS + # as `find: stdout: Undefined error: 0` (EPIPE, exit 1) once the second match exists. + ll="$(find target -path '*/release/build/*' \\( -name "$name.ll" -o -name "$name-*.ll" \\) -print -quit 2>/dev/null)" if [ -z "$ll" ]; then - ll="$(find target -path '*/release/deps/*' -name "$name*.ll" 2>/dev/null | coreutils head -n 1)" + ll="$(find target -path '*/release/deps/*' \\( -name "$name.ll" -o -name "$name-*.ll" \\) -print -quit 2>/dev/null)" fi if [ -z "$ll" ]; then echo "wasi-cov: no .ll found for $name"; exit 1; fi pd="$covdir/$name.profdata"