From 3a6e3977e0aeed4ec5dde072a6ed6bdb2249677d Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Tue, 18 Aug 2026 07:02:41 -0700 Subject: [PATCH 1/7] chore: lower the Node floor to 24.4.0 The package required Node 24.15.0 with no recorded reason. Measuring the runtime shows the real constraint is one SQLite option. The SQLite adapter builds DatabaseSync with readBigInts, which it needs to read 64-bit integers without losing precision. Node 24.0.0 through 24.3.x accept the option and ignore it, returning Number instead of BigInt. That is silent, so the suite is what catches it: the stale-process effect recovery test and the SQLite transaction retry test both fail on 24.3.0 and pass on 24.4.0. Node 24.15.0 is where node:sqlite stops being experimental and stops printing a warning on stderr. That is a reason to prefer it, not a reason to require it, so the docs now say both. A floor job runs the default suite, the build, and the recovery demo on 24.4.0, and the publish job waits for it, so the declared floor stays measured rather than assumed. --- .github/workflows/ci.yml | 15 ++++++++++++++- CHANGELOG.md | 11 +++++++++++ CONTRIBUTING.md | 4 +++- README.md | 8 +++++--- docs/support.md | 13 ++++++++++++- package.json | 2 +- 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1c3e35..f97094c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,19 @@ jobs: - run: pnpm run test:recovery - run: pnpm audit --audit-level=high + floor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 + with: + node-version: "24.4.0" + - run: corepack enable + - run: pnpm install --frozen-lockfile + - run: pnpm run test + - run: pnpm run build + - run: pnpm run test:recovery + postgresql: runs-on: ubuntu-latest strategy: @@ -127,7 +140,7 @@ jobs: publish: if: startsWith(github.ref, 'refs/tags/v') - needs: [quality, postgresql, mysql, browser, redis] + needs: [quality, floor, postgresql, mysql, browser, redis] runs-on: ubuntu-latest permissions: contents: read diff --git a/CHANGELOG.md b/CHANGELOG.md index c218d80..3a8ca33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +- Lower the supported Node.js floor from 24.15.0 to 24.4.0. Node.js 24.4.0 is + the first release that accepts `readBigInts` on the `DatabaseSync` + constructor, which the SQLite adapter needs to read 64-bit integers without + losing precision. Node.js 24.0.0 through 24.3.x ignore the option, and the + effect recovery and transaction retry tests fail there. A new CI job runs the + default suite, the build, and the recovery demo on the floor. +- Record that `node:sqlite` stays experimental until Node.js 24.15.0 and prints + a warning on stderr before it. + ## 0.13.1 - 2026-08-16 - Back idle actor, effect, reminder, and broadcast polling off exponentially diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a4917c..2586ed4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,9 @@ lowest layer that can prove it. ## Setup Install Node.js 24.15 or newer, enable Corepack, and install the locked -dependencies: +dependencies. The package supports Node.js 24.4.0 or newer and CI runs the +default suite on that floor, but 24.15 is where `node:sqlite` stops printing an +experimental warning: ```bash corepack enable diff --git a/README.md b/README.md index 8adebe9..947f212 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,9 @@ processes submit them concurrently. ## Run it now with SQLite -Node.js 24.15 or newer is required. The `0.13.1` release includes a -packaged quickstart: +Node.js 24.4.0 or newer is required. Node.js 24.15 or newer is preferred, +because `node:sqlite` prints an experimental warning before it. The `0.13.1` +release includes a packaged quickstart: ```bash npm exec --yes --package=solid-objects@0.13.1 -- solid-objects quickstart @@ -255,7 +256,8 @@ edge placement, cross-identity transactions, and operational data access—is in ## Requirements and supported systems -- Node.js 24.15 or newer +- Node.js 24.4.0 or newer; 24.15 or newer to avoid the `node:sqlite` + experimental warning - TypeScript 5.9 or newer for TypeScript applications - SQLite through `node:sqlite`, PostgreSQL 14 or newer, or MySQL 8.0 or newer with InnoDB diff --git a/docs/support.md b/docs/support.md index 5e2c379..a849cef 100644 --- a/docs/support.md +++ b/docs/support.md @@ -4,7 +4,7 @@ | Component | Supported or tested range | | -------------- | ----------------------------------------------------------- | -| Node.js | 24.15 or newer; CI uses 24.15 | +| Node.js | 24.4.0 or newer; CI runs 24.4.0 and 24.15.0 | | TypeScript | 5.9 or newer for TypeScript applications | | SQLite | Node's built-in `node:sqlite` on the supported Node runtime | | PostgreSQL | 14 or newer; CI runs 14 and 18 | @@ -15,6 +15,17 @@ The package is ESM-only. PostgreSQL, MySQL, and Redis require their optional peer dependency. SQLite has no driver dependency beyond Node.js. +The Node.js floor is 24.4.0 because the SQLite adapter reads integer columns as +`BigInt`. Node.js 24.4.0 is the first release that accepts `readBigInts` on the +`DatabaseSync` constructor. Node.js 24.0.0 through 24.3.x ignore the option and +return `Number`, which loses precision on 64-bit values and fails the effect +recovery and transaction retry tests. + +Node.js 24.15.0 is the first release where `node:sqlite` is no longer +experimental. Between 24.4.0 and 24.14.x the module works but prints +`ExperimentalWarning: SQLite is an experimental feature` on stderr, and its API +can change. Prefer 24.15.0 or newer where the choice is free. + ## What the matrix covers The default suite exercises actor definitions, mailbox ordering, state diff --git a/package.json b/package.json index 59d5f18..3053f55 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "url": "git+https://github.com/cardmagic/solid-objects-js.git" }, "engines": { - "node": ">=24.15.0" + "node": ">=24.4.0" }, "packageManager": "pnpm@11.11.0", "bin": { From 9442c7aa74bf5825fa2af7200b37b4238ce6bf3f Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Tue, 18 Aug 2026 07:07:33 -0700 Subject: [PATCH 2/7] chore: prepare 0.13.2 Bumps package.json, src/version.ts, the process metadata test literal, the README quickstart references, and dates the changelog section for the Node floor change. --- CHANGELOG.md | 2 +- CONTRIBUTING.md | 4 ++-- README.md | 4 ++-- package.json | 2 +- src/version.ts | 2 +- test/process-administration.test.ts | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a8ca33..9ce1c4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.13.2 - 2026-08-18 - Lower the supported Node.js floor from 24.15.0 to 24.4.0. Node.js 24.4.0 is the first release that accepts `readBigInts` on the `DatabaseSync` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2586ed4..c8461f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,8 +8,8 @@ lowest layer that can prove it. Install Node.js 24.15 or newer, enable Corepack, and install the locked dependencies. The package supports Node.js 24.4.0 or newer and CI runs the -default suite on that floor, but 24.15 is where `node:sqlite` stops printing an -experimental warning: +default suite, the build, and the recovery demo on that floor, but 24.15 is +where `node:sqlite` stops printing an experimental warning: ```bash corepack enable diff --git a/README.md b/README.md index 947f212..c6dc16d 100644 --- a/README.md +++ b/README.md @@ -58,11 +58,11 @@ processes submit them concurrently. ## Run it now with SQLite Node.js 24.4.0 or newer is required. Node.js 24.15 or newer is preferred, -because `node:sqlite` prints an experimental warning before it. The `0.13.1` +because `node:sqlite` prints an experimental warning before it. The `0.13.2` release includes a packaged quickstart: ```bash -npm exec --yes --package=solid-objects@0.13.1 -- solid-objects quickstart +npm exec --yes --package=solid-objects@0.13.2 -- solid-objects quickstart ``` The command needs no repository checkout, database server, Redis, container, or diff --git a/package.json b/package.json index 3053f55..20aba6e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "solid-objects", - "version": "0.13.1", + "version": "0.13.2", "description": "Race-free realtime state per application identity, backed by your SQL database", "type": "module", "license": "MIT", diff --git a/src/version.ts b/src/version.ts index 29a922e..8cc92a6 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const VERSION = "0.13.1" +export const VERSION = "0.13.2" diff --git a/test/process-administration.test.ts b/test/process-administration.test.ts index 7483245..889981f 100644 --- a/test/process-administration.test.ts +++ b/test/process-administration.test.ts @@ -42,7 +42,7 @@ describe("process administration", () => { hostProcessId: process.pid, metadata: { nodeVersion: process.version, - solidObjectsVersion: "0.13.1", + solidObjectsVersion: "0.13.2", }, shutdownState: "running", shutdownRequestedAt: null, From 9be5c936f6f36010147dd7c6be9a9f75c0993a00 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Tue, 18 Aug 2026 07:25:04 -0700 Subject: [PATCH 3/7] ci: smoke the packaged artifact on the floor Greptile noted the floor job proved the source tree but never the artifact a consumer installs. A packaging or executable problem specific to the floor could pass the publish gate. The job now runs test:package too. --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 3 ++- CONTRIBUTING.md | 7 ++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f97094c..a147a06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm run test - run: pnpm run build + - run: pnpm run test:package - run: pnpm run test:recovery postgresql: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce1c4d..82b0bab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ constructor, which the SQLite adapter needs to read 64-bit integers without losing precision. Node.js 24.0.0 through 24.3.x ignore the option, and the effect recovery and transaction retry tests fail there. A new CI job runs the - default suite, the build, and the recovery demo on the floor. + default suite, the build, the packaged artifact smoke test, and the + recovery demo on the floor. - Record that `node:sqlite` stays experimental until Node.js 24.15.0 and prints a warning on stderr before it. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8461f1..5823a25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,9 +7,10 @@ lowest layer that can prove it. ## Setup Install Node.js 24.15 or newer, enable Corepack, and install the locked -dependencies. The package supports Node.js 24.4.0 or newer and CI runs the -default suite, the build, and the recovery demo on that floor, but 24.15 is -where `node:sqlite` stops printing an experimental warning: +dependencies. The package supports Node.js 24.4.0 or newer, and CI runs the +default suite, the build, the packaged artifact smoke test, and the recovery +demo on that floor. Node.js 24.15 is where `node:sqlite` stops printing an +experimental warning: ```bash corepack enable From cad19246ab412cd606b3497cbbd25209c4b2912b Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Tue, 18 Aug 2026 08:58:52 -0700 Subject: [PATCH 4/7] fix: prove serialization by message, not by execution The failure-recovery demo asserted the serialization control file held exactly four events. A worker that loses its lease mid-operation leaves the replacement to execute the same message again, so a slower machine produces six, and the demo failed for the at-least-once behaviour the library documents and the crash and fencing proofs rely on. It failed five times on GitHub runners, on Node 24.15.0 as well as the lower floor, and passed on rerun, so the assertion was reporting the runner's speed rather than a durability property. The proof now asserts what the runtime guarantees: executions never overlap, every start has a matching finish, and exactly the two sent messages ran. The committed state check is unchanged, and it is the one that would catch a lost or doubled write. assertSerializedExecution moves into its own module so it can be tested at the lowest layer. The unit tests cover the retry, overlap, unfinished, and lost-message cases, and each rejection case fails when the rule is removed. The lease timing stays as it was. proveCrashRecovery and proveFencing depend on a short lease, so raising it to hide the retry would weaken them. --- CHANGELOG.md | 11 +++++ examples/failure-recovery/demo.ts | 26 +--------- examples/failure-recovery/serialization.ts | 53 +++++++++++++++++++++ test/failure-recovery-serialization.test.ts | 52 ++++++++++++++++++++ 4 files changed, 118 insertions(+), 24 deletions(-) create mode 100644 examples/failure-recovery/serialization.ts create mode 100644 test/failure-recovery-serialization.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c29d53..8242f8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +- Make the failure-recovery demo's serialization proof count messages rather + than executions. A worker that loses its lease mid-operation leaves the + replacement to execute the same message again, which is the at-least-once + contract, so the proof failed on slower machines for behaviour it documents + elsewhere. It now asserts that executions never overlap, that every start has + a finish, and that exactly the two sent messages ran, with the committed state + check unchanged. `assertSerializedExecution` moved into its own module and has + unit coverage for the retry, overlap, unfinished, and lost-message cases. + ## 0.13.2 - 2026-08-17 - Accept a `key` on `schedule`, naming a reminder for the item it is waiting diff --git a/examples/failure-recovery/demo.ts b/examples/failure-recovery/demo.ts index 9ab5edd..2643261 100644 --- a/examples/failure-recovery/demo.ts +++ b/examples/failure-recovery/demo.ts @@ -8,6 +8,7 @@ import { fork, type ChildProcess } from "node:child_process" import { createRuntime, type ActorReference, type MessageReference } from "solid-objects" import { sqlite } from "solid-objects/database/sqlite" import { RecoveryCounter } from "./actor.ts" +import { assertSerializedExecution, parseSerializationEvent } from "./serialization.ts" interface WorkerMessage { event: string @@ -15,12 +16,6 @@ interface WorkerMessage { processed?: number } -interface SerializationEvent { - event: "start" | "finish" - messageId: string - at: number -} - interface ExternalEffectEvent { messageId: string attempt: number @@ -74,12 +69,7 @@ async function proveSerialization(): Promise<{ finalState: number; overlap: fals join(controlDirectory, "serialization.jsonl"), parseSerializationEvent, ) - assert.equal(events.length, 4) - const starts = events.filter((event) => event.event === "start") - const finishes = events.filter((event) => event.event === "finish") - assert.equal(starts.length, 2) - assert.equal(finishes.length, 2) - assert(Number(starts[1]?.at) >= Number(finishes[0]?.at)) + assertSerializedExecution(events, { messageCount: 2 }) const snapshot = await reference.snapshot() assert.equal(snapshot.count, 2) return { finalState: snapshot.count, overlap: false } @@ -200,18 +190,6 @@ async function readJsonLines( return (await readFile(path, "utf8")).trim().split("\n").filter(Boolean).map(parse) } -function parseSerializationEvent(line: string): SerializationEvent { - const event = JSON.parse(line) as Partial - if ( - (event.event !== "start" && event.event !== "finish") || - typeof event.messageId !== "string" || - typeof event.at !== "number" - ) { - throw new TypeError("invalid serialization event") - } - return { event: event.event, messageId: event.messageId, at: event.at } -} - function parseExternalEffectEvent(line: string): ExternalEffectEvent { const event = JSON.parse(line) as Partial if ( diff --git a/examples/failure-recovery/serialization.ts b/examples/failure-recovery/serialization.ts new file mode 100644 index 0000000..4bf390a --- /dev/null +++ b/examples/failure-recovery/serialization.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict" + +export interface SerializationEvent { + event: "start" | "finish" + messageId: string + at: number +} + +export function parseSerializationEvent(line: string): SerializationEvent { + const event = JSON.parse(line) as Partial + if ( + (event.event !== "start" && event.event !== "finish") || + typeof event.messageId !== "string" || + typeof event.at !== "number" + ) { + throw new TypeError("invalid serialization event") + } + return { event: event.event, messageId: event.messageId, at: event.at } +} + +// One identity executes one operation at a time. A worker that loses its lease +// mid-operation leaves the replacement to execute the same message again, so +// the proof counts the messages that ran and the executions that overlapped, +// not the executions themselves. +export function assertSerializedExecution( + events: readonly SerializationEvent[], + options: { messageCount: number }, +): void { + const ordered = [...events].sort((left, right) => left.at - right.at) + + let open: string | undefined + for (const event of ordered) { + if (event.event === "start") { + assert.equal( + open, + undefined, + `execution of ${event.messageId} overlaps the open execution of ${open}`, + ) + open = event.messageId + continue + } + assert.equal(event.messageId, open, `finish of ${event.messageId} has no matching start`) + open = undefined + } + assert.equal(open, undefined, `execution of ${open} never wrote a finish`) + + const messageIds = new Set(ordered.map((event) => event.messageId)) + assert.equal( + messageIds.size, + options.messageCount, + `expected ${options.messageCount} messages to run, saw ${messageIds.size}`, + ) +} diff --git a/test/failure-recovery-serialization.test.ts b/test/failure-recovery-serialization.test.ts new file mode 100644 index 0000000..4698b0e --- /dev/null +++ b/test/failure-recovery-serialization.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest" +import { + assertSerializedExecution, + type SerializationEvent, +} from "../examples/failure-recovery/serialization.js" + +function execution(messageId: string, startedAt: number, finishedAt: number): SerializationEvent[] { + return [ + { event: "start", messageId, at: startedAt }, + { event: "finish", messageId, at: finishedAt }, + ] +} + +describe("serialization proof", () => { + it("accepts two executions that do not overlap", () => { + const events = [...execution("a", 10, 20), ...execution("b", 30, 40)] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).not.toThrow() + }) + + // A worker can lose its lease mid-operation, and the replacement executes the + // same message again. That is the at-least-once contract, so the proof counts + // messages rather than executions. + it("accepts a message that executes twice after a lost lease", () => { + const events = [...execution("a", 10, 20), ...execution("a", 30, 40), ...execution("b", 50, 60)] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).not.toThrow() + }) + + it("rejects executions that overlap", () => { + const events: SerializationEvent[] = [ + { event: "start", messageId: "a", at: 10 }, + { event: "start", messageId: "b", at: 15 }, + { event: "finish", messageId: "a", at: 20 }, + { event: "finish", messageId: "b", at: 25 }, + ] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/overlap/) + }) + + it("rejects an execution that never finished", () => { + const events = [...execution("a", 10, 20), { event: "start", messageId: "b", at: 30 } as const] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/finish/) + }) + + it("rejects a run that lost one of the messages", () => { + const events = [...execution("a", 10, 20), ...execution("a", 30, 40)] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/message/) + }) +}) From fd29399b1ee9fd246e3ae77f03ffa7e5a4ce7c08 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Tue, 18 Aug 2026 09:11:04 -0700 Subject: [PATCH 5/7] fix: let a fenced attempt overlap its replacement The first rule demanded strict alternation across the whole log, which rejects the case it was written to accept. The attempt that loses its lease is the one that was slow, so it is still running when the replacement starts, and both write to the control file. Overlap is now allowed only when a message ran more than once, which is the only thing that produces a stale owner. Without a retry the proof still demands strict serialization. The rule tracks open attempts per message, so a message may be open twice. The demo reports executions, retried, and overlapped instead of a hardcoded overlap: false, so a run says what it actually saw. Found by Greptile on the first commit. The stale-overlap case is now a test, and it fails against the alternation rule. --- CHANGELOG.md | 11 +++-- examples/failure-recovery/demo.ts | 12 +++-- examples/failure-recovery/serialization.ts | 53 +++++++++++++++------ test/failure-recovery-serialization.test.ts | 44 +++++++++++++++-- 4 files changed, 94 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8242f8a..2b75528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,13 @@ than executions. A worker that loses its lease mid-operation leaves the replacement to execute the same message again, which is the at-least-once contract, so the proof failed on slower machines for behaviour it documents - elsewhere. It now asserts that executions never overlap, that every start has - a finish, and that exactly the two sent messages ran, with the committed state - check unchanged. `assertSerializedExecution` moved into its own module and has - unit coverage for the retry, overlap, unfinished, and lost-message cases. + elsewhere. It now asserts that every start has a finish, that exactly the two + sent messages ran, and that executions overlap only when a lost lease explains + it, because the stale attempt keeps running until it notices. The committed + state check is unchanged, and the demo reports the executions it saw. + `assertSerializedExecution` moved into its own module with unit coverage for + the clean, retried, stale-overlap, unexplained-overlap, unfinished, + unmatched-finish, and lost-message cases. ## 0.13.2 - 2026-08-17 diff --git a/examples/failure-recovery/demo.ts b/examples/failure-recovery/demo.ts index 2643261..ba0f3a7 100644 --- a/examples/failure-recovery/demo.ts +++ b/examples/failure-recovery/demo.ts @@ -8,7 +8,11 @@ import { fork, type ChildProcess } from "node:child_process" import { createRuntime, type ActorReference, type MessageReference } from "solid-objects" import { sqlite } from "solid-objects/database/sqlite" import { RecoveryCounter } from "./actor.ts" -import { assertSerializedExecution, parseSerializationEvent } from "./serialization.ts" +import { + assertSerializedExecution, + parseSerializationEvent, + type SerializationProof, +} from "./serialization.ts" interface WorkerMessage { event: string @@ -54,7 +58,7 @@ try { assert.equal(existsSync(directory), false) -async function proveSerialization(): Promise<{ finalState: number; overlap: false }> { +async function proveSerialization(): Promise { const controlDirectory = join(directory, "serialization") await mkdir(controlDirectory) const reference = runtime.ref(RecoveryCounter, "serialized") @@ -69,10 +73,10 @@ async function proveSerialization(): Promise<{ finalState: number; overlap: fals join(controlDirectory, "serialization.jsonl"), parseSerializationEvent, ) - assertSerializedExecution(events, { messageCount: 2 }) + const proof = assertSerializedExecution(events, { messageCount: 2 }) const snapshot = await reference.snapshot() assert.equal(snapshot.count, 2) - return { finalState: snapshot.count, overlap: false } + return { ...proof, finalState: snapshot.count } } async function proveCrashRecovery(): Promise<{ diff --git a/examples/failure-recovery/serialization.ts b/examples/failure-recovery/serialization.ts index 4bf390a..d14444d 100644 --- a/examples/failure-recovery/serialization.ts +++ b/examples/failure-recovery/serialization.ts @@ -6,6 +6,12 @@ export interface SerializationEvent { at: number } +export interface SerializationProof { + executions: number + retried: boolean + overlapped: boolean +} + export function parseSerializationEvent(line: string): SerializationEvent { const event = JSON.parse(line) as Partial if ( @@ -18,31 +24,41 @@ export function parseSerializationEvent(line: string): SerializationEvent { return { event: event.event, messageId: event.messageId, at: event.at } } -// One identity executes one operation at a time. A worker that loses its lease -// mid-operation leaves the replacement to execute the same message again, so -// the proof counts the messages that ran and the executions that overlapped, -// not the executions themselves. +// One identity commits one state transition at a time. The control file is +// written outside the transaction, so it records execution attempts rather than +// commits: a worker that loses its lease keeps running until it notices, and +// its replacement executes the same message again. Those two attempts can +// overlap in this log, and the fenced write is what stops them both counting. +// The committed state is the assertion that proves it. +// +// Without a retry there is no stale owner, so nothing excuses an overlap and +// the proof still demands strict serialization. export function assertSerializedExecution( events: readonly SerializationEvent[], options: { messageCount: number }, -): void { +): SerializationProof { const ordered = [...events].sort((left, right) => left.at - right.at) - let open: string | undefined + const open = new Map() + let running = 0 + let concurrent = 0 + let executions = 0 + for (const event of ordered) { if (event.event === "start") { - assert.equal( - open, - undefined, - `execution of ${event.messageId} overlaps the open execution of ${open}`, - ) - open = event.messageId + open.set(event.messageId, (open.get(event.messageId) ?? 0) + 1) + executions += 1 + running += 1 + concurrent = Math.max(concurrent, running) continue } - assert.equal(event.messageId, open, `finish of ${event.messageId} has no matching start`) - open = undefined + const openForMessage = open.get(event.messageId) ?? 0 + assert(openForMessage > 0, `finish of ${event.messageId} has no matching start`) + open.set(event.messageId, openForMessage - 1) + running -= 1 } - assert.equal(open, undefined, `execution of ${open} never wrote a finish`) + + assert.equal(running, 0, "an execution never wrote a finish") const messageIds = new Set(ordered.map((event) => event.messageId)) assert.equal( @@ -50,4 +66,11 @@ export function assertSerializedExecution( options.messageCount, `expected ${options.messageCount} messages to run, saw ${messageIds.size}`, ) + + const retried = executions > options.messageCount + if (!retried) { + assert.equal(concurrent, 1, "executions overlapped without a lost lease to explain it") + } + + return { executions, retried, overlapped: concurrent > 1 } } diff --git a/test/failure-recovery-serialization.test.ts b/test/failure-recovery-serialization.test.ts index 4698b0e..1d86c49 100644 --- a/test/failure-recovery-serialization.test.ts +++ b/test/failure-recovery-serialization.test.ts @@ -15,7 +15,11 @@ describe("serialization proof", () => { it("accepts two executions that do not overlap", () => { const events = [...execution("a", 10, 20), ...execution("b", 30, 40)] - expect(() => assertSerializedExecution(events, { messageCount: 2 })).not.toThrow() + expect(assertSerializedExecution(events, { messageCount: 2 })).toEqual({ + executions: 2, + retried: false, + overlapped: false, + }) }) // A worker can lose its lease mid-operation, and the replacement executes the @@ -24,10 +28,34 @@ describe("serialization proof", () => { it("accepts a message that executes twice after a lost lease", () => { const events = [...execution("a", 10, 20), ...execution("a", 30, 40), ...execution("b", 50, 60)] - expect(() => assertSerializedExecution(events, { messageCount: 2 })).not.toThrow() + expect(assertSerializedExecution(events, { messageCount: 2 })).toEqual({ + executions: 3, + retried: true, + overlapped: false, + }) }) - it("rejects executions that overlap", () => { + // The stale attempt is what lost the lease, so it is still running when the + // replacement starts. Its writes are fenced out, and the committed state is + // what proves that, so the log is allowed to interleave here. + it("accepts a stale attempt that is still running when its replacement starts", () => { + const events: SerializationEvent[] = [ + { event: "start", messageId: "a", at: 10 }, + { event: "start", messageId: "a", at: 20 }, + { event: "finish", messageId: "a", at: 30 }, + { event: "finish", messageId: "a", at: 40 }, + ...execution("b", 50, 60), + ] + + expect(assertSerializedExecution(events, { messageCount: 2 })).toEqual({ + executions: 3, + retried: true, + overlapped: true, + }) + }) + + // Without a retry there is no stale owner, so nothing excuses an overlap. + it("rejects executions that overlap when no message ran twice", () => { const events: SerializationEvent[] = [ { event: "start", messageId: "a", at: 10 }, { event: "start", messageId: "b", at: 15 }, @@ -44,6 +72,16 @@ describe("serialization proof", () => { expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/finish/) }) + it("rejects a finish with no start", () => { + const events: SerializationEvent[] = [ + ...execution("a", 10, 20), + { event: "finish", messageId: "b", at: 30 }, + ...execution("b", 40, 50), + ] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/start/) + }) + it("rejects a run that lost one of the messages", () => { const events = [...execution("a", 10, 20), ...execution("a", 30, 40)] From 59e4a4f32ec77cd8ea9eb1e3d77b06be1b6f01a5 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Tue, 18 Aug 2026 09:16:46 -0700 Subject: [PATCH 6/7] fix: excuse the superseded attempt, not the survivors Treating any retry as a licence to overlap let a real violation through: once one message retried, two different messages could run at the same time and the proof still passed. A retry excuses only the attempt it superseded. The rule now tracks which attempt of each message is the last one started, and asserts that at most one message has its surviving attempt open at any moment. Earlier attempts may overlap anything, because their writes are fenced. Found by Greptile on the previous commit. The case is now a test, and removing the assertion fails it along with the no-retry overlap case. --- CHANGELOG.md | 9 +++-- examples/failure-recovery/serialization.ts | 45 ++++++++++++++------- test/failure-recovery-serialization.test.ts | 14 +++++++ 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b75528..402c3dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,13 @@ replacement to execute the same message again, which is the at-least-once contract, so the proof failed on slower machines for behaviour it documents elsewhere. It now asserts that every start has a finish, that exactly the two - sent messages ran, and that executions overlap only when a lost lease explains - it, because the stale attempt keeps running until it notices. The committed + sent messages ran, and that the surviving attempt of each message had the + identity to itself. A superseded attempt may overlap, because it keeps running + until it notices the lost lease and its write is fenced out. The committed state check is unchanged, and the demo reports the executions it saw. `assertSerializedExecution` moved into its own module with unit coverage for - the clean, retried, stale-overlap, unexplained-overlap, unfinished, - unmatched-finish, and lost-message cases. + the clean, retried, stale-overlap, surviving-overlap, unexplained-overlap, + unfinished, unmatched-finish, and lost-message cases. ## 0.13.2 - 2026-08-17 diff --git a/examples/failure-recovery/serialization.ts b/examples/failure-recovery/serialization.ts index d14444d..4296116 100644 --- a/examples/failure-recovery/serialization.ts +++ b/examples/failure-recovery/serialization.ts @@ -27,35 +27,51 @@ export function parseSerializationEvent(line: string): SerializationEvent { // One identity commits one state transition at a time. The control file is // written outside the transaction, so it records execution attempts rather than // commits: a worker that loses its lease keeps running until it notices, and -// its replacement executes the same message again. Those two attempts can -// overlap in this log, and the fenced write is what stops them both counting. -// The committed state is the assertion that proves it. +// its replacement executes the same message again. A superseded attempt may +// therefore overlap anything, because its write is fenced out and the committed +// state is what proves it. // -// Without a retry there is no stale owner, so nothing excuses an overlap and -// the proof still demands strict serialization. +// The surviving attempts are the ones that count, so the last attempt of each +// message must still have the identity to itself. export function assertSerializedExecution( events: readonly SerializationEvent[], options: { messageCount: number }, ): SerializationProof { const ordered = [...events].sort((left, right) => left.at - right.at) - const open = new Map() + const lastStartedAt = new Map() + for (const event of ordered) { + if (event.event === "start") lastStartedAt.set(event.messageId, event.at) + } + + const open = new Map() + const surviving = new Set() let running = 0 let concurrent = 0 let executions = 0 for (const event of ordered) { if (event.event === "start") { - open.set(event.messageId, (open.get(event.messageId) ?? 0) + 1) + open.set(event.messageId, [...(open.get(event.messageId) ?? []), event.at]) executions += 1 running += 1 concurrent = Math.max(concurrent, running) + if (event.at === lastStartedAt.get(event.messageId)) { + surviving.add(event.messageId) + assert.equal( + surviving.size, + 1, + `${[...surviving].join(" and ")} overlap, and neither was superseded by a retry`, + ) + } continue } - const openForMessage = open.get(event.messageId) ?? 0 - assert(openForMessage > 0, `finish of ${event.messageId} has no matching start`) - open.set(event.messageId, openForMessage - 1) + const openForMessage = open.get(event.messageId) ?? [] + assert(openForMessage.length > 0, `finish of ${event.messageId} has no matching start`) + const startedAt = openForMessage.pop() + open.set(event.messageId, openForMessage) running -= 1 + if (startedAt === lastStartedAt.get(event.messageId)) surviving.delete(event.messageId) } assert.equal(running, 0, "an execution never wrote a finish") @@ -67,10 +83,9 @@ export function assertSerializedExecution( `expected ${options.messageCount} messages to run, saw ${messageIds.size}`, ) - const retried = executions > options.messageCount - if (!retried) { - assert.equal(concurrent, 1, "executions overlapped without a lost lease to explain it") + return { + executions, + retried: executions > options.messageCount, + overlapped: concurrent > 1, } - - return { executions, retried, overlapped: concurrent > 1 } } diff --git a/test/failure-recovery-serialization.test.ts b/test/failure-recovery-serialization.test.ts index 1d86c49..9b6e8a3 100644 --- a/test/failure-recovery-serialization.test.ts +++ b/test/failure-recovery-serialization.test.ts @@ -54,6 +54,20 @@ describe("serialization proof", () => { }) }) + // A retry excuses the superseded attempt, not the surviving ones. Message a + // retries, and then the attempt that replaced it runs at the same time as b. + it("rejects overlap between the surviving attempts even after a retry", () => { + const events: SerializationEvent[] = [ + ...execution("a", 10, 20), + { event: "start", messageId: "a", at: 30 }, + { event: "start", messageId: "b", at: 35 }, + { event: "finish", messageId: "a", at: 40 }, + { event: "finish", messageId: "b", at: 45 }, + ] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/overlap/) + }) + // Without a retry there is no stale owner, so nothing excuses an overlap. it("rejects executions that overlap when no message ran twice", () => { const events: SerializationEvent[] = [ From b201fe2d01387fc238541af6f8c9c74d8b5add47 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Tue, 18 Aug 2026 10:40:01 -0700 Subject: [PATCH 7/7] fix: pair each start with its own finish Two attempts of the same message can both be writing to the control file at once: the superseded attempt keeps running until it notices the lost lease, and its finish can land after the replacement's start. The previous rule paired a start with whichever finish came next, so that late finish read as the replacement's, and a second message could then overlap a replacement that was in fact still running. Each event now carries the message's attempt and the writing process, so a start pairs only with the finish of that same execution. The proof still asserts one thing about the surviving attempts, that none of them overlap another message's surviving attempt, and lets a superseded attempt overlap anything, because its write is fenced out and the committed state is the assertion that proves it. Found by Greptile on the previous commit, and reproduced as a test: a superseded attempt that finishes after its replacement starts, followed by a second message that starts while the replacement is still running. Removing the overlap assertion fails that case and the no-retry case. --- CHANGELOG.md | 19 +-- examples/failure-recovery/actor.ts | 14 +- examples/failure-recovery/serialization.ts | 138 +++++++++++++------- test/failure-recovery-serialization.test.ts | 124 ++++++++++++------ 4 files changed, 191 insertions(+), 104 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 402c3dc..ed09695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,17 @@ than executions. A worker that loses its lease mid-operation leaves the replacement to execute the same message again, which is the at-least-once contract, so the proof failed on slower machines for behaviour it documents - elsewhere. It now asserts that every start has a finish, that exactly the two - sent messages ran, and that the surviving attempt of each message had the - identity to itself. A superseded attempt may overlap, because it keeps running - until it notices the lost lease and its write is fenced out. The committed - state check is unchanged, and the demo reports the executions it saw. - `assertSerializedExecution` moved into its own module with unit coverage for - the clean, retried, stale-overlap, surviving-overlap, unexplained-overlap, - unfinished, unmatched-finish, and lost-message cases. + elsewhere. Each serialization event now carries its attempt and process, so a + start pairs with its own finish instead of with whichever finish came next. + The proof asserts that every start has its own finish, that exactly the two + sent messages ran, and that the surviving attempt of each message never + overlaps another message's surviving attempt; a superseded attempt may + overlap anything, because it keeps running until it notices the lost lease and + its write is fenced out. The committed state check is unchanged, and the demo + reports the executions it saw. `assertSerializedExecution` moved into its own + module with unit coverage for the clean, retried, superseded-overlap, + still-running-replacement, unexplained-overlap, boundary, unfinished, + unmatched-finish, double-start, restart-after-finish, and lost-message cases. ## 0.13.2 - 2026-08-17 diff --git a/examples/failure-recovery/actor.ts b/examples/failure-recovery/actor.ts index a6235b1..c372732 100644 --- a/examples/failure-recovery/actor.ts +++ b/examples/failure-recovery/actor.ts @@ -25,16 +25,14 @@ export class RecoveryCounter extends Actor { async serialize({ controlDirectory }: { controlDirectory: string }): Promise { const message = this.currentMessage if (!message) throw new Error("serialize requires a durable message") - await appendFile( - join(controlDirectory, "serialization.jsonl"), - `${JSON.stringify({ event: "start", messageId: message.id, at: Date.now() })}\n`, - ) + // The attempt and the process identify the execution, so a start pairs with + // its own finish even when a superseded attempt outlives its replacement. + const execution = { messageId: message.id, attempt: message.attempt, processId: process.pid } + const path = join(controlDirectory, "serialization.jsonl") + await appendFile(path, `${JSON.stringify({ event: "start", ...execution, at: Date.now() })}\n`) await new Promise((resolve) => setTimeout(resolve, 100)) this.count += 1 - await appendFile( - join(controlDirectory, "serialization.jsonl"), - `${JSON.stringify({ event: "finish", messageId: message.id, at: Date.now() })}\n`, - ) + await appendFile(path, `${JSON.stringify({ event: "finish", ...execution, at: Date.now() })}\n`) return this.count } } diff --git a/examples/failure-recovery/serialization.ts b/examples/failure-recovery/serialization.ts index 4296116..61cdd64 100644 --- a/examples/failure-recovery/serialization.ts +++ b/examples/failure-recovery/serialization.ts @@ -3,13 +3,23 @@ import assert from "node:assert/strict" export interface SerializationEvent { event: "start" | "finish" messageId: string + attempt: number + processId: number at: number } export interface SerializationProof { executions: number retried: boolean - overlapped: boolean + supersededOverlap: boolean +} + +interface Execution { + messageId: string + attempt: number + processId: number + startedAt: number + finishedAt: number } export function parseSerializationEvent(line: string): SerializationEvent { @@ -17,75 +27,107 @@ export function parseSerializationEvent(line: string): SerializationEvent { if ( (event.event !== "start" && event.event !== "finish") || typeof event.messageId !== "string" || + typeof event.attempt !== "number" || + typeof event.processId !== "number" || typeof event.at !== "number" ) { throw new TypeError("invalid serialization event") } - return { event: event.event, messageId: event.messageId, at: event.at } + return { + event: event.event, + messageId: event.messageId, + attempt: event.attempt, + processId: event.processId, + at: event.at, + } } // One identity commits one state transition at a time. The control file is // written outside the transaction, so it records execution attempts rather than -// commits: a worker that loses its lease keeps running until it notices, and -// its replacement executes the same message again. A superseded attempt may -// therefore overlap anything, because its write is fenced out and the committed -// state is what proves it. +// commits: a worker that loses its lease keeps running until it notices, and its +// replacement executes the same message under a higher attempt. The superseded +// attempt may therefore overlap anything, because its write is fenced out and +// the committed state is what proves it. // -// The surviving attempts are the ones that count, so the last attempt of each -// message must still have the identity to itself. +// Each event carries its attempt and process, so a start pairs with its own +// finish rather than with whichever finish arrived next. Without that, a +// superseded attempt finishing late reads as its replacement finishing, and a +// second message could then overlap a replacement that is still running. export function assertSerializedExecution( events: readonly SerializationEvent[], options: { messageCount: number }, ): SerializationProof { - const ordered = [...events].sort((left, right) => left.at - right.at) + const executions = pairExecutions(events) + + const messageIds = new Set(executions.map((execution) => execution.messageId)) + assert.equal( + messageIds.size, + options.messageCount, + `expected ${options.messageCount} messages to run, saw ${messageIds.size}`, + ) + + const survivingAttempt = new Map() + for (const execution of executions) { + const highest = survivingAttempt.get(execution.messageId) ?? 0 + if (execution.attempt > highest) survivingAttempt.set(execution.messageId, execution.attempt) + } + const surviving = executions.filter( + (execution) => survivingAttempt.get(execution.messageId) === execution.attempt, + ) - const lastStartedAt = new Map() - for (const event of ordered) { - if (event.event === "start") lastStartedAt.set(event.messageId, event.at) + for (const [index, execution] of surviving.entries()) { + for (const other of surviving.slice(index + 1)) { + assert( + !overlaps(execution, other), + `${describe(execution)} and ${describe(other)} overlap, and neither was superseded`, + ) + } } - const open = new Map() - const surviving = new Set() - let running = 0 - let concurrent = 0 - let executions = 0 + const supersededOverlap = executions.some((execution) => + executions.some((other) => other !== execution && overlaps(execution, other)), + ) - for (const event of ordered) { + return { + executions: executions.length, + retried: executions.length > options.messageCount, + supersededOverlap, + } +} + +function pairExecutions(events: readonly SerializationEvent[]): Execution[] { + const started = new Map() + const executions: Execution[] = [] + + for (const event of [...events].sort((left, right) => left.at - right.at)) { + const key = `${event.messageId}#${event.attempt}#${event.processId}` if (event.event === "start") { - open.set(event.messageId, [...(open.get(event.messageId) ?? []), event.at]) - executions += 1 - running += 1 - concurrent = Math.max(concurrent, running) - if (event.at === lastStartedAt.get(event.messageId)) { - surviving.add(event.messageId) - assert.equal( - surviving.size, - 1, - `${[...surviving].join(" and ")} overlap, and neither was superseded by a retry`, - ) - } + assert(!started.has(key), `${describe(event)} started twice`) + started.set(key, event) continue } - const openForMessage = open.get(event.messageId) ?? [] - assert(openForMessage.length > 0, `finish of ${event.messageId} has no matching start`) - const startedAt = openForMessage.pop() - open.set(event.messageId, openForMessage) - running -= 1 - if (startedAt === lastStartedAt.get(event.messageId)) surviving.delete(event.messageId) + const start = started.get(key) + assert(start !== undefined, `${describe(event)} finished with no matching start`) + started.delete(key) + executions.push({ + messageId: event.messageId, + attempt: event.attempt, + processId: event.processId, + startedAt: start.at, + finishedAt: event.at, + }) } - assert.equal(running, 0, "an execution never wrote a finish") + const unfinished = [...started.values()].map(describe) + assert.equal(unfinished.length, 0, `${unfinished.join(", ")} never wrote a finish`) - const messageIds = new Set(ordered.map((event) => event.messageId)) - assert.equal( - messageIds.size, - options.messageCount, - `expected ${options.messageCount} messages to run, saw ${messageIds.size}`, - ) + return executions +} - return { - executions, - retried: executions > options.messageCount, - overlapped: concurrent > 1, - } +function overlaps(left: Execution, right: Execution): boolean { + return left.startedAt < right.finishedAt && right.startedAt < left.finishedAt +} + +function describe(execution: { messageId: string; attempt: number }): string { + return `${execution.messageId} attempt ${execution.attempt}` } diff --git a/test/failure-recovery-serialization.test.ts b/test/failure-recovery-serialization.test.ts index 9b6e8a3..0982178 100644 --- a/test/failure-recovery-serialization.test.ts +++ b/test/failure-recovery-serialization.test.ts @@ -4,100 +4,144 @@ import { type SerializationEvent, } from "../examples/failure-recovery/serialization.js" -function execution(messageId: string, startedAt: number, finishedAt: number): SerializationEvent[] { +function execution(options: { + messageId: string + attempt: number + startedAt: number + finishedAt: number + processId?: number +}): SerializationEvent[] { + const { messageId, attempt, startedAt, finishedAt, processId = attempt } = options return [ - { event: "start", messageId, at: startedAt }, - { event: "finish", messageId, at: finishedAt }, + { event: "start", messageId, attempt, processId, at: startedAt }, + { event: "finish", messageId, attempt, processId, at: finishedAt }, ] } describe("serialization proof", () => { it("accepts two executions that do not overlap", () => { - const events = [...execution("a", 10, 20), ...execution("b", 30, 40)] + const events = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + ...execution({ messageId: "b", attempt: 1, startedAt: 30, finishedAt: 40 }), + ] expect(assertSerializedExecution(events, { messageCount: 2 })).toEqual({ executions: 2, retried: false, - overlapped: false, + supersededOverlap: false, }) }) // A worker can lose its lease mid-operation, and the replacement executes the - // same message again. That is the at-least-once contract, so the proof counts - // messages rather than executions. + // same message again under a higher attempt. That is the at-least-once + // contract, so the proof counts messages rather than executions. it("accepts a message that executes twice after a lost lease", () => { - const events = [...execution("a", 10, 20), ...execution("a", 30, 40), ...execution("b", 50, 60)] + const events = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + ...execution({ messageId: "a", attempt: 2, startedAt: 30, finishedAt: 40 }), + ...execution({ messageId: "b", attempt: 1, startedAt: 50, finishedAt: 60 }), + ] expect(assertSerializedExecution(events, { messageCount: 2 })).toEqual({ executions: 3, retried: true, - overlapped: false, + supersededOverlap: false, }) }) - // The stale attempt is what lost the lease, so it is still running when the - // replacement starts. Its writes are fenced out, and the committed state is - // what proves that, so the log is allowed to interleave here. - it("accepts a stale attempt that is still running when its replacement starts", () => { - const events: SerializationEvent[] = [ - { event: "start", messageId: "a", at: 10 }, - { event: "start", messageId: "a", at: 20 }, - { event: "finish", messageId: "a", at: 30 }, - { event: "finish", messageId: "a", at: 40 }, - ...execution("b", 50, 60), + // The attempt that lost the lease is the one that was slow, so it is still + // running when its replacement starts. Its write is fenced out, and the + // committed state is what proves that, so the log may interleave here. + it("accepts a superseded attempt that outlives the start of its replacement", () => { + const events = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 30 }), + ...execution({ messageId: "a", attempt: 2, startedAt: 20, finishedAt: 40 }), + ...execution({ messageId: "b", attempt: 1, startedAt: 50, finishedAt: 60 }), ] expect(assertSerializedExecution(events, { messageCount: 2 })).toEqual({ executions: 3, retried: true, - overlapped: true, + supersededOverlap: true, }) }) - // A retry excuses the superseded attempt, not the surviving ones. Message a - // retries, and then the attempt that replaced it runs at the same time as b. - it("rejects overlap between the surviving attempts even after a retry", () => { - const events: SerializationEvent[] = [ - ...execution("a", 10, 20), - { event: "start", messageId: "a", at: 30 }, - { event: "start", messageId: "b", at: 35 }, - { event: "finish", messageId: "a", at: 40 }, - { event: "finish", messageId: "b", at: 45 }, + // The superseded attempt finishing late must not be read as its replacement + // finishing. The replacement is still running, so a second message that + // starts here is a real serialization failure. + it("rejects a second message that overlaps a still-running replacement", () => { + const events = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 35 }), + ...execution({ messageId: "a", attempt: 2, startedAt: 20, finishedAt: 60 }), + ...execution({ messageId: "b", attempt: 1, startedAt: 40, finishedAt: 50 }), ] expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/overlap/) }) - // Without a retry there is no stale owner, so nothing excuses an overlap. it("rejects executions that overlap when no message ran twice", () => { - const events: SerializationEvent[] = [ - { event: "start", messageId: "a", at: 10 }, - { event: "start", messageId: "b", at: 15 }, - { event: "finish", messageId: "a", at: 20 }, - { event: "finish", messageId: "b", at: 25 }, + const events = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + ...execution({ messageId: "b", attempt: 1, startedAt: 15, finishedAt: 25 }), ] expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/overlap/) }) + it("accepts one execution that ends exactly as the next begins", () => { + const events = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + ...execution({ messageId: "b", attempt: 1, startedAt: 20, finishedAt: 30 }), + ] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).not.toThrow() + }) + it("rejects an execution that never finished", () => { - const events = [...execution("a", 10, 20), { event: "start", messageId: "b", at: 30 } as const] + const events: SerializationEvent[] = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + { event: "start", messageId: "b", attempt: 1, processId: 1, at: 30 }, + ] expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/finish/) }) it("rejects a finish with no start", () => { const events: SerializationEvent[] = [ - ...execution("a", 10, 20), - { event: "finish", messageId: "b", at: 30 }, - ...execution("b", 40, 50), + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + { event: "finish", messageId: "b", attempt: 1, processId: 1, at: 30 }, ] expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/start/) }) + it("rejects one attempt that started twice", () => { + const events: SerializationEvent[] = [ + { event: "start", messageId: "a", attempt: 1, processId: 1, at: 10 }, + { event: "start", messageId: "a", attempt: 1, processId: 1, at: 15 }, + { event: "finish", messageId: "a", attempt: 1, processId: 1, at: 20 }, + { event: "finish", messageId: "a", attempt: 1, processId: 1, at: 25 }, + ...execution({ messageId: "b", attempt: 1, startedAt: 30, finishedAt: 40 }), + ] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/twice/) + }) + + it("rejects an attempt that starts again after it finished", () => { + const events: SerializationEvent[] = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + { event: "start", messageId: "a", attempt: 1, processId: 1, at: 25 }, + ...execution({ messageId: "b", attempt: 1, startedAt: 30, finishedAt: 40 }), + ] + + expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/finish/) + }) + it("rejects a run that lost one of the messages", () => { - const events = [...execution("a", 10, 20), ...execution("a", 30, 40)] + const events = [ + ...execution({ messageId: "a", attempt: 1, startedAt: 10, finishedAt: 20 }), + ...execution({ messageId: "a", attempt: 2, startedAt: 30, finishedAt: 40 }), + ] expect(() => assertSerializedExecution(events, { messageCount: 2 })).toThrow(/message/) })