diff --git a/.changeset/reuse-prepared-replay-payloads.md b/.changeset/reuse-prepared-replay-payloads.md new file mode 100644 index 0000000000..c2578131cd --- /dev/null +++ b/.changeset/reuse-prepared-replay-payloads.md @@ -0,0 +1,9 @@ +--- +"@workflow/core": patch +"@workflow/cli": patch +"@workflow/web-shared": patch +"@workflow/world": patch +"@workflow/world-vercel": patch +--- + +Simplify payload codecs and reuse prepared replay payloads across workflow VMs. diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index eb043aa7d2..b74fc9a622 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -6,7 +6,12 @@ */ import { inspect } from 'node:util'; -import { getCommonRevivers, maybeDecrypt } from '@workflow/core/serialization'; +import { + type DecryptionKey, + decrypt, + deriveRunPayloadKeys, + getCommonRevivers, +} from '@workflow/core/serialization'; import { ClassInstanceRef, extractClassName, @@ -22,14 +27,12 @@ import { parseClassName } from '@workflow/utils/parse-name'; import { getEventDataRefFields } from '@workflow/world'; import chalk from 'chalk'; -/** - * A function that resolves an encryption key for a run, or null to skip - * decryption. Accepts a runId — the resolver is responsible for looking - * up the WorkflowRun internally (with caching) if the World needs it. - */ -export type EncryptionKeyResolver = - | ((runId: string) => Promise) - | null; +async function decryptPayload( + value: unknown, + key: DecryptionKey | undefined +): Promise { + return value instanceof Uint8Array ? decrypt(value, key) : value; +} // Re-export types and utilities that consumers need export { @@ -313,52 +316,47 @@ function getRevivers(): Revivers { * Pre-process a resource's data fields: if the resolver is provided and * the field is encrypted, decrypt it before generic hydration. * - * Uses core's `maybeDecrypt()` which handles the 'encr' prefix stripping - * and AES-GCM decryption transparently. + * Binary envelopes go through the core decryptor; legacy values remain + * unchanged for the generic hydrator. * - * When the resolver is null (no --decrypt flag), encrypted fields pass + * Without a resolver (no --decrypt flag), encrypted fields pass * through as Uint8Array and are replaced with EncryptedDataRef in post-processing. */ -async function maybeDecryptFields< - T extends { - runId?: string; - input?: any; - output?: any; - metadata?: any; - eventType?: string; - eventData?: any; - }, ->(resource: T, resolver: EncryptionKeyResolver): Promise { - if (!resolver) return resource; +async function maybeDecryptFields( + resource: T, + resolveKey?: (runId: string) => Promise +): Promise { + if (!resolveKey || !resource || typeof resource !== 'object') return resource; - const runId = (resource as any).runId as string | undefined; - if (!runId) return resource; + const source = resource as Record; + if (typeof source.runId !== 'string') return resource; - const result = { ...resource }; + const result = { ...source }; try { - const rawKey = await resolver(runId); + const rawKey = await resolveKey(source.runId); // Resolve the full key capability so `--decrypt` can open sealed // ('encp') payloads that other runs wrote to this one, not just the // run's own symmetric ('encr') payloads. - const { deriveRunPayloadKeys } = await import( - '@workflow/core/serialization' - ); const k = rawKey ? await deriveRunPayloadKeys(rawKey) : undefined; // Decrypt input/output/error fields (WorkflowRun, Step) - result.input = await maybeDecrypt(result.input, k); - result.output = await maybeDecrypt(result.output, k); - (result as any).error = await maybeDecrypt((result as any).error, k); + result.input = await decryptPayload(result.input, k); + result.output = await decryptPayload(result.output, k); + result.error = await decryptPayload(result.error, k); // Decrypt metadata field (Hook) - result.metadata = await maybeDecrypt(result.metadata, k); + result.metadata = await decryptPayload(result.metadata, k); // Decrypt eventData fields (Event) if (result.eventData && typeof result.eventData === 'object') { - const eventData = { ...result.eventData }; - for (const field of getEventDataRefFields(result.eventType ?? '')) { - eventData[field] = await maybeDecrypt(eventData[field], k); + const eventData = { + ...(result.eventData as Record), + }; + const eventType = + typeof result.eventType === 'string' ? result.eventType : ''; + for (const field of getEventDataRefFields(eventType)) { + eventData[field] = await decryptPayload(eventData[field], k); } result.eventData = eventData; } @@ -375,10 +373,10 @@ async function maybeDecryptFields< // Decryption failed (bad key, corrupted ciphertext, etc.) — fall back // to showing encrypted placeholders instead of crashing the CLI. const { logger } = await import('../config/log.js'); - logger.warn(`Decryption failed for resource ${runId}: ${message}`); + logger.warn(`Decryption failed for resource ${source.runId}: ${message}`); } - return result; + return result as T; } // --------------------------------------------------------------------------- @@ -421,21 +419,30 @@ function replaceEncryptedAndExpiredWithRef(resource: T): T { /** * Hydrate the serialized data fields of a resource for CLI display. * - * When `encryptorResolver` is null (default / no --decrypt flag), encrypted + * Without `resolveKey` (the default / no --decrypt flag), encrypted * fields are shown as styled "🔒 Encrypted" placeholders via EncryptedDataRef. * - * When `encryptorResolver` is provided (--decrypt flag), encrypted fields + * When `resolveKey` is provided (--decrypt flag), encrypted fields * are decrypted before hydration so the actual user data is displayed. */ -export async function hydrateResourceIO( +export async function hydrateResourceIO< + T extends { + stepId?: string; + hookId?: string; + eventId?: string; + eventType?: string; + input?: unknown; + output?: unknown; + metadata?: unknown; + eventData?: unknown; + executionContext?: unknown; + }, +>( resource: T, - keyResolver?: EncryptionKeyResolver + resolveKey?: (runId: string) => Promise ): Promise { // Pre-process: decrypt any encrypted fields when a resolver is provided - const preprocessed = await maybeDecryptFields( - resource as any, - keyResolver ?? null - ); + const preprocessed = await maybeDecryptFields(resource, resolveKey); const hydrated = hydrateResourceIOGeneric(preprocessed, getRevivers()) as T; // Post-process: swap encrypted Uint8Arrays and expired stubs for CLI-styled objects return replaceEncryptedAndExpiredWithRef(hydrated); diff --git a/packages/cli/src/lib/inspect/output.ts b/packages/cli/src/lib/inspect/output.ts index 3785b9ca16..a4b9f0de90 100644 --- a/packages/cli/src/lib/inspect/output.ts +++ b/packages/cli/src/lib/inspect/output.ts @@ -25,7 +25,6 @@ import { isObservabilityUpgradeRequiredError, } from './errors.js'; import { - type EncryptionKeyResolver, hydrateResourceIO, isEncryptedRef, isExpiredRef, @@ -33,15 +32,18 @@ import { import { resolveTimeWindow } from './time-window.js'; /** - * Create an EncryptionKeyResolver from a World instance. - * Returns null if decrypt is false — encrypted data will show as a placeholder. + * Create a run-key lookup from a World instance. + * Returns undefined when encrypted data should remain a placeholder. * * The resolver fetches the full WorkflowRun (cached per runId) so that the * World can inspect deployment-specific fields for key resolution. */ -function createResolver(world: World, decrypt: boolean): EncryptionKeyResolver { - if (!decrypt) return null; - if (!world.getEncryptionKeyForRun) return null; +function createResolver( + world: World, + decrypt: boolean +): ((runId: string) => Promise) | undefined { + if (!decrypt) return; + if (!world.getEncryptionKeyForRun) return; const cache = new Map>(); return (runId: string) => { let cached = cache.get(runId); diff --git a/packages/core/src/encryption.test.ts b/packages/core/src/encryption.test.ts index f3544fc9e8..acfd0e7b93 100644 --- a/packages/core/src/encryption.test.ts +++ b/packages/core/src/encryption.test.ts @@ -14,9 +14,18 @@ async function getOtherKey(): Promise { return importKey(OTHER_RAW_KEY); } +async function captureError(action: () => Promise): Promise { + try { + await action(); + } catch (error) { + return error; + } + throw new Error('Expected operation to fail'); +} + describe('encryption', () => { describe('round-trip', () => { - it('encrypt() + decrypt() returns the original plaintext', async () => { + it('uses one asynchronous contract on Node', async () => { const key = await getKey(); const plaintext = new TextEncoder().encode('hello, workflow'); const ciphertext = await encrypt(key, plaintext); @@ -24,8 +33,9 @@ describe('encryption', () => { // Ciphertext is longer than plaintext: 12-byte nonce + 16-byte GCM tag. expect(ciphertext.byteLength).toBe(plaintext.byteLength + 12 + 16); - const decoded = await decrypt(key, ciphertext); - expect(new TextDecoder().decode(decoded)).toBe('hello, workflow'); + const decoded = decrypt(key, ciphertext); + expect(decoded).toBeInstanceOf(Promise); + expect(new TextDecoder().decode(await decoded)).toBe('hello, workflow'); }); }); @@ -42,20 +52,15 @@ describe('encryption', () => { const key = await getKey(); // 12-byte nonce + 16-byte tag = 28 bytes minimum. 10 bytes is too short. const tooShort = new Uint8Array(10).fill(0); - const error = await decrypt(key, tooShort).catch((e) => e); + const error = await captureError(() => decrypt(key, tooShort)); expect(RuntimeDecryptionError.is(error)).toBe(true); - expect(error.message).toMatch(/Encrypted data too short/); - expect(error.context).toMatchObject({ - operation: 'decrypt', - byteLength: 10, + expect(error).toMatchObject({ + message: expect.stringMatching(/Encrypted data too short/), + context: { operation: 'decrypt', byteLength: 10 }, }); }); it('throws RuntimeDecryptionError (not a bare OperationError) on auth-tag failure', async () => { - // GCM auth-tag verification failure surfaces from Node's Web - // Crypto API as `OperationError: The operation failed for an - // operation-specific reason at AESCipherJob.onDone`. The - // encryption module must rewrap this as a RuntimeDecryptionError. const key = await getKey(); const plaintext = new TextEncoder().encode('hello, workflow'); const ciphertext = await encrypt(key, plaintext); @@ -64,16 +69,14 @@ describe('encryption', () => { const tampered = new Uint8Array(ciphertext); tampered[tampered.length - 1] ^= 0xff; - const error = await decrypt(key, tampered).catch((e) => e); + const error = await captureError(() => decrypt(key, tampered)); expect(RuntimeDecryptionError.is(error)).toBe(true); - expect(error.cause).toBeDefined(); - // The original DOMException carries name OperationError on Node 20+, - // which is what the wrapping is meant to capture as cause. - const cause = error.cause as { name?: string }; - expect(cause?.name).toBe('OperationError'); - expect(error.context).toMatchObject({ - operation: 'decrypt', - byteLength: tampered.byteLength, + expect(error).toMatchObject({ + cause: expect.anything(), + context: { + operation: 'decrypt', + byteLength: tampered.byteLength, + }, }); }); @@ -85,11 +88,9 @@ describe('encryption', () => { new TextEncoder().encode('secret') ); - const error = await decrypt(readerKey, ciphertext).catch((e) => e); + const error = await captureError(() => decrypt(readerKey, ciphertext)); expect(RuntimeDecryptionError.is(error)).toBe(true); - // Wrong key → auth tag mismatch → same OperationError as ciphertext corruption. - const cause = error.cause as { name?: string }; - expect(cause?.name).toBe('OperationError'); + expect(error).toMatchObject({ cause: expect.anything() }); }); it('does not record a formatPrefix at the low-level layer', async () => { @@ -100,13 +101,14 @@ describe('encryption', () => { // The serialization layer attaches the real envelope prefix. const key = await getKey(); const bogus = new Uint8Array(28).fill(0x41); // 28 bytes, passes length check - const error = await decrypt(key, bogus).catch((e) => e); + const error = await captureError(() => decrypt(key, bogus)); expect(RuntimeDecryptionError.is(error)).toBe(true); - expect(error.context).toMatchObject({ - operation: 'decrypt', - byteLength: 28, + expect(error).toMatchObject({ + context: { operation: 'decrypt', byteLength: 28 }, + }); + expect(error).not.toMatchObject({ + context: { formatPrefix: expect.anything() }, }); - expect(error.context.formatPrefix).toBeUndefined(); }); }); diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index 7e0675f9ce..02fda38763 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -1,11 +1,11 @@ import { RuntimeDecryptionError, WorkflowRuntimeError } from '@workflow/errors'; /** - * Browser-compatible AES-256-GCM encryption module. + * Portable AES-256-GCM encryption with a native Node decrypt path. * - * Uses the Web Crypto API (`globalThis.crypto.subtle`) which works in - * both modern browsers and Node.js 20+. This module is intentionally - * free of Node.js-specific imports so it can be bundled for the browser. + * Key import, encryption, and the browser fallback use Web Crypto. On Node, + * importKey also retains a native key handle so decrypt can call OpenSSL + * synchronously without making the module unsafe to bundle for browsers. * * The World interface (`getEncryptionKeyForRun`) returns a raw 32-byte * AES-256 key. Callers should use `importKey()` once to convert it to a @@ -13,9 +13,8 @@ import { RuntimeDecryptionError, WorkflowRuntimeError } from '@workflow/errors'; * operations within the same run. This avoids repeated `importKey()` * calls on every encrypt/decrypt invocation. * - * Wire format: `[nonce (12 bytes)][ciphertext + auth tag]` - * The `encr` format prefix is NOT part of this module — it's added/stripped - * by the serialization layer in `maybeEncrypt`/`maybeDecrypt`. + * Wire format: `[nonce (12 bytes)][ciphertext + auth tag]`. The serialization + * encryption module owns the outer `encr` format prefix. */ // CryptoKey is a global type in browsers and Node.js 20+, but TypeScript's @@ -23,6 +22,29 @@ import { RuntimeDecryptionError, WorkflowRuntimeError } from '@workflow/errors'; // so consumers can reference it without adding `dom` lib. export type CryptoKey = import('node:crypto').webcrypto.CryptoKey; +/** + * Node key handles retained alongside keys imported by this module. + * + * Node's synchronous cipher API needs a native key handle. Creating it while + * the raw bytes are already available avoids trying to convert a deliberately + * non-extractable `CryptoKey` later. Passing the `CryptoKey` directly to + * `createDecipheriv`, or converting it later with `KeyObject.from`, is + * deprecated by Node for non-extractable keys. Browser/edge callers never + * populate or consult this map. + */ +const nodeKeys = new WeakMap(); + +/** Resolve node:crypto without a static import, preserving browser bundles. */ +const nodeCrypto = (() => { + try { + return typeof process === 'undefined' + ? undefined + : process.getBuiltinModule('node:crypto'); + } catch { + return undefined; + } +})(); + /** AES-GCM nonce length in bytes. */ export const NONCE_LENGTH = 12; /** AES-GCM authentication tag length in bits. */ @@ -55,7 +77,7 @@ export async function importKey( `Encryption key must be exactly ${KEY_LENGTH} bytes, got ${raw.byteLength}` ); } - return globalThis.crypto.subtle.importKey( + const key = await globalThis.crypto.subtle.importKey( 'raw', raw, 'AES-GCM', @@ -65,6 +87,86 @@ export async function importKey( // a strict subset of `KeyUsage[]`, so this cast is sound. usages as ('encrypt' | 'decrypt')[] ); + if (nodeCrypto) { + nodeKeys.set(key, nodeCrypto.createSecretKey(raw)); + } + return key; +} + +function assertAesGcmEnvelopeLength(data: Uint8Array): void { + const minLength = NONCE_LENGTH + TAG_BYTES; + if (data.byteLength < minLength) { + throw new RuntimeDecryptionError( + `Encrypted data too short: expected at least ${minLength} bytes, got ${data.byteLength}`, + { + context: { operation: 'decrypt', byteLength: data.byteLength }, + } + ); + } +} + +function wrapDecryptionError(cause: unknown, byteLength: number): never { + throw new RuntimeDecryptionError( + `AES-256-GCM decryption failed: ${cause instanceof Error ? cause.message : String(cause)}`, + { + cause, + context: { operation: 'decrypt', byteLength }, + } + ); +} + +function decryptWithNode( + crypto: typeof import('node:crypto'), + nodeKey: import('node:crypto').KeyObject, + data: Uint8Array, + aad?: Uint8Array +): Uint8Array { + const ciphertextEnd = data.byteLength - TAG_BYTES; + const nonce = data.subarray(0, NONCE_LENGTH); + const ciphertext = data.subarray(NONCE_LENGTH, ciphertextEnd); + const authTag = data.subarray(ciphertextEnd); + try { + const decipher = crypto.createDecipheriv('aes-256-gcm', nodeKey, nonce, { + authTagLength: TAG_BYTES, + }); + if (aad) decipher.setAAD(aad); + decipher.setAuthTag(authTag); + const head = decipher.update(ciphertext); + const tail = decipher.final(); + if (tail.byteLength === 0) { + return new Uint8Array(head.buffer, head.byteOffset, head.byteLength); + } + const plaintext = new Uint8Array(head.byteLength + tail.byteLength); + plaintext.set(head, 0); + plaintext.set(tail, head.byteLength); + return plaintext; + } catch (cause) { + wrapDecryptionError(cause, data.byteLength); + } +} + +async function decryptWithWebCrypto( + key: CryptoKey, + data: Uint8Array, + aad?: Uint8Array +): Promise { + const nonce = data.subarray(0, NONCE_LENGTH); + const ciphertextAndTag = data.subarray(NONCE_LENGTH); + try { + const plaintext = await globalThis.crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: nonce, + tagLength: TAG_LENGTH, + ...(aad ? { additionalData: aad } : {}), + }, + key, + ciphertextAndTag + ); + return new Uint8Array(plaintext); + } catch (cause) { + wrapDecryptionError(cause, data.byteLength); + } } /** @@ -140,57 +242,26 @@ export async function encrypt( * @param aad - Optional additional authenticated data. Must match the bytes * passed to {@link encrypt} exactly, otherwise the GCM tag fails to verify * and a {@link RuntimeDecryptionError} is thrown. - * @returns Decrypted plaintext + * @returns Decrypted plaintext. */ export async function decrypt( key: CryptoKey, data: Uint8Array, aad?: Uint8Array ): Promise { - const minLength = NONCE_LENGTH + TAG_LENGTH / 8; // nonce + auth tag - if (data.byteLength < minLength) { + assertAesGcmEnvelopeLength(data); + if (!key.usages.includes('decrypt')) { throw new RuntimeDecryptionError( - `Encrypted data too short: expected at least ${minLength} bytes, got ${data.byteLength}`, + 'AES-256-GCM decryption failed: CryptoKey does not support decrypt', { - context: { - operation: 'decrypt', - byteLength: data.byteLength, - }, + context: { operation: 'decrypt', byteLength: data.byteLength }, } ); } - const nonce = data.subarray(0, NONCE_LENGTH); - const ciphertext = data.subarray(NONCE_LENGTH); - let plaintext: ArrayBuffer; - try { - plaintext = await globalThis.crypto.subtle.decrypt( - { - name: 'AES-GCM', - iv: nonce, - tagLength: TAG_LENGTH, - ...(aad ? { additionalData: aad } : {}), - }, - key, - ciphertext - ); - } catch (cause) { - // The most common shape we see in the wild is a DOMException with - // `name: 'OperationError'` and message "The operation failed for - // an operation-specific reason" — this is what Web Crypto throws - // when the GCM auth tag does not verify. Re-throw as - // RuntimeDecryptionError, attaching diagnostic context (byte length) - // that the bare DOMException lacks. - const causeMsg = cause instanceof Error ? cause.message : String(cause); - throw new RuntimeDecryptionError( - `AES-256-GCM decryption failed: ${causeMsg}`, - { - cause, - context: { - operation: 'decrypt', - byteLength: data.byteLength, - }, - } - ); + + const nodeKey = nodeKeys.get(key); + if (nodeCrypto && nodeKey) { + return decryptWithNode(nodeCrypto, nodeKey, data, aad); } - return new Uint8Array(plaintext); + return decryptWithWebCrypto(key, data, aad); } diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 1e13b195b3..d4fa637695 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -8,7 +8,7 @@ import type { EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import type { ReplayPayloadCache } from './replay-payload-cache.js'; import type { Serializable } from './schemas.js'; -import type { PayloadKey } from './serialization/encryption.js'; +import type { DecryptionKey } from './serialization/encryption.js'; export type StepFunction< Args extends Serializable[] = any[], @@ -132,7 +132,7 @@ export function getStepFunction(stepId: string): StepFunction | undefined { export interface WorkflowOrchestratorContext { runId: string; - encryptionKey: PayloadKey | undefined; + encryptionKey: DecryptionKey | undefined; worldCapabilities?: WorldCapabilities; globalThis: typeof globalThis; /** diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 0d3d930619..895eaaa888 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -1,12 +1,11 @@ import type { Event, WorkflowRun } from '@workflow/world'; -import { describe, expect, it, vi } from 'vitest'; +import { assert, describe, expect, it, vi } from 'vitest'; import { importKey } from './encryption.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { prepareReplayPayload } from './serialization/replay.js'; import { dehydrateStepReturnValue, deserializePreparedReplayPayload, - prepareReplayPayload, - type ReplayPayloadPreparer, } from './serialization.js'; function makeRun(input: unknown): WorkflowRun { @@ -55,26 +54,45 @@ function makeEvents(payloads: unknown[]): Event[] { } describe('ReplayPayloadCache', () => { - it('deduplicates preparation and accepts a synchronous preparer', async () => { + it('deduplicates preparation', async () => { const payload = new Uint8Array([1]); - const preparer = vi.fn((value) => ({ data: value })); + const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); const first = cache.prepareEventPayload('evnt_one', 'result', payload); const second = cache.prepareEventPayload('evnt_one', 'result', payload); expect(first).toBe(second); - await expect(first).resolves.toEqual({ data: payload }); + await expect(first).resolves.toBe(payload); expect(preparer).toHaveBeenCalledOnce(); }); + it('compacts prepared bytes before retaining them', async () => { + const backing = new Uint8Array(64 * 1024); + const prepared = backing.subarray(1024, 2048); + const cache = new ReplayPayloadCache( + undefined, + vi.fn(async () => prepared) + ); + + const retained = await cache.prepareEventPayload( + 'evnt_compact', + 'result', + new Uint8Array([1]) + ); + + assert(retained instanceof Uint8Array); + expect(retained).toEqual(prepared); + expect(retained.buffer.byteLength).toBe(retained.byteLength); + }); + it('keeps a failed prewarm until its consumer observes it, then retries', async () => { const payload = new Uint8Array([1]); const run = makeRun(payload); const preparer = vi - .fn() + .fn() .mockRejectedValueOnce(new Error('decrypt failed')) - .mockReturnValueOnce({ data: payload }); + .mockResolvedValueOnce(payload); const cache = new ReplayPayloadCache(undefined, preparer); await cache.prewarm(run, []); @@ -83,19 +101,17 @@ describe('ReplayPayloadCache', () => { ); expect(preparer).toHaveBeenCalledOnce(); - await expect(cache.prepareWorkflowInput(run)).resolves.toEqual({ - data: payload, - }); + await expect(cache.prepareWorkflowInput(run)).resolves.toBe(payload); expect(preparer).toHaveBeenCalledTimes(2); }); it('prewarms workflow, step, error, and hook payloads concurrently', async () => { const payloads = [0, 1, 2, 3].map((value) => new Uint8Array([value])); const resolvers: Array<() => void> = []; - const preparer = vi.fn( + const preparer = vi.fn( (value) => new Promise((resolve) => { - resolvers.push(() => resolve({ data: value })); + resolvers.push(() => resolve(value)); }) ); const cache = new ReplayPayloadCache(undefined, preparer); @@ -126,7 +142,7 @@ describe('ReplayPayloadCache', () => { false, true ); - const preparer = vi.fn(prepareReplayPayload); + const preparer = vi.fn(prepareReplayPayload); const cache = new ReplayPayloadCache(key, preparer); const prepared = await cache.prepareEventPayload( @@ -158,7 +174,7 @@ describe('ReplayPayloadCache', () => { // shift every later position. Resuming from that length skips exactly the // events the reload was for, which is what `resetScan` exists to prevent. const payloads = [0, 1, 2].map((value) => new Uint8Array([value])); - const preparer = vi.fn((value) => ({ data: value })); + const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); const run = makeRun(undefined); const [first, missing, second] = makeEvents(payloads); @@ -181,17 +197,17 @@ describe('ReplayPayloadCache', () => { it('bypasses legacy values and ignores missing event data during prewarm', async () => { const legacy = [0, { value: 1 }]; - const preparer = vi.fn((value) => ({ data: value })); + const preparer = vi.fn(async (value) => value); const cache = new ReplayPayloadCache(undefined, preparer); await cache.prepareEventPayload('evnt_legacy', 'result', legacy); await cache.prepareEventPayload('evnt_legacy', 'result', legacy); - expect(preparer).toHaveBeenCalledTimes(2); + expect(preparer).not.toHaveBeenCalled(); const events = makeEvents([legacy, legacy, legacy]); events[2] = { ...events[2], eventData: undefined } as unknown as Event; await cache.prewarm(makeRun(legacy), events); - expect(preparer).toHaveBeenCalledTimes(2); + expect(preparer).not.toHaveBeenCalled(); }); it('memoizes primitive step results, including undefined', async () => { diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 31144a6add..2299149e17 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -1,14 +1,20 @@ import type { Event, WorkflowRun } from '@workflow/world'; -import type { PayloadKey } from './serialization/encryption.js'; +import type { DecryptionKey } from './serialization/encryption.js'; import { type PreparedReplayPayload, prepareReplayPayload, - type ReplayPayloadPreparer, -} from './serialization.js'; +} from './serialization/replay.js'; const MAX_MEMOIZED_PRIMITIVE_LENGTH = 4096; type ReplayPayloadField = 'result' | 'error' | 'payload'; +/** Copy a view only when retaining it would also retain unrelated bytes. */ +function compactOwnedBytes(data: Uint8Array): Uint8Array { + return data.byteOffset === 0 && data.byteLength === data.buffer.byteLength + ? data + : data.slice(); +} + function isMemoizablePrimitive(value: unknown): boolean { if (value === null) return true; const type = typeof value; @@ -43,8 +49,8 @@ export class ReplayPayloadCache { private nextUnscannedEventIndex = 0; constructor( - private readonly encryptionKey: PayloadKey | undefined, - private readonly preparer: ReplayPayloadPreparer = prepareReplayPayload + private readonly encryptionKey: DecryptionKey | undefined, + private readonly preparer: typeof prepareReplayPayload = prepareReplayPayload ) {} /** @@ -173,7 +179,9 @@ export class ReplayPayloadCache { cacheKey: string, value: unknown ): Promise { - if (!(value instanceof Uint8Array)) return this.runPreparation(value); + if (!(value instanceof Uint8Array)) { + return Promise.resolve({ legacy: value }); + } const preparation = this.ensurePreparation(cacheKey, value); void preparation.catch(() => { @@ -197,9 +205,11 @@ export class ReplayPayloadCache { return preparation; } - /** Normalize synchronous and asynchronous preparers to one promise contract. */ - private async runPreparation(value: unknown): Promise { - return this.preparer(value, this.encryptionKey); + /** Compact prepared bytes before retaining them for the invocation. */ + private async runPreparation( + value: Uint8Array + ): Promise { + return compactOwnedBytes(await this.preparer(value, this.encryptionKey)); } private workflowInputKey(runId: string): string { diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 21852fac4b..0ce474ef07 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -941,7 +941,7 @@ describe('memoizeEncryptionKey', () => { sealed ) as Uint8Array; - await expect(decrypt(prefixed, resolved)).resolves.toEqual( + expect(await decrypt(prefixed, resolved)).toEqual( new TextEncoder().encode('"hi"') ); }); @@ -955,7 +955,7 @@ describe('memoizeEncryptionKey', () => { const encrypted = await encrypt(new TextEncoder().encode('"hi"'), resolved); expect(peekFormatPrefix(encrypted)).toBe(SerializationFormat.ENCRYPTED); - await expect(decrypt(encrypted, resolved)).resolves.toEqual( + expect(await decrypt(encrypted, resolved)).toEqual( new TextEncoder().encode('"hi"') ); }); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 03c08f88c3..f24944478f 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -27,8 +27,8 @@ import { monotonicFactory } from 'ulid'; import { runtimeLogger } from '../logger.js'; import { bytesToBase64, deriveRunKeyPair } from '../sealed-box.js'; import { + type DecryptionKey, deriveRunPayloadKeys, - type PayloadKey, } from '../serialization/encryption.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { getSpanKind, trace } from '../telemetry.js'; @@ -1247,8 +1247,8 @@ export function getQueueOverhead(message: { requestedAt?: Date }) { export function memoizeEncryptionKey( world: World, runOrId: WorkflowRun | string -): () => Promise { - let cached: Promise | undefined; +): () => Promise { + let cached: Promise | undefined; return () => { if (!cached) { cached = (async () => { diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index 238d34752d..3ec4a5bad5 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -37,11 +37,7 @@ import { encrypt as encryptSerializedData, type RunPayloadKeys, } from '../serialization/encryption.js'; -import { - dehydrateRunError, - hydrateRunError, - maybeEncrypt, -} from '../serialization.js'; +import { dehydrateRunError, hydrateRunError } from '../serialization.js'; import { remapErrorStack, stripInlineSourceMap } from '../source-map.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { serializeTraceCarrier } from '../telemetry.js'; @@ -320,6 +316,12 @@ async function dispatchPendingOps(params: { // EntityConflictError, which we swallow below. This drops one // network round-trip per pending hook. try { + if ( + hook.metadata !== undefined && + !(hook.metadata instanceof Uint8Array) + ) { + throw new TypeError('QuickJS hook metadata must be binary'); + } const encryptedMetadata = typeof hook.metadata === 'undefined' ? undefined @@ -1903,10 +1905,10 @@ export async function runWorkflowWithQuickJS(params: { message: (rehydrateErr as Error)?.message, } ); - dehydratedError = (await maybeEncrypt( + dehydratedError = await encryptSerializedData( result.failed.valueBytes, encryptionKey - )) as Uint8Array; + ); } } else { if (errorStack) { diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index cf4f02a9c2..ad70c6191d 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -46,9 +46,8 @@ import { import seedrandom from 'seedrandom'; import { monotonicFactory } from 'ulid'; import { runtimeLogger } from '../logger.js'; -import { decompress } from '../serialization/compression.js'; import type { DecryptionKey } from '../serialization/encryption.js'; -import { decrypt } from '../serialization/encryption.js'; +import { decodePayload } from '../serialization/payload.js'; import { getReplayTimeoutMs, isQuickJSBaselineSnapshotEnabled, @@ -74,14 +73,13 @@ import { runIdCreatedAt } from './run-id-time.js'; * X25519 keypair) so sealed `encp` hook payloads from cross-deployment * resumeHook() calls open here too, not just symmetric `encr` ones. * Both stages are format-prefix dispatched, so plaintext/uncompressed - * data passes through unchanged. Mirrors `prepareReplayPayload` in - * serialization.ts (the node:vm engine's equivalent host-side stage). + * data passes through unchanged. */ -async function prepareBytesForVM( +function prepareBytesForVM( data: Uint8Array, key?: DecryptionKey ): Promise { - return (await decompress(await decrypt(data, key))) as Uint8Array; + return decodePayload(data, key); } // ---- Types ---- diff --git a/packages/core/src/runtime/run.ts b/packages/core/src/runtime/run.ts index 1663df9b8d..201eb5a4f8 100644 --- a/packages/core/src/runtime/run.ts +++ b/packages/core/src/runtime/run.ts @@ -12,8 +12,8 @@ import { type World, } from '@workflow/world'; import { + type DecryptionKey, deriveRunPayloadKeys, - type PayloadKey, } from '../serialization/encryption.js'; import { getExternalRevivers, @@ -118,7 +118,7 @@ export class Run { * reused for returnValue, getReadable(), etc. * @internal */ - #encryptionKeyPromise: Promise | null = null; + #encryptionKeyPromise: Promise | null = null; /** * When true, run_created failed and the run may not exist yet (the @@ -140,7 +140,7 @@ export class Run { * to be resolved once. * @internal */ - #getEncryptionKey(): Promise { + #getEncryptionKey(): Promise { if (!this.#encryptionKeyPromise) { this.#encryptionKeyPromise = (async () => { const world = await this.#lazyWorldPromise; @@ -158,7 +158,7 @@ export class Run { * unobserved run lookup. * @internal */ - #getEncryptionKeyLazily(): () => Promise { + #getEncryptionKeyLazily(): () => Promise { return () => this.#getEncryptionKey(); } diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 554a7da7a3..cb9ea096ef 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -29,7 +29,7 @@ import { } from '@workflow/world'; import { runtimeLogger, stepLogger } from '../logger.js'; import { getStepFunction } from '../private.js'; -import type { PayloadKey } from '../serialization/encryption.js'; +import type { DecryptionKey } from '../serialization/encryption.js'; import { cancelAbortReaders, dehydrateStepError, @@ -105,7 +105,7 @@ export interface StepExecutorParams { rootRunId?: string; stepId: string; stepName: string; - encryptionKey?: PayloadKey; + encryptionKey?: DecryptionKey; /** * The workflow run's specVersion, used to gate payload compression. * Step outputs/errors are only compressed when the run is marked as diff --git a/packages/core/src/serialization-format.test.ts b/packages/core/src/serialization-format.test.ts index 5a19584a8f..bd52868167 100644 --- a/packages/core/src/serialization-format.test.ts +++ b/packages/core/src/serialization-format.test.ts @@ -62,7 +62,7 @@ describe('encodeWithFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded).toBeInstanceOf(Uint8Array); expect(encoded.length).toBe(4 + 3); // "devl" (4 bytes) + payload (3 bytes) @@ -71,14 +71,6 @@ describe('encodeWithFormatPrefix', () => { expect(prefix).toBe('devl'); expect(Array.from(encoded.subarray(4))).toEqual([1, 2, 3]); }); - - it('should pass through non-Uint8Array values', () => { - const result = encodeWithFormatPrefix( - SerializationFormat.DEVALUE_V1, - 'hello' - ); - expect(result).toBe('hello'); - }); }); describe('decodeFormatPrefix', () => { @@ -87,7 +79,7 @@ describe('decodeFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); const { format, payload: decoded } = decodeFormatPrefix(encoded); expect(format).toBe('devl'); diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index 831caaae6d..9799d1ee64 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -3,11 +3,32 @@ * * This module contains the format prefix handling, generic hydrate/dehydrate * dispatch, and shared types/classes used by all environments (runtime, web - * o11y, CLI o11y). It has NO Node.js dependencies. + * o11y, CLI o11y). Node acceleration is discovered at runtime without static + * Node imports, so this module remains safe to bundle for browsers. */ import { getEventDataRefFields } from '@workflow/world'; import { parse, unflatten } from 'devalue'; +import { + decompress, + decompressSync, + type ZstdDecoder, +} from './serialization/compression.js'; +import { + type DecryptionKey, + decrypt, + isRunPayloadKeys, +} from './serialization/encryption.js'; +import { + decodeFormatPrefix as decodePrefix, + encodeWithFormatPrefix, + isEncrypted, + peekFormatPrefix, +} from './serialization/format.js'; +import { + SerializationFormat, + type SerializationFormatType, +} from './serialization/types.js'; // --------------------------------------------------------------------------- // Key material (browser-safe re-exports) @@ -19,15 +40,12 @@ import { parse, unflatten } from 'devalue'; * `@workflow/core/serialization`, whose module graph reaches Node built-ins * (`node:util`, `node:async_hooks`) and cannot be bundled for the browser. * - * Everything below is Web Crypto only: `serialization/encryption.ts`, - * `encryption.ts` and `sealed-box.ts` are all free of Node dependencies. + * The encryption and compression modules use portable web APIs in browsers and + * conditionally discover native Node codecs at runtime. */ export { - type DecryptionKey, - decrypt as decryptEnvelope, deriveRunPayloadKeys, encrypt as encryptEnvelope, - isRunPayloadKeys, isSealTarget, type PayloadKey, type RunPayloadKeys, @@ -35,98 +53,42 @@ export { type SealTarget, sealTo, } from './serialization/encryption.js'; +export { type DecryptionKey, decrypt as decryptEnvelope, isRunPayloadKeys }; // --------------------------------------------------------------------------- // Format prefix constants and encoding/decoding // --------------------------------------------------------------------------- -export const SerializationFormat = { - /** devalue stringify/parse with TextEncoder/TextDecoder */ - DEVALUE_V1: 'devl', - /** Encrypted payload (inner payload has its own format prefix after decryption) */ - ENCRYPTED: 'encr', - /** - * Sealed payload — asymmetrically encrypted to a run's X25519 public key - * (inner payload has its own format prefix after opening). - * - * Written by cross-run writers that hold only the recipient run's public - * key. Opening it requires the run's private scalar rather than the - * symmetric per-run key, so o11y display treats it as ciphertext via - * {@link isEncryptedData} but {@link hydrateDataWithKey} does not attempt - * an AES-GCM decrypt on it. - */ - SEALED: 'encp', - /** Gzip-compressed payload (inner payload has its own format prefix after decompression) */ - GZIP: 'gzip', - /** Zstandard-compressed payload (inner payload has its own format prefix after decompression) */ - ZSTD: 'zstd', -} as const; - -export type SerializationFormatType = - (typeof SerializationFormat)[keyof typeof SerializationFormat]; - -/** Length of the format prefix in bytes */ -const FORMAT_PREFIX_LENGTH = 4; - -const formatEncoder = new TextEncoder(); -const formatDecoder = new TextDecoder(); +export { encodeWithFormatPrefix, SerializationFormat }; -/** - * Encode a payload with a format prefix. - */ -export function encodeWithFormatPrefix( - format: SerializationFormatType, - payload: Uint8Array | unknown -): Uint8Array | unknown { - if (!(payload instanceof Uint8Array)) { - return payload; - } - - const prefixBytes = formatEncoder.encode(format); - if (prefixBytes.length !== FORMAT_PREFIX_LENGTH) { - throw new Error( - `Format identifier must be exactly ${FORMAT_PREFIX_LENGTH} ASCII characters, got "${format}" (${prefixBytes.length} bytes)` - ); - } +export type { SerializationFormatType }; - const result = new Uint8Array(FORMAT_PREFIX_LENGTH + payload.length); - result.set(prefixBytes, 0); - result.set(payload, FORMAT_PREFIX_LENGTH); - return result; +export interface HydrateDataOptions { + /** + * Runtime-specific zstd decoder, such as the browser observability WASM + * adapter. + */ + zstdDecoder?: ZstdDecoder; } /** * Decode a format-prefixed payload. */ -export function decodeFormatPrefix(data: Uint8Array | unknown): { +export function decodeFormatPrefix(data: Uint8Array): { format: SerializationFormatType; payload: Uint8Array; } { - if (!(data instanceof Uint8Array)) { - return { - format: SerializationFormat.DEVALUE_V1, - payload: new TextEncoder().encode(JSON.stringify(data)), - }; - } - - if (data.length < FORMAT_PREFIX_LENGTH) { - throw new Error( - `Data too short to contain format prefix: expected at least ${FORMAT_PREFIX_LENGTH} bytes, got ${data.length}` - ); - } - - const prefixBytes = data.subarray(0, FORMAT_PREFIX_LENGTH); - const format = formatDecoder.decode(prefixBytes); - - const knownFormats = Object.values(SerializationFormat) as string[]; - if (!knownFormats.includes(format)) { + const decoded = decodePrefix(data); + const knownFormats = Object.values(SerializationFormat); + if (!knownFormats.includes(decoded.format as SerializationFormatType)) { throw new Error( - `Unknown serialization format: "${format}". Known formats: ${knownFormats.join(', ')}` + `Unknown serialization format: "${decoded.format}". Known formats: ${knownFormats.join(', ')}` ); } - - const payload = data.subarray(FORMAT_PREFIX_LENGTH); - return { format: format as SerializationFormatType, payload }; + return decoded as { + format: SerializationFormatType; + payload: Uint8Array; + }; } // --------------------------------------------------------------------------- @@ -190,14 +152,7 @@ export function isExpiredStub(data: unknown): boolean { * Browser-safe — does not depend on the full serialization module. */ export function isEncryptedData(data: unknown): boolean { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { - return false; - } - const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); - return ( - prefix === SerializationFormat.ENCRYPTED || - prefix === SerializationFormat.SEALED - ); + return isEncrypted(data); } /** @@ -207,11 +162,10 @@ export function isEncryptedData(data: unknown): boolean { * Browser-safe — does not depend on the full serialization module. */ export function isSealedData(data: unknown): boolean { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { - return false; - } - const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); - return prefix === SerializationFormat.SEALED; + return ( + data instanceof Uint8Array && + peekFormatPrefix(data) === SerializationFormat.SEALED + ); } /** @@ -219,125 +173,13 @@ export function isSealedData(data: unknown): boolean { * Browser-safe — does not depend on the full serialization module. */ export function isCompressedData(data: unknown): boolean { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { - return false; - } - const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); + if (!(data instanceof Uint8Array)) return false; + const prefix = peekFormatPrefix(data); return ( prefix === SerializationFormat.GZIP || prefix === SerializationFormat.ZSTD ); } -interface NodeZlibDecode { - gunzipSync?: (data: Uint8Array) => Uint8Array; - zstdDecompressSync?: (data: Uint8Array) => Uint8Array; -} - -/** - * Resolve `node:zlib` via `process.getBuiltinModule` — no static Node - * dependency, invisible to browser bundlers. Returns undefined off Node. - */ -function getNodeZlib(): NodeZlibDecode | undefined { - try { - return ( - globalThis as { - process?: { getBuiltinModule?: (id: string) => NodeZlibDecode }; - } - ).process?.getBuiltinModule?.('node:zlib'); - } catch { - return undefined; - } -} - -/** - * Synchronously decompress a `gzip`/`zstd` payload when running on Node.js. - * - * Returns `undefined` when sync decompression isn't available (e.g. in the - * browser, or zstd on Node < 22.15) — callers fall back to leaving the data - * un-hydrated (the async `hydrateDataWithKey` path handles decompression in - * browsers via `DecompressionStream` / a registered zstd decoder). - */ -function decompressSyncIfAvailable( - format: string, - payload: Uint8Array -): Uint8Array | undefined { - try { - const zlib = getNodeZlib(); - if (format === SerializationFormat.GZIP && zlib?.gunzipSync) { - return new Uint8Array(zlib.gunzipSync(payload)); - } - if (format === SerializationFormat.ZSTD && zlib?.zstdDecompressSync) { - return new Uint8Array(zlib.zstdDecompressSync(payload)); - } - } catch { - // Fall through — treat as unavailable - } - return undefined; -} - -/** - * Browser zstd decoder, registered by the o11y host (web-shared) since the - * Web `DecompressionStream` has no zstd support. Node decodes via `node:zlib` - * and never needs this. See `registerZstdDecoder`. - */ -let zstdBrowserDecoder: - | ((payload: Uint8Array) => Promise) - | undefined; - -/** - * Register a browser zstd decoder (e.g. a WASM-backed one). The web o11y UI - * calls this at init so `hydrateDataWithKey` can inflate zstd payloads after - * client-side decryption. Node readers use `node:zlib` and ignore this. - */ -export function registerZstdDecoder( - decoder: (payload: Uint8Array) => Promise -): void { - zstdBrowserDecoder = decoder; -} - -/** - * Asynchronously decompress a `gzip`/`zstd` payload. - * - gzip: web-standard `DecompressionStream` (Node 18+, browsers, edge). - * - zstd: `node:zlib` when on Node, else the registered browser decoder. - */ -async function decompressAsync( - format: string, - payload: Uint8Array -): Promise { - if (format === SerializationFormat.ZSTD) { - const sync = decompressSyncIfAvailable(format, payload); - if (sync) return sync; - if (zstdBrowserDecoder) return zstdBrowserDecoder(payload); - throw new Error( - 'zstd-compressed workflow data encountered but no zstd decoder is ' + - 'available. Node.js 22.15+ decodes natively; in the browser register ' + - 'one via registerZstdDecoder (the web o11y package does this).' - ); - } - - const transform = new DecompressionStream('gzip'); - const writer = transform.writable.getWriter(); - const writePromise = writer.write(payload).then(() => writer.close()); - writePromise.catch(() => {}); - const chunks: Uint8Array[] = []; - let total = 0; - const reader = transform.readable.getReader(); - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - total += value.length; - } - await writePromise; - const out = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - out.set(chunk, offset); - offset += chunk.length; - } - return out; -} - // --------------------------------------------------------------------------- // Revivers type (shared across all environments) // --------------------------------------------------------------------------- @@ -390,7 +232,7 @@ export function hydrateData(value: unknown, revivers: Revivers): unknown { // pass the data through untouched (like encrypted data) so async // consumers can route it through `hydrateDataWithKey`, which // decompresses via DecompressionStream / a registered zstd decoder. - const inflated = decompressSyncIfAvailable(format, payload); + const inflated = decompressSync(value); if (inflated === undefined) { return value; } @@ -426,16 +268,14 @@ export function hydrateData(value: unknown, revivers: Revivers): unknown { export async function hydrateDataWithKey( value: unknown, revivers: Revivers, - key: import('./serialization/encryption.js').DecryptionKey | undefined + key: DecryptionKey | undefined, + options?: HydrateDataOptions ): Promise { let data = value; if (data instanceof Uint8Array && isEncryptedData(data) && key) { // Envelope-aware decrypt: handles both `encr` (AES-GCM under the run's // symmetric key) and `encp` (sealed to the run's X25519 public key by // some other run), dispatching on the format prefix. - const { decrypt, isRunPayloadKeys } = await import( - './serialization/encryption.js' - ); // Opening a sealed payload needs the run's private scalar. If the caller // supplied only a symmetric key, leave the bytes as ciphertext so the UI // keeps showing its "Encrypted" affordance, rather than surfacing a @@ -447,10 +287,9 @@ export async function hydrateDataWithKey( if (data instanceof Uint8Array && isCompressedData(data)) { // Decompress: strip the codec prefix and inflate. gzip uses the // web-standard DecompressionStream (works in browsers); zstd uses - // node:zlib on Node or the registered WASM decoder in the browser. + // node:zlib on Node or an explicitly supplied WASM decoder in the browser. // The inflated bytes carry their own format prefix (e.g. 'devl'). - const { format, payload } = decodeFormatPrefix(data); - data = await decompressAsync(format, payload); + data = await decompress(data, undefined, options); } // Delegate the (decrypted/decompressed) result to sync hydrateData return hydrateData(data, revivers); diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index 6a9cbdfc23..d354a2d635 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -14,6 +14,7 @@ import { getStepFunction, registerStepFunction } from './private.js'; import { bytesToBase64, deriveRunKeyPair } from './sealed-box.js'; import { decrypt as decryptEnvelope, + encrypt as encryptEnvelope, runPayloadKeys, sealTo, } from './serialization/encryption.js'; @@ -38,8 +39,6 @@ import { hydrateWorkflowArguments, hydrateWorkflowReturnValue, isEncrypted, - maybeDecrypt, - maybeEncrypt, SerializationFormat, } from './serialization.js'; import { hydrateData } from './serialization-format.js'; @@ -4892,42 +4891,7 @@ describe('format prefix system', () => { }); }); -describe('decodeFormatPrefix legacy compatibility', () => { - it('should handle legacy object data (non-Uint8Array)', () => { - const legacyData = { message: 'hello', count: 42 }; - const result = decodeFormatPrefix(legacyData); - - expect(result.format).toBe(SerializationFormat.DEVALUE_V1); - expect(result.payload).toBeInstanceOf(Uint8Array); - - // The payload should be JSON-encoded - const decoded = new TextDecoder().decode(result.payload); - expect(JSON.parse(decoded)).toEqual(legacyData); - }); - - it('should handle legacy array data (non-Uint8Array)', () => { - const legacyData = [1, 2, 'three', { nested: true }]; - const result = decodeFormatPrefix(legacyData); - - expect(result.format).toBe(SerializationFormat.DEVALUE_V1); - expect(result.payload).toBeInstanceOf(Uint8Array); - - const decoded = new TextDecoder().decode(result.payload); - expect(JSON.parse(decoded)).toEqual(legacyData); - }); - - it('should handle legacy undefined data (non-Uint8Array)', () => { - const legacyData = undefined; - const result = decodeFormatPrefix(legacyData); - - expect(result.format).toBe(SerializationFormat.DEVALUE_V1); - expect(result.payload).toBeInstanceOf(Uint8Array); - - // JSON.stringify(undefined) returns undefined (not a string), - // which when encoded produces an empty Uint8Array - expect(result.payload.length).toBe(0); - }); - +describe('decodeFormatPrefix', () => { it('should still correctly handle v2 Uint8Array data', () => { // Create valid v2 data with 'devl' prefix const payload = new TextEncoder().encode('["test"]'); @@ -6102,7 +6066,7 @@ describe('encrypt/decrypt primitives', () => { }); }); -describe('maybeEncrypt / maybeDecrypt', () => { +describe('serialized payload encryption', () => { const testKeyRaw = new Uint8Array([ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, @@ -6115,31 +6079,25 @@ describe('maybeEncrypt / maybeDecrypt', () => { it('should pass through data unchanged when key is undefined', async () => { const data = new Uint8Array([1, 2, 3, 4]); - const result = await maybeEncrypt(data, undefined); + const result = await encryptEnvelope(data, undefined); expect(result).toBe(data); // Same reference }); it('should encrypt and add "encr" prefix when key is provided', async () => { const data = new Uint8Array([1, 2, 3, 4]); - const result = await maybeEncrypt(data, testKey); + const result = await encryptEnvelope(data, testKey); expect(result).not.toBe(data); expect(isEncrypted(result)).toBe(true); }); - it('should round-trip through maybeEncrypt/maybeDecrypt', async () => { + it('should round-trip through encrypt/decrypt', async () => { const data = new Uint8Array([10, 20, 30, 40, 50]); - const encrypted = await maybeEncrypt(data, testKey); - const decrypted = await maybeDecrypt(encrypted, testKey); + const encrypted = await encryptEnvelope(data, testKey); + const decrypted = await decryptEnvelope(encrypted, testKey); expect(decrypted).toEqual(data); }); - it('should pass through non-Uint8Array values in maybeDecrypt', async () => { - const legacyData = [1, 'hello', { key: 'value' }]; - const result = await maybeDecrypt(legacyData, testKey); - expect(result).toBe(legacyData); // Same reference - }); - - it('should pass through unencrypted Uint8Array in maybeDecrypt', async () => { + it('should pass through an unencrypted Uint8Array', async () => { // Data with 'devl' prefix (not encrypted) const prefix = new TextEncoder().encode('devl'); const payload = new TextEncoder().encode('test'); @@ -6147,14 +6105,14 @@ describe('maybeEncrypt / maybeDecrypt', () => { data.set(prefix, 0); data.set(payload, prefix.length); - const result = await maybeDecrypt(data, testKey); + const result = await decryptEnvelope(data, testKey); expect(result).toBe(data); // Same reference — not encrypted, passed through }); it('should throw when encrypted data has no key', async () => { const data = new Uint8Array([1, 2, 3]); - const encrypted = await maybeEncrypt(data, testKey); - await expect(maybeDecrypt(encrypted, undefined)).rejects.toThrow( + const encrypted = await encryptEnvelope(data, testKey); + await expect(decryptEnvelope(encrypted, undefined)).rejects.toThrow( 'Encrypted data encountered but no encryption key' ); }); diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index e626cd3f91..c38fb98d8f 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -34,13 +34,10 @@ import { decodeRunPublicKey, } from './sealed-box.js'; import * as clientModule from './serialization/client.js'; -import { - type CompressionStats, - compress, - decompress, -} from './serialization/compression.js'; +import type { CompressionStats } from './serialization/compression.js'; import { aesKeyOf, + type DecryptionKey, decrypt, deriveRunPayloadKeys, type EncryptionKeyParam, @@ -69,6 +66,7 @@ import { isInstanceOfPrototype, readProperty, } from './serialization/hardened.js'; +import { decodePayload, encodePayload } from './serialization/payload.js'; import { getClassReducers, getClassRevivers, @@ -82,11 +80,16 @@ import { getStepFunctionReducer, getStepFunctionReviver, } from './serialization/reducers/step-function.js'; +import { + type PreparedReplayPayload, + prepareReplayPayload, +} from './serialization/replay.js'; import * as stepModule from './serialization/step.js'; import { type FormatPrefix, isFormatPrefix, SerializationFormat, + type SerializationFormatType, } from './serialization/types.js'; import * as workflowModule from './serialization/workflow.js'; import { contextStorage } from './step/context-storage.js'; @@ -115,6 +118,7 @@ import { WorkflowAbortSignal } from './workflow/abort-controller.js'; // so existing consumers of `@workflow/core/serialization` keep working. export { SerializationFormat, + type SerializationFormatType, type FormatPrefix, isFormatPrefix, encodeWithFormatPrefix, @@ -123,10 +127,9 @@ export { isEncrypted, encrypt, decrypt, - compress, - decompress, type EncryptionKeyParam, // Sealed-box ('encp') key variants — see serialization/encryption.ts. + type DecryptionKey, type PayloadKey, type RunPayloadKeys, type SealTarget, @@ -138,10 +141,7 @@ export { aesKeyOf, }; -// Re-export the legacy SerializationFormatType for backwards compatibility. -// New code should use FormatPrefix from './serialization/types.js'. -export type SerializationFormatType = - (typeof SerializationFormat)[keyof typeof SerializationFormat]; +export { compress, decompress } from './serialization/compression.js'; /** * Default ULID generator for contexts where VM's seeded `stableUlid` isn't available. @@ -302,7 +302,7 @@ export function getSerializeStream( let prefixed = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); // Encrypt the frame payload if a key is provided. // The length header remains in the clear so the deserializer can @@ -322,9 +322,9 @@ export function getSerializeStream( prefixed = encodeWithFormatPrefix( SerializationFormat.SEALED, await sealSession.seal(prefixed) - ) as Uint8Array; + ); } else if (keyState.key) { - prefixed = (await encrypt(prefixed, keyState.key)) as Uint8Array; + prefixed = await encrypt(prefixed, keyState.key); } // Write length-prefixed frame: [4-byte length][prefixed data] @@ -419,10 +419,8 @@ export function getDeserializeStream( // A sealed frame needs the run's keypair; a symmetric frame needs an // AES key. Report the shortfall precisely — "no key at all" and "the // wrong kind of key" have very different causes. - const usable = sealed - ? isRunPayloadKeys(keyState.key) - : aesKeyOf(keyState.key) !== undefined; - if (!usable) { + const aesKey = aesKeyOf(keyState.key); + if (sealed ? !isRunPayloadKeys(keyState.key) : !aesKey) { controller.error( new RuntimeDecryptionError( sealed @@ -449,7 +447,7 @@ export function getDeserializeStream( // with the one-shot path. decrypted = sealed ? await openSession!.open(decodeFormatPrefix(frameData).payload) - : ((await decrypt(frameData, keyState.key)) as Uint8Array); + : await decrypt(frameData, aesKey); } catch (error) { // The low-level crypto layer only sees the stripped payload, so it // cannot record the outer envelope prefix. We peeked it here, so @@ -3378,79 +3376,17 @@ function getStepRevivers( }; } -// ============================================================================ -// Encryption Helpers -// ============================================================================ -// These delegate to the modular `encrypt`/`decrypt` from `./serialization/encryption.js` -// but are kept as named exports for backwards compatibility with existing consumers. - -/** - * Encrypt data if the world supports encryption. - * Returns original data if encryption is not available. - * - * @deprecated Use `encrypt` from `./serialization/encryption.js` instead. - */ -export async function maybeEncrypt( - data: Uint8Array, - key: PayloadKey | undefined -): Promise { - return (await encrypt(data, key)) as Uint8Array; -} - -/** - * Decrypt data if it has the 'encr' prefix. - * - * @deprecated Use `decrypt` from `./serialization/encryption.js` instead. - */ -export async function maybeDecrypt( - data: Uint8Array | unknown, - key: PayloadKey | undefined -): Promise { - return decrypt(data, key); -} - -/** - * Replay hydration has two stages: - * - * 1. Host-side preparation decrypts and decompresses persisted data. That work - * is independent of a workflow VM and can be cached across replay VMs. - * 2. Deserialization revives the prepared representation against the current - * VM's globals. It must run again for every VM to produce fresh object graphs - * and correctly scoped Workflow objects. - * - * `data` is the boundary between those stages. For current-format payloads it - * is still format-prefixed serialized bytes, not a live JavaScript value. - */ -export interface PreparedReplayPayload { - readonly data: unknown; -} - -/** - * Swappable implementation of the host-side preparation stage. Supporting - * both direct and promised results lets a future synchronous Node decryptor use - * the same cache contract as today's asynchronous Web Crypto implementation. - */ -export type ReplayPayloadPreparer = ( +async function prepareReplayPayloadWithTelemetry( value: unknown, - key: PayloadKey | undefined -) => PreparedReplayPayload | Promise; + key: DecryptionKey | undefined +): Promise { + if (!(value instanceof Uint8Array)) return { legacy: value }; -/** - * Decrypt and decompress persisted data without parsing it into JavaScript. - * Legacy non-binary values pass through unchanged for their consumer to revive. - */ -export const prepareReplayPayload: ReplayPayloadPreparer = async ( - value, - key -) => { const compressionStats: CompressionStats = {}; - const prepared = await decompress( - await decrypt(value, key), - compressionStats - ); + const prepared = await prepareReplayPayload(value, key, compressionStats); await recordCompression(compressionStats, 'deserialize'); - return { data: prepared }; -}; + return prepared; +} /** * Parse a prepared workflow argument or successful step/hook payload using the @@ -3462,7 +3398,8 @@ export function deserializePreparedReplayPayload( global: Record = globalThis, extraRevivers: Record any> = {} ): any { - return workflowModule.deserialize(prepared.data, { + const data = prepared instanceof Uint8Array ? prepared : prepared.legacy; + return workflowModule.deserialize(data, { global, extraRevivers: { ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), @@ -3480,7 +3417,7 @@ export function deserializePreparedStepError( global: Record = globalThis, extraRevivers: Record any> = {} ): unknown { - const { data } = prepared; + const data = prepared instanceof Uint8Array ? prepared : prepared.legacy; if (!(data instanceof Uint8Array)) { return unflatten(data as any[], { ...getWorkflowRevivers(global), @@ -3533,7 +3470,7 @@ export async function dehydrateWorkflowArguments( v1Compat = false, framedByteStreams = false, compression = false -): Promise { +): Promise { if (v1Compat) { const str = stringify( value, @@ -3569,18 +3506,16 @@ export async function dehydrateWorkflowArguments( * payload skips host-side decrypt/decompress but always performs VM revival. */ export async function hydrateWorkflowArguments( - value: Uint8Array | unknown, + value: unknown, _runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload ): Promise { - return deserializePreparedReplayPayload( - prepared ?? (await prepareReplayPayload(value, key)), - global, - extraRevivers - ); + const payload = + prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)); + return deserializePreparedReplayPayload(payload, global, extraRevivers); } /** @@ -3602,7 +3537,7 @@ export async function dehydrateWorkflowReturnValue( * runtime/suspension-handler.ts passes one for step-input dehydration. */ guestCodeStatsOut?: GuestCodeStats -): Promise { +): Promise { if (v1Compat) { const str = stringify(value, getWorkflowReducers(global)); return revive(str); @@ -3637,9 +3572,9 @@ export async function dehydrateWorkflowReturnValue( * of a completed workflow run. */ export async function hydrateWorkflowReturnValue( - value: Uint8Array | unknown, + value: unknown, runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, ops: Promise[] = [], global: Record = globalThis, extraRevivers: Record any> = {} @@ -3672,7 +3607,7 @@ export async function dehydrateStepArguments( compression = false, /** See `dehydrateWorkflowReturnValue`. */ guestCodeStatsOut?: GuestCodeStats -): Promise { +): Promise { if (v1Compat) { const str = stringify(value, getWorkflowReducers(global)); return revive(str); @@ -3704,9 +3639,9 @@ export async function dehydrateStepArguments( * from the database at the start of the step execution. */ export async function hydrateStepArguments( - value: Uint8Array | unknown, + value: unknown, runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, ops: Promise[] = [], global: Record = globalThis, extraRevivers: Record any> = {}, @@ -3757,7 +3692,7 @@ export async function dehydrateStepReturnValue( // the backgrounded `run_started`. Threaded into the step reducers' stream // sink. Undefined outside turbo / on the await path. runReadyBarrier?: Promise -): Promise { +): Promise { if (v1Compat) { const str = stringify( value, @@ -3831,18 +3766,14 @@ export async function dehydrateStepError( const serialized = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; - // Compress before encrypting — encrypted bytes don't compress. + ); const compressionStats: CompressionStats = {}; - const compressed = await compress( + const encrypted = await encodePayload( serialized, + key, compression, compressionStats ); - const encrypted = (await maybeEncrypt( - compressed as Uint8Array, - key - )) as Uint8Array; await recordCompression(compressionStats, 'serialize'); return encrypted; } catch (error) { @@ -3866,18 +3797,16 @@ export async function dehydrateStepError( * @returns The hydrated thrown value, ready to reject the step promise */ export async function hydrateStepError( - value: Uint8Array | unknown, + value: unknown, _runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload ): Promise { - return deserializePreparedStepError( - prepared ?? (await prepareReplayPayload(value, key)), - global, - extraRevivers - ); + const payload = + prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)); + return deserializePreparedStepError(payload, global, extraRevivers); } /** @@ -3904,18 +3833,14 @@ export async function dehydrateRunError( const serialized = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; - // Compress before encrypting — encrypted bytes don't compress. + ); const compressionStats: CompressionStats = {}; - const compressed = await compress( + const encrypted = await encodePayload( serialized, + key, compression, compressionStats ); - const encrypted = (await maybeEncrypt( - compressed as Uint8Array, - key - )) as Uint8Array; await recordCompression(compressionStats, 'serialize'); return encrypted; } catch (error) { @@ -3938,33 +3863,24 @@ export async function dehydrateRunError( * @returns The hydrated thrown value, ready to be consumed by the client */ export async function hydrateRunError( - value: Uint8Array | unknown, + value: unknown, runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, ops: Promise[] = [], global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - const compressionStats: CompressionStats = {}; - const decrypted = await decompress( - await decrypt(value, key), - compressionStats - ); - await recordCompression(compressionStats, 'deserialize'); - - if (!(decrypted instanceof Uint8Array)) { - // See the matching note in `hydrateStepError`: this branch is for - // devalue flattened arrays from legacy callers; current SDK versions - // always emit a Uint8Array, and a misshapen value here intentionally - // throws via `unflatten` so the surrounding try/catch in o11y helpers - // surfaces the issue rather than masking it. - return unflatten(decrypted as any[], { + if (!(value instanceof Uint8Array)) { + return unflatten(value as any[], { ...getExternalRevivers(global, ops, runId, key), ...extraRevivers, }); } - const { format, payload } = decodeFormatPrefix(decrypted); + const compressionStats: CompressionStats = {}; + const prepared = await decodePayload(value, key, compressionStats); + await recordCompression(compressionStats, 'deserialize'); + const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { const str = new TextDecoder().decode(payload); @@ -3991,18 +3907,16 @@ export async function hydrateRunError( * of a `step_completed` event. */ export async function hydrateStepReturnValue( - value: Uint8Array | unknown, + value: unknown, _runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload ): Promise { - return deserializePreparedReplayPayload( - prepared ?? (await prepareReplayPayload(value, key)), - global, - extraRevivers - ); + const payload = + prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)); + return deserializePreparedReplayPayload(payload, global, extraRevivers); } // ---- Helpers to extract stream/Request/Response reducers and revivers ---- diff --git a/packages/core/src/serialization/client.ts b/packages/core/src/serialization/client.ts index 5d9b0e6d78..b23586000a 100644 --- a/packages/core/src/serialization/client.ts +++ b/packages/core/src/serialization/client.ts @@ -8,14 +8,10 @@ import { SerializationError } from '@workflow/errors'; import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; -import { compress, decompress } from './compression.js'; -import { - decrypt as decryptData, - encrypt as encryptData, - type PayloadKey, -} from './encryption.js'; +import type { DecryptionKey, PayloadKey } from './encryption.js'; import { formatSerializationError, rethrowIfRuntimeError } from './errors.js'; import { decodeFormatPrefix, encodeWithFormatPrefix } from './format.js'; +import { decodePayload, encodePayload } from './payload.js'; import { SerializationFormat } from './types.js'; /** @@ -25,20 +21,19 @@ export async function serialize( value: unknown, encryptionKey?: PayloadKey, options?: CodecOptions -): Promise { +): Promise { try { const payload = devalueCodec.serialize(value, 'client', options); const prefixed = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; - // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress( + ); + return await encodePayload( prefixed, - options?.compression === true, + encryptionKey, + options?.compression ?? false, options?.compressionStats ); - return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); const { message, hint } = formatSerializationError('client value', error); @@ -50,25 +45,25 @@ export async function serialize( * Deserialize a value for the client environment (e.g. workflow return value). */ export async function deserialize( - data: Uint8Array | unknown, - encryptionKey?: PayloadKey, + data: unknown, + encryptionKey?: DecryptionKey, options?: CodecOptions ): Promise { - const decrypted = await decompress( - await decryptData(data, encryptionKey), - options?.compressionStats - ); - - if (!(decrypted instanceof Uint8Array)) { + if (!(data instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { - return devalueCodec.deserializeLegacy(decrypted, 'client', options); + return devalueCodec.deserializeLegacy(data, 'client', options); } throw new Error( 'Cannot deserialize non-binary data without legacy support' ); } - const { format, payload } = decodeFormatPrefix(decrypted); + const prepared = await decodePayload( + data, + encryptionKey, + options?.compressionStats + ); + const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { return devalueCodec.deserialize(payload, 'client', options); diff --git a/packages/core/src/serialization/compression-capabilities.test.ts b/packages/core/src/serialization/compression-capabilities.test.ts new file mode 100644 index 0000000000..45730d27b7 --- /dev/null +++ b/packages/core/src/serialization/compression-capabilities.test.ts @@ -0,0 +1,25 @@ +import { afterEach, expect, it, vi } from 'vitest'; + +vi.mock('@workflow/world/serialization-compression.js', () => ({ + decompressSerializedDataSync: vi.fn(), + getNativeCompressionCodec: vi.fn(() => undefined), +})); + +import { compress } from './compression.js'; + +afterEach(() => { + delete process.env.WORKFLOW_COMPRESSION_CODEC; + vi.unstubAllGlobals(); +}); + +it('does not write portable gzip without a matching reader', async () => { + process.env.WORKFLOW_COMPRESSION_CODEC = 'gzip'; + const CompressionStream = vi.fn(); + vi.stubGlobal('CompressionStream', CompressionStream); + vi.stubGlobal('DecompressionStream', undefined); + + const data = new Uint8Array(2048); + + await expect(compress(data, true)).resolves.toBe(data); + expect(CompressionStream).not.toHaveBeenCalled(); +}); diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts index 355183b7ef..e75826c41a 100644 --- a/packages/core/src/serialization/compression.test.ts +++ b/packages/core/src/serialization/compression.test.ts @@ -99,11 +99,9 @@ describe('compression layer (compress/decompress)', () => { expect(result).toBe(original); }); - it('decompress passes non-compressed and non-binary data through', async () => { + it('decompress passes non-compressed bytes through', async () => { const plain = textEncoder.encode('devl"hello"'); expect(await decompress(plain)).toBe(plain); - const legacy = [1, 2, 3]; - expect(await decompress(legacy)).toBe(legacy); }); }); @@ -165,12 +163,6 @@ describe('CompressionStats telemetry sink', () => { expect(stats.storedBytes).toBe(original.length); }); - it('does not record for non-binary (legacy) data', async () => { - const stats: CompressionStats = {}; - await compress({ not: 'binary' }, true, stats); - expect(stats.recorded).toBeFalsy(); - }); - it('records the inflate on the read path', async () => { const original = devlBytes(JSON.stringify(makeCompressibleValue())); const compressed = (await compress(original, true)) as Uint8Array; @@ -333,26 +325,32 @@ describe('codec selection (zstd preferred, gzip fallback)', () => { textEncoder.encode(JSON.stringify(makeCompressibleValue())) ) as Uint8Array; const stats: CompressionStats = {}; - const compressed = await compress(original, true, stats); + const compression = compress(original, true, stats); + expect(compression).toBeInstanceOf(Promise); + const compressed = await compression; expect(isCompressed(compressed)).toBe(true); expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.ZSTD); expect(stats.codec).toBe('zstd'); }); - it('WORKFLOW_COMPRESSION_CODEC=gzip forces the portable codec', async () => { + it('WORKFLOW_COMPRESSION_CODEC=gzip forces gzip', async () => { process.env.WORKFLOW_COMPRESSION_CODEC = 'gzip'; const original = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, textEncoder.encode(JSON.stringify(makeCompressibleValue())) ) as Uint8Array; const stats: CompressionStats = {}; - const compressed = await compress(original, true, stats); + const compression = compress(original, true, stats); + expect(compression).toBeInstanceOf(Promise); + const compressed = await compression; expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.GZIP); expect(stats.codec).toBe('gzip'); // Read path still inflates gzip and reports the codec. const readStats: CompressionStats = {}; - const inflated = (await decompress(compressed, readStats)) as Uint8Array; + const result = decompress(compressed, readStats); + expect(result).toBeInstanceOf(Promise); + const inflated = await result; expect(inflated).toEqual(original); expect(readStats.codec).toBe('gzip'); }); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index 2a99303be4..34da049c65 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -10,14 +10,12 @@ * compression runs at every step boundary so the write CPU is a per-step * tax. zstd requires `node:zlib` >= 22.15 (Web `CompressionStream` has no * zstd), so on a runtime without it we fall back to gzip via the portable - * `CompressionStream`. `WORKFLOW_COMPRESSION_CODEC=gzip` forces the - * portable codec. + * `CompressionStream`. `WORKFLOW_COMPRESSION_CODEC=gzip` forces gzip. * * Read side: dispatch on the format prefix, so both 'zstd' and 'gzip' * payloads are always decodable regardless of which codec wrote them. - * (The browser o11y read path decodes zstd via a registered WASM decoder — - * see `serialization-format.ts`; this module's `decompress` is the Node - * runtime/replay path and uses `node:zlib`.) + * (The browser o11y read path passes a WASM decoder explicitly — see + * `serialization-format.ts`; the Node runtime/replay path uses `node:zlib`.) * * Layering order with encryption: compression is applied BEFORE * encryption (encr(zstd(devl))) — encrypted bytes are high-entropy and @@ -32,6 +30,10 @@ * archives, etc.) from wasted CPU and size inflation. */ +import { + decompressSerializedDataSync, + getNativeCompressionCodec, +} from '@workflow/world/serialization-compression.js'; import { decodeFormatPrefix, encodeWithFormatPrefix, @@ -77,9 +79,8 @@ function isCompressionDisabledByEnv(): boolean { } /** - * Optional codec override (`WORKFLOW_COMPRESSION_CODEC=gzip|zstd`). Lets an - * operator pin the portable codec (gzip) — useful for A/B comparisons or - * runtimes where zstd read support isn't yet everywhere. + * Optional codec override (`WORKFLOW_COMPRESSION_CODEC=gzip|zstd`). Useful + * for A/B comparisons or runtimes where zstd read support isn't everywhere. */ function codecOverrideFromEnv(): 'gzip' | 'zstd' | undefined { try { @@ -90,44 +91,30 @@ function codecOverrideFromEnv(): 'gzip' | 'zstd' | undefined { } } -interface NodeZlib { - zstdCompressSync?: (data: Uint8Array, opts?: unknown) => Uint8Array; - zstdDecompressSync?: (data: Uint8Array) => Uint8Array; - constants?: Record; -} +const nativeGzip = getNativeCompressionCodec('gzip'); +const nativeZstd = getNativeCompressionCodec('zstd'); -/** - * Resolve `node:zlib` via `process.getBuiltinModule` — no static import, so - * this module stays bundler-safe for browser/edge targets (where it returns - * undefined and we fall back to gzip). - */ -function getNodeZlib(): NodeZlib | undefined { - try { - return ( - globalThis as { - process?: { getBuiltinModule?: (id: string) => NodeZlib }; - } - ).process?.getBuiltinModule?.('node:zlib'); - } catch { - return undefined; - } +/** Runtime-specific zstd decoder used when native Node support is unavailable. */ +export type ZstdDecoder = ( + payload: Uint8Array +) => Uint8Array | Promise; + +export interface DecompressionOptions { + zstdDecoder?: ZstdDecoder; } function isZstdAvailable(): boolean { - const z = getNodeZlib(); - return ( - typeof z?.zstdCompressSync === 'function' && - typeof z?.zstdDecompressSync === 'function' - ); + return nativeZstd !== undefined; } /** * gzip via the web-standard `CompressionStream` (Node 18+, browsers, edge). */ -function isGzipAvailable(): boolean { +function canCompressGzip(): boolean { return ( - typeof CompressionStream === 'function' && - typeof DecompressionStream === 'function' + nativeGzip !== undefined || + (typeof CompressionStream === 'function' && + typeof DecompressionStream === 'function') ); } @@ -151,12 +138,9 @@ async function pipeThroughTransform( writePromise.catch(() => {}); const chunks: Uint8Array[] = []; let total = 0; - const reader = transform.readable.getReader(); - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - total += value.length; + for await (const chunk of transform.readable) { + chunks.push(chunk); + total += chunk.length; } await writePromise; const out = new Uint8Array(total); @@ -168,34 +152,44 @@ async function pipeThroughTransform( return out; } -async function gzipBytes(data: Uint8Array): Promise { +function gzipBytes(data: Uint8Array): Promise { return pipeThroughTransform(data, new CompressionStream('gzip')); } -async function gunzipBytes(data: Uint8Array): Promise { +async function gzip(data: Uint8Array): Promise { + return nativeGzip ? nativeGzip.compress(data) : gzipBytes(data); +} + +function gunzipBytes(data: Uint8Array): Promise { return pipeThroughTransform(data, new DecompressionStream('gzip')); } function zstdBytes(data: Uint8Array): Uint8Array { - const z = getNodeZlib(); - const level = z?.constants?.ZSTD_c_compressionLevel; - const opts = - level !== undefined ? { params: { [level]: ZSTD_LEVEL } } : undefined; - // biome-ignore lint/style/noNonNullAssertion: guarded by isZstdAvailable() - return new Uint8Array(z!.zstdCompressSync!(data, opts)); + if (!nativeZstd) { + throw new Error('zstd compression is not available in this runtime'); + } + return nativeZstd.compress(data, ZSTD_LEVEL); } -function unzstdBytes(data: Uint8Array): Uint8Array { - const z = getNodeZlib(); - if (!z?.zstdDecompressSync) { - throw new Error( - 'Compressed (zstd) workflow data encountered but node:zlib zstd ' + - 'support is not available in this runtime (requires Node.js 22.15+). ' + - 'In the browser, register a zstd decoder via registerZstdDecoder ' + - '(serialization-format.ts).' - ); - } - return new Uint8Array(z.zstdDecompressSync(data)); +async function decompressZstd( + payload: Uint8Array, + decoder?: ZstdDecoder +): Promise { + if (nativeZstd) return nativeZstd.decompress(payload); + if (decoder) return decoder(payload); + throw new Error( + 'Compressed (zstd) workflow data encountered but no zstd decoder is ' + + 'available. Node.js 22.15+ decodes natively; in the browser ' + + 'pass one in DecompressionOptions.' + ); +} + +async function decompressGzip(payload: Uint8Array): Promise { + if (nativeGzip) return nativeGzip.decompress(payload); + if (typeof DecompressionStream === 'function') return gunzipBytes(payload); + throw new Error( + 'Compressed (gzip) workflow data encountered but no gzip decoder is available.' + ); } /** @@ -246,10 +240,10 @@ function recordStats( */ function selectWriteCodec(): 'zstd' | 'gzip' | 'none' { const override = codecOverrideFromEnv(); - if (override === 'gzip') return isGzipAvailable() ? 'gzip' : 'none'; + if (override === 'gzip') return canCompressGzip() ? 'gzip' : 'none'; // Default and explicit 'zstd' both prefer zstd, then fall back to gzip. if (isZstdAvailable()) return 'zstd'; - if (isGzipAvailable()) return 'gzip'; + if (canCompressGzip()) return 'gzip'; return 'none'; } @@ -268,12 +262,10 @@ function selectWriteCodec(): 'zstd' | 'gzip' | 'none' { * when compression is disabled, unavailable, or not worthwhile. */ export async function compress( - data: Uint8Array | unknown, + data: Uint8Array, enabled: boolean, stats?: CompressionStats -): Promise { - if (!(data instanceof Uint8Array)) return data; - // From here `data` is binary, so every return path records stats. +): Promise { if ( !enabled || data.length < COMPRESSION_MIN_BYTES || @@ -289,16 +281,18 @@ export async function compress( return data; } - const compressed = codec === 'zstd' ? zstdBytes(data) : await gzipBytes(data); - const format = - codec === 'zstd' ? SerializationFormat.ZSTD : SerializationFormat.GZIP; - const wrappedLength = 4 + compressed.length; // format prefix + payload + const compressed = codec === 'zstd' ? zstdBytes(data) : await gzip(data); + const wrappedLength = 4 + compressed.length; if (wrappedLength >= data.length * (1 - COMPRESSION_MIN_SAVINGS_RATIO)) { recordStats(stats, 'none', data.length, data.length); return data; } + recordStats(stats, codec, data.length, wrappedLength); - return encodeWithFormatPrefix(format, compressed); + return encodeWithFormatPrefix( + codec === 'zstd' ? SerializationFormat.ZSTD : SerializationFormat.GZIP, + compressed + ); } /** @@ -306,45 +300,47 @@ export async function compress( * Dispatches on the prefix ('zstd' or 'gzip') and inflates the inner * payload (which carries its own format prefix, e.g. 'devl'). * - * Non-compressed data (including non-binary legacy data) is returned - * unchanged, so this is safe to apply unconditionally on read paths. + * Non-compressed data is returned unchanged. */ export async function decompress( - data: Uint8Array | unknown, - stats?: CompressionStats -): Promise { - if (!(data instanceof Uint8Array)) return data; + data: Uint8Array, + stats?: CompressionStats, + options?: DecompressionOptions +): Promise { const prefix = peekFormatPrefix(data); - if (prefix === SerializationFormat.ZSTD) { - const { payload } = decodeFormatPrefix(data); - const inflated = unzstdBytes(payload); - recordStats(stats, 'zstd', inflated.length, data.length); - return inflated; + const codec = + prefix === SerializationFormat.ZSTD + ? 'zstd' + : prefix === SerializationFormat.GZIP + ? 'gzip' + : undefined; + if (!codec) { + recordStats(stats, 'none', data.length, data.length); + return data; } - if (prefix === SerializationFormat.GZIP) { - if (!isGzipAvailable()) { - throw new Error( - 'Compressed (gzip) workflow data encountered but DecompressionStream ' + - 'is not available in this runtime. Node.js 18+, browsers, and edge ' + - 'runtimes all support it.' - ); - } - const { payload } = decodeFormatPrefix(data); - const inflated = await gunzipBytes(payload); - recordStats(stats, 'gzip', inflated.length, data.length); - return inflated; - } + const { payload } = decodeFormatPrefix(data); + const inflated = await (codec === 'zstd' + ? decompressZstd(payload, options?.zstdDecoder) + : decompressGzip(payload)); + recordStats(stats, codec, inflated.length, data.length); + return inflated; +} - recordStats(stats, 'none', data.length, data.length); - return data; +/** + * Decompress without starting a portable asynchronous codec. Observability's + * synchronous hydration path uses this to leave browser data untouched until + * its async hydration path is requested. + */ +export function decompressSync(data: Uint8Array): Uint8Array | undefined { + return decompressSerializedDataSync(data); } /** * Check if data is compressed (has a 'zstd' or 'gzip' format prefix). */ -export function isCompressed(data: Uint8Array | unknown): boolean { +export function isCompressed(data: Uint8Array): boolean { const prefix = peekFormatPrefix(data); return ( prefix === SerializationFormat.ZSTD || prefix === SerializationFormat.GZIP diff --git a/packages/core/src/serialization/encryption.ts b/packages/core/src/serialization/encryption.ts index e3942d0ce2..cc63d8cd27 100644 --- a/packages/core/src/serialization/encryption.ts +++ b/packages/core/src/serialization/encryption.ts @@ -180,8 +180,9 @@ export function aesKeyOf(key: PayloadKey | undefined): CryptoKey | undefined { } /** - * Encryption key parameter type. Accepts a resolved key, undefined (no encryption), - * a promise, or a resolver that can defer fetching the key until data needs it. + * A stream may receive a resolved key, an in-flight lookup, or a lazy lookup + * that should not start until the first frame arrives. It resolves this input + * only once. */ export type EncryptionKeyParam = | PayloadKey @@ -207,10 +208,10 @@ export async function resolveEncryptionKey( * @returns The encrypted data with its format prefix, or the original data if no key */ export async function encrypt( - data: Uint8Array | unknown, + data: Uint8Array, key: PayloadKey | undefined -): Promise { - if (!key || !(data instanceof Uint8Array)) return data; +): Promise { + if (!key) return data; if (isSealTarget(key)) { const sealed = await sealToPublicKey(key.recipientPublicKey, data, key.aad); @@ -224,25 +225,13 @@ export async function encrypt( return encodeWithFormatPrefix(SerializationFormat.ENCRYPTED, encrypted); } -/** - * Decrypt a format-prefixed payload if it's encrypted or sealed. - * - * Strips the `encr`/`encp` format prefix and recovers the inner payload. - * Opening a sealed (`encp`) payload requires the run's X25519 keypair, so the - * caller must supply {@link RunPayloadKeys} — a bare symmetric key cannot do - * it, and neither can a {@link SealTarget} (which is write-only by design). - * - * @param data - The potentially encrypted data - * @param key - Encryption key (undefined to skip decryption) - * @returns The decrypted inner payload, or the original data if not encrypted - */ /** * Open a sealed (`encp`) envelope. Split out from {@link decrypt} to keep * each scheme's error handling readable on its own. */ async function openSealedEnvelope( data: Uint8Array, - key: PayloadKey | undefined + key: DecryptionKey | undefined ): Promise { // Sealed payloads need the private scalar. Anything else — no key, a bare // symmetric key, or a write-only seal target — cannot open them. @@ -265,22 +254,37 @@ async function openSealedEnvelope( try { return await openSealed(key.keyPair, payload, key.aad); } catch (error) { - // The sealed-box layer only sees the stripped payload, so it cannot - // record the outer envelope prefix. Enrich it here before rethrowing. - if (RuntimeDecryptionError.is(error) && error.context) { - error.context.formatPrefix = SerializationFormat.SEALED; - } - throw error; + throw addFormatPrefix(error, SerializationFormat.SEALED); } } -export async function decrypt( - data: Uint8Array | unknown, - key: PayloadKey | undefined -): Promise { - // Non-binary data is returned as-is. - if (!(data instanceof Uint8Array)) return data; +function requireAesDecryptionKey( + data: Uint8Array, + key: DecryptionKey | undefined +): CryptoKey { + const aesKey = aesKeyOf(key); + if (aesKey) return aesKey; + throw new RuntimeDecryptionError( + 'Encrypted data encountered but no encryption key is available. ' + + 'Encryption is not configured or no key was provided for this run.', + { + context: { + operation: 'decrypt', + byteLength: data.byteLength, + formatPrefix: 'encr', + }, + } + ); +} + +/** + * Decrypt a format-prefixed payload if it is encrypted or sealed. + */ +export async function decrypt( + data: Uint8Array, + key: DecryptionKey | undefined +): Promise { const format = peekFormatPrefix(data); if (format === SerializationFormat.SEALED) { @@ -290,33 +294,19 @@ export async function decrypt( // If the data is not encrypted, return it unchanged. if (format !== SerializationFormat.ENCRYPTED) return data; - // If the data is encrypted but no symmetric key is available, fail fast. - const aesKey = aesKeyOf(key); - if (!aesKey) { - throw new RuntimeDecryptionError( - 'Encrypted data encountered but no encryption key is available. ' + - 'Encryption is not configured or no key was provided for this run.', - { - context: { - operation: 'decrypt', - byteLength: data.byteLength, - formatPrefix: 'encr', - }, - } - ); - } + const aesKey = requireAesDecryptionKey(data, key); const { payload } = decodeFormatPrefix(data); try { return await aesGcmDecrypt(aesKey, payload); } catch (error) { - // The low-level AES layer only sees the stripped payload, so it cannot - // record the outer envelope prefix. This layer peeked it (`encr`), so - // enrich the diagnostic context with the real format prefix before - // rethrowing. - if (RuntimeDecryptionError.is(error) && error.context) { - error.context.formatPrefix = format; - } - throw error; + throw addFormatPrefix(error, format); + } +} + +function addFormatPrefix(error: unknown, format: string): unknown { + if (RuntimeDecryptionError.is(error) && error.context) { + error.context.formatPrefix = format; } + return error; } diff --git a/packages/core/src/serialization/format.ts b/packages/core/src/serialization/format.ts index 28f1bc134d..19ff1472bf 100644 --- a/packages/core/src/serialization/format.ts +++ b/packages/core/src/serialization/format.ts @@ -36,12 +36,8 @@ const formatDecoder = new TextDecoder(); */ export function encodeWithFormatPrefix( format: FormatPrefix, - payload: Uint8Array | unknown -): Uint8Array | unknown { - if (!(payload instanceof Uint8Array)) { - return payload; - } - + payload: Uint8Array +): Uint8Array { const prefixBytes = formatEncoder.encode(format); const result = new Uint8Array(FORMAT_PREFIX_LENGTH + payload.length); result.set(prefixBytes, 0); @@ -59,10 +55,8 @@ export function encodeWithFormatPrefix( * @param data - The format-prefixed data * @returns The format prefix, or null */ -export function peekFormatPrefix( - data: Uint8Array | unknown -): FormatPrefix | null { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { +export function peekFormatPrefix(data: Uint8Array): FormatPrefix | null { + if (data.length < FORMAT_PREFIX_LENGTH) { return null; } const prefixBytes = data.subarray(0, FORMAT_PREFIX_LENGTH); @@ -71,10 +65,16 @@ export function peekFormatPrefix( } /** - * Check if data is encrypted (has 'encr' format prefix). + * Check if data is encrypted, whether symmetrically (`encr`) or sealed to a + * run's public key (`encp`). Use the exact prefix when the scheme matters. */ -export function isEncrypted(data: Uint8Array | unknown): boolean { - return peekFormatPrefix(data) === SerializationFormat.ENCRYPTED; +export function isEncrypted(data: unknown): boolean { + if (!(data instanceof Uint8Array)) return false; + const prefix = peekFormatPrefix(data); + return ( + prefix === SerializationFormat.ENCRYPTED || + prefix === SerializationFormat.SEALED + ); } /** @@ -92,18 +92,10 @@ export function isEncrypted(data: Uint8Array | unknown): boolean { * @returns An object with the format prefix and payload * @throws Error if the data is too short or has an invalid prefix */ -export function decodeFormatPrefix(data: Uint8Array | unknown): { +export function decodeFormatPrefix(data: Uint8Array): { format: FormatPrefix; payload: Uint8Array; } { - // Compat for legacy specVersion 1 runs that don't have a format prefix - if (!(data instanceof Uint8Array)) { - return { - format: SerializationFormat.DEVALUE_V1, - payload: new TextEncoder().encode(JSON.stringify(data)), - }; - } - if (data.length < FORMAT_PREFIX_LENGTH) { throw new Error( `Data too short to contain format prefix: expected at least ${FORMAT_PREFIX_LENGTH} bytes, got ${data.length}` diff --git a/packages/core/src/serialization/index.ts b/packages/core/src/serialization/index.ts index 8fe3d68213..a241ca034d 100644 --- a/packages/core/src/serialization/index.ts +++ b/packages/core/src/serialization/index.ts @@ -20,6 +20,7 @@ export { // Re-export composable encryption export { type CryptoKey, + type DecryptionKey, decrypt, type EncryptionKeyParam, encrypt, @@ -40,6 +41,7 @@ export type { Reducers, Revivers, SerializableSpecial, + SerializationFormatType, } from './types.js'; export { isFormatPrefix, SerializationFormat } from './types.js'; diff --git a/packages/core/src/serialization/payload.ts b/packages/core/src/serialization/payload.ts new file mode 100644 index 0000000000..d62b03cbf9 --- /dev/null +++ b/packages/core/src/serialization/payload.ts @@ -0,0 +1,26 @@ +import { type CompressionStats, compress, decompress } from './compression.js'; +import { + type DecryptionKey, + decrypt, + encrypt, + type PayloadKey, +} from './encryption.js'; + +/** Apply the storage layers in their only valid order: compress, then encrypt. */ +export async function encodePayload( + data: Uint8Array, + key: PayloadKey | undefined, + compression: boolean, + stats?: CompressionStats +): Promise { + return encrypt(await compress(data, compression, stats), key); +} + +/** Remove the storage layers in reverse order: decrypt, then decompress. */ +export async function decodePayload( + data: Uint8Array, + key: DecryptionKey | undefined, + stats?: CompressionStats +): Promise { + return decompress(await decrypt(data, key), stats); +} diff --git a/packages/core/src/serialization/replay.ts b/packages/core/src/serialization/replay.ts new file mode 100644 index 0000000000..3a3a116bc1 --- /dev/null +++ b/packages/core/src/serialization/replay.ts @@ -0,0 +1,19 @@ +import type { CompressionStats } from './compression.js'; +import type { DecryptionKey } from './encryption.js'; +import { decodePayload } from './payload.js'; + +/** Host-owned bytes, or a tagged legacy value from before binary envelopes. */ +export type PreparedReplayPayload = Uint8Array | { readonly legacy: unknown }; + +/** + * Decrypt and decompress persisted bytes without creating VM-owned values. + * + * Native Node and portable browser codecs share one asynchronous contract. + */ +export async function prepareReplayPayload( + value: Uint8Array, + key: DecryptionKey | undefined, + compressionStats?: CompressionStats +): Promise { + return decodePayload(value, key, compressionStats); +} diff --git a/packages/core/src/serialization/serialization.test.ts b/packages/core/src/serialization/serialization.test.ts index 4a9a07afbb..89df0bdad0 100644 --- a/packages/core/src/serialization/serialization.test.ts +++ b/packages/core/src/serialization/serialization.test.ts @@ -66,6 +66,7 @@ describe('isFormatPrefix', () => { it('should reject strings that are too long', () => { expect(isFormatPrefix('abcde')).toBe(false); expect(isFormatPrefix('abcdef')).toBe(false); + expect(isFormatPrefix('abcd\n')).toBe(false); }); it('should reject uppercase characters', () => { @@ -116,7 +117,7 @@ describe('encodeWithFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded.length).toBe(4 + 3); // First 4 bytes should be 'devl' @@ -125,28 +126,12 @@ describe('encodeWithFormatPrefix', () => { expect(Array.from(encoded.subarray(4))).toEqual([1, 2, 3]); }); - it('should return non-Uint8Array values unchanged', () => { - const str = 'hello'; - expect(encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, str)).toBe( - str - ); - - const num = 42; - expect(encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, num)).toBe( - num - ); - - expect( - encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, null) - ).toBeNull(); - }); - it('should handle empty payload', () => { const payload = new Uint8Array(0); const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded.length).toBe(4); }); @@ -156,7 +141,7 @@ describe('encodeWithFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded.length).toBe(4 + 100000); }); }); @@ -167,22 +152,13 @@ describe('decodeFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); const decoded = decodeFormatPrefix(encoded); expect(decoded.format).toBe('devl'); expect(decoded.payload).toEqual(new Uint8Array([1, 2, 3])); }); - it('should handle legacy non-binary data', () => { - const legacyData = [1, 'hello', { a: 2 }]; - const decoded = decodeFormatPrefix(legacyData); - expect(decoded.format).toBe('devl'); - expect(decoded.payload).toEqual( - new TextEncoder().encode(JSON.stringify(legacyData)) - ); - }); - it('should throw for data too short', () => { expect(() => decodeFormatPrefix(new Uint8Array([1, 2, 3]))).toThrow( /Data too short to contain format prefix/ @@ -203,7 +179,7 @@ describe('decodeFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.ENCRYPTED, payload - ) as Uint8Array; + ); const decoded = decodeFormatPrefix(encoded); expect(decoded.format).toBe('encr'); @@ -222,13 +198,6 @@ describe('peekFormatPrefix', () => { expect(peekFormatPrefix(encoded)).toBe('devl'); }); - it('should return null for non-binary data', () => { - expect(peekFormatPrefix('not binary')).toBeNull(); - expect(peekFormatPrefix(42)).toBeNull(); - expect(peekFormatPrefix(null)).toBeNull(); - expect(peekFormatPrefix(undefined)).toBeNull(); - }); - it('should return null for data too short', () => { expect(peekFormatPrefix(new Uint8Array([1]))).toBeNull(); expect(peekFormatPrefix(new Uint8Array([1, 2, 3]))).toBeNull(); @@ -286,19 +255,12 @@ describe('encrypt', () => { expect(result).toBe(data); }); - it('should return non-Uint8Array data unchanged even with key', async () => { - const key = await makeKey(); - const data = 'string data'; - const result = await encrypt(data, key); - expect(result).toBe(data); - }); - it('should encrypt and add encr prefix when key provided', async () => { const key = await makeKey(); const data = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, new Uint8Array([1, 2, 3]) - ) as Uint8Array; + ); const encrypted = await encrypt(data, key); expect(encrypted).toBeInstanceOf(Uint8Array); @@ -307,10 +269,17 @@ describe('encrypt', () => { }); describe('decrypt', () => { - it('should return non-binary data unchanged', async () => { - const data = [1, 2, 3]; - const result = await decrypt(data, undefined); - expect(result).toBe(data); + it('uses one asynchronous contract for AES envelopes on Node', async () => { + const key = await makeKey(); + const data = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + new Uint8Array([1, 2, 3]) + ) as Uint8Array; + const encrypted = (await encrypt(data, key)) as Uint8Array; + + const decrypted = decrypt(encrypted, key); + expect(decrypted).toBeInstanceOf(Promise); + expect(await decrypted).toEqual(data); }); it('should return non-encrypted binary data unchanged', async () => { @@ -364,7 +333,7 @@ describe('decrypt', () => { const tampered = new Uint8Array(encrypted); tampered[tampered.length - 1] ^= 0xff; - const error = await decrypt(tampered, key).catch((e) => e); + const error = await decrypt(tampered, key).catch((caught) => caught); expect(RuntimeDecryptionError.is(error)).toBe(true); expect(error.context).toMatchObject({ operation: 'decrypt', @@ -403,9 +372,9 @@ describe('sealed envelopes', () => { )) as Uint8Array; expect(peekFormatPrefix(sealed)).toBe(SerializationFormat.SEALED); - // Sealed data must not be mistaken for symmetrically encrypted data by - // anything dispatching on the prefix. - expect(isEncrypted(sealed)).toBe(false); + // Both symmetric and sealed envelopes are encrypted. Callers that need to + // distinguish the schemes dispatch on the exact prefix. + expect(isEncrypted(sealed)).toBe(true); }); it('round-trips: cross-run writer seals, owning run opens', async () => { @@ -454,7 +423,7 @@ describe('sealed envelopes', () => { // A bare CryptoKey is the legacy "symmetric only" shape. It has no // scalar, so it cannot open a sealed payload — and must say so clearly // rather than failing an auth tag somewhere deeper. - const error = await decrypt(sealed, aes).catch((e) => e); + const error = await decrypt(sealed, aes).catch((caught) => caught); expect(RuntimeDecryptionError.is(error)).toBe(true); expect(error.message).toMatch(/no run keypair is available/); expect(error.context).toMatchObject({ @@ -488,7 +457,7 @@ describe('sealed envelopes', () => { // A seal target carries no symmetric capability, so it must be treated // exactly like "no key" rather than silently coerced. const error = await decrypt(encrypted, sealTo(keyPair.publicKey)).catch( - (e) => e + (caught) => caught ); expect(RuntimeDecryptionError.is(error)).toBe(true); expect(error.message).toMatch(/no encryption key is available/); diff --git a/packages/core/src/serialization/step.ts b/packages/core/src/serialization/step.ts index 7e8c8730d7..efcb03e878 100644 --- a/packages/core/src/serialization/step.ts +++ b/packages/core/src/serialization/step.ts @@ -8,14 +8,10 @@ import { SerializationError } from '@workflow/errors'; import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; -import { compress, decompress } from './compression.js'; -import { - decrypt as decryptData, - encrypt as encryptData, - type PayloadKey, -} from './encryption.js'; +import type { DecryptionKey, PayloadKey } from './encryption.js'; import { formatSerializationError, rethrowIfRuntimeError } from './errors.js'; import { decodeFormatPrefix, encodeWithFormatPrefix } from './format.js'; +import { decodePayload, encodePayload } from './payload.js'; import { SerializationFormat } from './types.js'; /** @@ -25,20 +21,19 @@ export async function serialize( value: unknown, encryptionKey?: PayloadKey, options?: CodecOptions -): Promise { +): Promise { try { const payload = devalueCodec.serialize(value, 'step', options); const prefixed = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; - // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress( + ); + return await encodePayload( prefixed, - options?.compression === true, + encryptionKey, + options?.compression ?? false, options?.compressionStats ); - return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); const { message, hint } = formatSerializationError('step value', error); @@ -50,25 +45,25 @@ export async function serialize( * Deserialize a value for the step execution environment. */ export async function deserialize( - data: Uint8Array | unknown, - encryptionKey?: PayloadKey, + data: unknown, + encryptionKey?: DecryptionKey, options?: CodecOptions ): Promise { - const decrypted = await decompress( - await decryptData(data, encryptionKey), - options?.compressionStats - ); - - if (!(decrypted instanceof Uint8Array)) { + if (!(data instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { - return devalueCodec.deserializeLegacy(decrypted, 'step', options); + return devalueCodec.deserializeLegacy(data, 'step', options); } throw new Error( 'Cannot deserialize non-binary data without legacy support' ); } - const { format, payload } = decodeFormatPrefix(decrypted); + const prepared = await decodePayload( + data, + encryptionKey, + options?.compressionStats + ); + const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { return devalueCodec.deserialize(payload, 'step', options); diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 617c4cd205..5da3211b18 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -3,50 +3,19 @@ */ import type { RuntimeDecryptionErrorContext } from '@workflow/errors'; +import { + type FormatPrefix, + isFormatPrefix, + SerializationFormat, + type SerializationFormatType, +} from '@workflow/world/serialization-format.js'; -// ---- Format Prefix ---- - -/** - * A format prefix is exactly 4 lowercase alphanumeric characters [a-z0-9]. - * - * This is a branded string type — use `isFormatPrefix()` to validate - * at runtime. The `SerializationFormat` object provides well-known - * constants, but codecs may define additional prefixes. - */ -export type FormatPrefix = string & { readonly __brand: 'FormatPrefix' }; - -/** - * Runtime type guard for format prefix strings. - * - * Validates that a string is exactly 4 characters of [a-z0-9]. - */ -export function isFormatPrefix(value: string): value is FormatPrefix { - return value.length === 4 && /^[a-z0-9]{4}$/.test(value); -} - -/** - * Well-known format prefix constants. Codecs may define additional ones. - */ -export const SerializationFormat = { - /** devalue stringify/parse with TextEncoder/TextDecoder */ - DEVALUE_V1: 'devl' as FormatPrefix, - /** Encrypted payload (inner payload has its own format prefix) */ - ENCRYPTED: 'encr' as FormatPrefix, - /** - * Sealed payload — asymmetrically encrypted to a run's X25519 public key - * (inner payload has its own format prefix). - * - * Used for *cross-run* writes (hook payloads, forwarded stream frames), - * where the writer holds only the recipient run's public key and therefore - * cannot decrypt. A run's own payloads continue to use {@link ENCRYPTED}. - * See `sealed-box.ts` for the construction. - */ - SEALED: 'encp' as FormatPrefix, - /** Gzip-compressed payload (inner payload has its own format prefix) */ - GZIP: 'gzip' as FormatPrefix, - /** Zstandard-compressed payload (inner payload has its own format prefix) */ - ZSTD: 'zstd' as FormatPrefix, -} as const; +export { + type FormatPrefix, + isFormatPrefix, + SerializationFormat, + type SerializationFormatType, +}; // ---- Serializable Types ---- diff --git a/packages/core/src/serialization/workflow-vm.ts b/packages/core/src/serialization/workflow-vm.ts index 7ebf7c938f..4bad5fd921 100644 --- a/packages/core/src/serialization/workflow-vm.ts +++ b/packages/core/src/serialization/workflow-vm.ts @@ -49,7 +49,7 @@ export function serialize(value: unknown): Uint8Array { * @param data - Uint8Array with format prefix, or legacy non-binary data * @returns The deserialized value */ -export function deserialize(data: Uint8Array | unknown): unknown { +export function deserialize(data: unknown): unknown { // Legacy: non-binary data if (!(data instanceof Uint8Array)) { if (devalueVmCodec.deserializeLegacy) { diff --git a/packages/core/src/serialization/workflow.ts b/packages/core/src/serialization/workflow.ts index 344fec0151..a0791ba733 100644 --- a/packages/core/src/serialization/workflow.ts +++ b/packages/core/src/serialization/workflow.ts @@ -27,10 +27,7 @@ import { SerializationFormat } from './types.js'; export function serialize(value: unknown, options?: CodecOptions): Uint8Array { try { const payload = devalueCodec.serialize(value, 'workflow', options); - return encodeWithFormatPrefix( - SerializationFormat.DEVALUE_V1, - payload - ) as Uint8Array; + return encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, payload); } catch (error) { const { message, hint } = formatSerializationError('workflow value', error); throw new SerializationError(message, { hint, cause: error }); @@ -44,10 +41,7 @@ export function serialize(value: unknown, options?: CodecOptions): Uint8Array { * @param options - Optional global, extra revivers for VM-context deserialization * @returns The deserialized value */ -export function deserialize( - data: Uint8Array | unknown, - options?: CodecOptions -): unknown { +export function deserialize(data: unknown, options?: CodecOptions): unknown { if (!(data instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { return devalueCodec.deserializeLegacy(data, 'workflow', options); diff --git a/packages/core/src/step/context-storage.ts b/packages/core/src/step/context-storage.ts index 91f41e099e..d19a1216e0 100644 --- a/packages/core/src/step/context-storage.ts +++ b/packages/core/src/step/context-storage.ts @@ -1,6 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { FlushableStreamState } from '../flushable-stream.js'; -import type { PayloadKey } from '../serialization/encryption.js'; +import type { DecryptionKey } from '../serialization/encryption.js'; import type { WorkflowMetadata } from '../workflow/get-workflow-metadata.js'; import type { StepMetadata } from './get-step-metadata.js'; @@ -57,7 +57,7 @@ export type StepContext = { */ preCompletionOps: Promise[]; closureVars?: Record; - encryptionKey?: PayloadKey; + encryptionKey?: DecryptionKey; writables?: Map; /** * Turbo mode only: a promise that resolves once the backgrounded diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 98b8728269..3bcd3f08fa 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -26,7 +26,7 @@ import { getPortLazy } from './runtime/get-port-lazy.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; -import type { PayloadKey } from './serialization/encryption.js'; +import type { DecryptionKey } from './serialization/encryption.js'; import { dehydrateWorkflowReturnValue, hydrateWorkflowArguments, @@ -123,7 +123,7 @@ interface WorkflowSessionOptions { readonly workflowCode: string; readonly workflowRun: WorkflowRun; readonly events: Event[]; - readonly encryptionKey: PayloadKey | undefined; + readonly encryptionKey: DecryptionKey | undefined; readonly replayPayloadCache: ReplayPayloadCache; readonly runReadyBarrier?: Promise; readonly worldCapabilities?: WorldCapabilities; @@ -271,7 +271,7 @@ export async function runWorkflow( workflowCode: string, workflowRun: WorkflowRun, events: Event[], - encryptionKey: PayloadKey | undefined, + encryptionKey: DecryptionKey | undefined, /** * Optional per-run cache for replay payload preparation and immutable final * values. Owned by the inline execution loop for this invocation. @@ -291,7 +291,7 @@ export async function runWorkflow( * capabilities are treated as unsupported. */ worldCapabilities?: WorldCapabilities -): Promise { +): Promise { const result = await replayWorkflow({ workflowCode, workflowRun, diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index 25297fb15f..f1a10b04bb 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -7,7 +7,9 @@ */ import { + deriveRunPayloadKeys, extractClassName, + hydrateDataWithKey, hydrateResourceIO as hydrateResourceIOGeneric, isEncryptedData, isExpiredStub, @@ -18,6 +20,15 @@ import { } from '@workflow/core/serialization-format'; import { getEventDataRefFields } from '@workflow/world'; +const browserHydrationOptions = { + zstdDecoder: async (payload: Uint8Array): Promise => { + const { decompressZstdInBrowser } = await import( + './zstd-browser-decoder.js' + ); + return decompressZstdInBrowser(payload); + }, +}; + // Re-export types and utilities that consumers need export { CLASS_INSTANCE_REF_TYPE, @@ -479,29 +490,14 @@ export async function hydrateResourceIOWithKey( * Async hydration for web display. * * This follows the same resource-field mapping as {@link hydrateResourceIO}, - * but can also inflate compressed browser payloads through the registered - * zstd WASM decoder. When a key is provided, encrypted fields are decrypted - * first and then inflated/hydrated. + * but can also inflate compressed browser payloads through the zstd WASM + * decoder. When a key is provided, encrypted fields are decrypted first and + * then inflated/hydrated. */ export async function hydrateResourceIOAsync( resource: T, key?: Uint8Array ): Promise { - const { hydrateDataWithKey, deriveRunPayloadKeys } = await import( - '@workflow/core/serialization-format' - ); - // Payloads may be zstd-compressed (the Web DecompressionStream has no zstd); - // register the WASM-backed browser decoder before hydrating. Idempotent and - // lazy — the WASM is only compiled when a zstd payload is actually decoded. - const { ensureZstdDecoderRegistered } = await import( - './zstd-browser-decoder.js' - ); - ensureZstdDecoderRegistered(); - // Resolve the *full* key capability, not just the symmetric key: a run's - // event log can contain sealed ('encp') payloads that another run wrote to - // it (a cross-deployment hook resumption, say), and opening those needs the - // run's X25519 scalar in addition to its AES key. Both are derived from the - // same 32 bytes the key-retrieval endpoint returns. // Resolve the *full* key capability, not just the symmetric key: a run's // event log can contain sealed ('encp') payloads that another run wrote to // it (a cross-deployment hook resumption, say), and opening those needs the @@ -514,11 +510,18 @@ export async function hydrateResourceIOAsync( // Already-hydrated: encrypted marker with stored bytes if (isEncryptedMarker(value)) { const raw = (value as any).__encryptedData as Uint8Array; - return cryptoKey ? hydrateDataWithKey(raw, revivers, cryptoKey) : value; + return cryptoKey + ? hydrateDataWithKey(raw, revivers, cryptoKey, browserHydrationOptions) + : value; } // Raw Uint8Array: may be encrypted, compressed, or plain devalue. if (value instanceof Uint8Array) { - return hydrateDataWithKey(value, revivers, cryptoKey); + return hydrateDataWithKey( + value, + revivers, + cryptoKey, + browserHydrationOptions + ); } // Not serialized — return as-is. return value; diff --git a/packages/web-shared/src/lib/zstd-browser-decoder.ts b/packages/web-shared/src/lib/zstd-browser-decoder.ts index 05cf339dfd..57fd551333 100644 --- a/packages/web-shared/src/lib/zstd-browser-decoder.ts +++ b/packages/web-shared/src/lib/zstd-browser-decoder.ts @@ -2,18 +2,14 @@ * Browser zstd decoder for the o11y read path. * * The Web `DecompressionStream` has no zstd support, so `@workflow/core`'s - * `hydrateDataWithKey` delegates zstd inflation to a decoder registered via - * `registerZstdDecoder`. This module supplies that decoder, backed by the - * `@tootallnate/zstd-wasm` single-file WASM decoder. + * `hydrateDataWithKey` delegates zstd inflation to the decoder exported by + * this module, backed by the `@tootallnate/zstd-wasm` single-file WASM decoder. * * The package leaves WASM sourcing to the caller; we resolve the shipped * `zstd.wasm` as a bundler asset (`new URL(..., import.meta.url)`, the same * pattern the trace-viewer Worker uses) and compile it once, lazily — the * WASM is fetched only the first time a zstd payload is actually decoded. */ -import { registerZstdDecoder } from '@workflow/core/serialization-format'; - -let registered = false; let modulePromise: Promise | undefined; function loadWasmModule(): Promise { @@ -27,16 +23,13 @@ function loadWasmModule(): Promise { } /** - * Register the browser zstd decoder with `@workflow/core` (idempotent). - * Call this before hydrating payloads that may be zstd-compressed; the - * actual WASM compile + decode happens lazily on first use. + * Decode zstd in browsers. The WASM compile + decode remains lazy and the + * compiled module is reused for subsequent payloads. */ -export function ensureZstdDecoderRegistered(): void { - if (registered) return; - registered = true; - registerZstdDecoder(async (payload) => { - const { decompressBytes } = await import('@tootallnate/zstd-wasm'); - const wasmModule = await loadWasmModule(); - return decompressBytes(wasmModule, payload); - }); +export async function decompressZstdInBrowser( + payload: Uint8Array +): Promise { + const { decompressBytes } = await import('@tootallnate/zstd-wasm'); + const wasmModule = await loadWasmModule(); + return decompressBytes(wasmModule, payload); } diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index f2ef4a3a30..734e68dc51 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -34,6 +34,7 @@ import { type ListEventsByCorrelationIdParams, type ListEventsParams, type PaginatedResponse, + peekSerializationFormat, StructuredErrorSchema, WaitSchema, WorkflowRunSchema, @@ -59,7 +60,6 @@ import { recordClientSpanStatus, withHttpClientSpan, } from './http-core.js'; -import { hasSerializedDataFormatPrefix } from './serialized-data.js'; import { deserializeStep, StepWireSchema } from './steps.js'; import { ErrorType, @@ -390,7 +390,7 @@ const legacyStructuredErrorEventTypes = new Set([ ]); function decodeLegacyStructuredError(payload: Uint8Array): unknown { - if (hasSerializedDataFormatPrefix(payload)) return payload; + if (peekSerializationFormat(payload) !== null) return payload; try { const parsed = StructuredErrorSchema.safeParse(decode(payload.slice())); diff --git a/packages/world-vercel/src/serialized-data.test.ts b/packages/world-vercel/src/serialized-data.test.ts new file mode 100644 index 0000000000..f1731f53dc --- /dev/null +++ b/packages/world-vercel/src/serialized-data.test.ts @@ -0,0 +1,46 @@ +import { gzipSync, zstdCompressSync } from 'node:zlib'; +import { peekSerializationFormat } from '@workflow/world/serialization-format.js'; +import { describe, expect, it } from 'vitest'; +import { normalizeSerializedData } from './serialized-data.js'; + +const encoder = new TextEncoder(); + +function envelope(format: string, payload: Uint8Array): Uint8Array { + const data = new Uint8Array(4 + payload.length); + data.set(encoder.encode(format)); + data.set(payload, 4); + return data; +} + +describe('serialized data normalization', () => { + it.each([ + 'devl', + 'encr', + 'encp', + 'gzip', + 'zstd', + ])('recognizes the %s envelope', (format) => { + expect(peekSerializationFormat(envelope(format, new Uint8Array()))).toBe( + format + ); + }); + + it('leaves non-compressed envelopes untouched', () => { + const encrypted = envelope('encr', new Uint8Array([1, 2, 3])); + expect(normalizeSerializedData(encrypted)).toBe(encrypted); + }); + + it('decompresses gzip envelopes with the shared world codec', () => { + const original = new TextEncoder().encode('persisted workflow payload'); + const compressed = envelope('gzip', gzipSync(original)); + + expect(normalizeSerializedData(compressed)).toEqual(original); + }); + + it('decompresses zstd envelopes with the shared world codec', () => { + const original = new TextEncoder().encode('persisted workflow payload'); + const compressed = envelope('zstd', zstdCompressSync(original)); + + expect(normalizeSerializedData(compressed)).toEqual(original); + }); +}); diff --git a/packages/world-vercel/src/serialized-data.ts b/packages/world-vercel/src/serialized-data.ts index 8f17150fd4..5485aa55ce 100644 --- a/packages/world-vercel/src/serialized-data.ts +++ b/packages/world-vercel/src/serialized-data.ts @@ -1,72 +1,25 @@ import { WorkflowWorldError } from '@workflow/errors'; +import { decompressSerializedDataSync } from '@workflow/world/serialization-compression.js'; +import { + peekSerializationFormat, + SerializationFormat, +} from '@workflow/world/serialization-format.js'; -const FORMAT_PREFIX_LENGTH = 4; -const DEVALUE_FORMAT_PREFIX = 'devl'; -const ENCRYPTED_FORMAT_PREFIX = 'encr'; -const GZIP_FORMAT_PREFIX = 'gzip'; -const ZSTD_FORMAT_PREFIX = 'zstd'; -const formatDecoder = new TextDecoder(); - -const SERIALIZED_DATA_FORMAT_PREFIXES = new Set([ - DEVALUE_FORMAT_PREFIX, - ENCRYPTED_FORMAT_PREFIX, - GZIP_FORMAT_PREFIX, - ZSTD_FORMAT_PREFIX, -]); - -interface NodeZlibDecode { - gunzipSync?: (data: Uint8Array) => Uint8Array; - zstdDecompressSync?: (data: Uint8Array) => Uint8Array; -} - -function getNodeZlib(): NodeZlibDecode | undefined { - try { - return ( - globalThis as { - process?: { getBuiltinModule?: (id: string) => NodeZlibDecode }; - } - ).process?.getBuiltinModule?.('node:zlib'); - } catch { - return undefined; - } -} - -function peekFormatPrefix(value: unknown): string | null { +export function normalizeSerializedData(value: unknown): unknown { + const format = peekSerializationFormat(value); if ( - !(value instanceof Uint8Array) || - value.byteLength < FORMAT_PREFIX_LENGTH + format !== SerializationFormat.ZSTD && + format !== SerializationFormat.GZIP ) { - return null; - } - return formatDecoder.decode(value.subarray(0, FORMAT_PREFIX_LENGTH)); -} - -export function hasSerializedDataFormatPrefix(value: unknown): boolean { - const format = peekFormatPrefix(value); - return format !== null && SERIALIZED_DATA_FORMAT_PREFIXES.has(format); -} - -function decompress(format: string, payload: Uint8Array): Uint8Array { - const zlib = getNodeZlib(); - const decompress = - format === ZSTD_FORMAT_PREFIX ? zlib?.zstdDecompressSync : zlib?.gunzipSync; - - if (!decompress) { - throw new WorkflowWorldError( - `Received ${format}-compressed workflow data, but this Node.js runtime does not support ${format} decompression. Use a compatible Node.js runtime or request unresolved data.` - ); - } - - return new Uint8Array(decompress(payload)); -} - -export function normalizeSerializedData(value: unknown): unknown { - const format = peekFormatPrefix(value); - if (format !== ZSTD_FORMAT_PREFIX && format !== GZIP_FORMAT_PREFIX) { return value; } const bytes = value as Uint8Array; - return decompress(format, bytes.subarray(FORMAT_PREFIX_LENGTH)); + const decompressed = decompressSerializedDataSync(bytes); + if (decompressed) return decompressed; + + throw new WorkflowWorldError( + `Received ${format}-compressed workflow data, but this Node.js runtime does not support ${format} decompression. Use a compatible Node.js runtime or request unresolved data.` + ); } export function normalizeWorkflowRunData>( diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 701acb4ea3..6068b365b1 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -111,6 +111,15 @@ export { LegacySerializedDataSchemaV1, SerializedDataSchema, } from './serialization.js'; +export { + type FormatPrefix, + isFormatPrefix, + isSerializationFormat, + peekSerializationFormat, + SERIALIZATION_FORMAT_PREFIX_LENGTH, + SerializationFormat, + type SerializationFormatType, +} from './serialization-format.js'; export type * from './shared.js'; export type { GetChunksOptions, diff --git a/packages/world/src/serialization-compression.ts b/packages/world/src/serialization-compression.ts new file mode 100644 index 0000000000..5b6b34589d --- /dev/null +++ b/packages/world/src/serialization-compression.ts @@ -0,0 +1,102 @@ +import { + peekSerializationFormat, + SERIALIZATION_FORMAT_PREFIX_LENGTH, + SerializationFormat, +} from './serialization-format.js'; + +type NodeCompress = (data: Uint8Array, options?: unknown) => Uint8Array; +type NodeDecompress = (data: Uint8Array) => Uint8Array; + +interface NodeZlib { + constants?: Record; + gzipSync?: NodeCompress; + gunzipSync?: NodeDecompress; + zstdCompressSync?: NodeCompress; + zstdDecompressSync?: NodeDecompress; +} + +interface NativeCompressionCodec { + compress(data: Uint8Array, level?: number): Uint8Array; + decompress(data: Uint8Array): Uint8Array; +} + +/** Resolve Node codecs without introducing a static Node import for browsers. */ +const nodeZlib = (() => { + try { + return ( + globalThis as { + process?: { getBuiltinModule?: (id: string) => NodeZlib }; + } + ).process?.getBuiltinModule?.('node:zlib'); + } catch { + return undefined; + } +})(); + +function asUint8Array(value: Uint8Array): Uint8Array { + return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); +} + +function nativeCodec( + compress: NodeCompress | undefined, + decompress: NodeDecompress | undefined, + levelParameter?: number +): NativeCompressionCodec | undefined { + if (!compress || !decompress) return undefined; + + return { + compress(data, level) { + const options = + level !== undefined && levelParameter !== undefined + ? { params: { [levelParameter]: level } } + : undefined; + return asUint8Array(compress(data, options)); + }, + decompress(data) { + return asUint8Array(decompress(data)); + }, + }; +} + +const nativeCodecs = { + gzip: nativeCodec(nodeZlib?.gzipSync, nodeZlib?.gunzipSync), + zstd: nativeCodec( + nodeZlib?.zstdCompressSync, + nodeZlib?.zstdDecompressSync, + nodeZlib?.constants?.ZSTD_c_compressionLevel + ), +}; + +/** Return a native codec only when both its reader and writer are available. */ +export function getNativeCompressionCodec( + codec: 'gzip' | 'zstd' +): NativeCompressionCodec | undefined { + return nativeCodecs[codec]; +} + +/** + * Inflate a persisted gzip/zstd envelope synchronously when Node exposes the + * corresponding codec. Non-compressed bytes pass through unchanged; an + * unavailable codec returns `undefined` so the owning runtime can choose its + * fallback or error contract. + */ +export function decompressSerializedDataSync( + data: Uint8Array +): Uint8Array | undefined { + const format = peekSerializationFormat(data); + const codec = + format === SerializationFormat.ZSTD + ? nativeCodecs.zstd + : format === SerializationFormat.GZIP + ? nativeCodecs.gzip + : undefined; + + if (!codec) { + return format === SerializationFormat.ZSTD || + format === SerializationFormat.GZIP + ? undefined + : data; + } + + return codec.decompress(data.subarray(SERIALIZATION_FORMAT_PREFIX_LENGTH)); +} diff --git a/packages/world/src/serialization-format.ts b/packages/world/src/serialization-format.ts new file mode 100644 index 0000000000..65dd20a4e5 --- /dev/null +++ b/packages/world/src/serialization-format.ts @@ -0,0 +1,72 @@ +declare const formatPrefixBrand: unique symbol; + +/** A validated four-byte lowercase alphanumeric payload envelope prefix. */ +export type FormatPrefix = string & { + readonly [formatPrefixBrand]: 'FormatPrefix'; +}; + +/** Whether a value is a valid persisted payload envelope prefix. */ +export function isFormatPrefix(value: unknown): value is FormatPrefix { + return ( + typeof value === 'string' && + value.length === 4 && + /^[a-z0-9]{4}$/.test(value) + ); +} + +function defineFormatPrefix( + value: T +): T & FormatPrefix { + if (!isFormatPrefix(value)) { + throw new Error(`Invalid serialization format prefix: ${value}`); + } + return value; +} + +/** Known four-byte envelope prefixes in the persisted payload protocol. */ +export const SerializationFormat = { + /** devalue stringify/parse with TextEncoder/TextDecoder */ + DEVALUE_V1: defineFormatPrefix('devl'), + /** Symmetrically encrypted payload */ + ENCRYPTED: defineFormatPrefix('encr'), + /** Payload sealed to a run's public key */ + SEALED: defineFormatPrefix('encp'), + /** Gzip-compressed payload */ + GZIP: defineFormatPrefix('gzip'), + /** Zstandard-compressed payload */ + ZSTD: defineFormatPrefix('zstd'), +} as const; + +export type SerializationFormatType = + (typeof SerializationFormat)[keyof typeof SerializationFormat]; + +const serializationFormats = new Set( + Object.values(SerializationFormat) +); + +/** Whether a value is one of the persisted payload protocol's known formats. */ +export function isSerializationFormat( + value: unknown +): value is SerializationFormatType { + return typeof value === 'string' && serializationFormats.has(value); +} + +export const SERIALIZATION_FORMAT_PREFIX_LENGTH = 4; +const formatDecoder = new TextDecoder(); + +/** Read a known persisted payload format without consuming its bytes. */ +export function peekSerializationFormat( + value: unknown +): SerializationFormatType | null { + if ( + !(value instanceof Uint8Array) || + value.byteLength < SERIALIZATION_FORMAT_PREFIX_LENGTH + ) { + return null; + } + + const format = formatDecoder.decode( + value.subarray(0, SERIALIZATION_FORMAT_PREFIX_LENGTH) + ); + return isSerializationFormat(format) ? format : null; +}