From 9590486c4b9b59167e86490446d40a3ab2e2c3fe Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:14:48 -0700 Subject: [PATCH 01/16] Decode replay payloads synchronously --- .../decode-replay-payloads-synchronously.md | 5 + packages/core/src/encryption.test.ts | 31 ++++- packages/core/src/encryption.ts | 118 +++++++++++++++++- packages/core/src/serialization.ts | 59 +++++++-- .../core/src/serialization/compression.ts | 19 ++- packages/core/src/serialization/encryption.ts | 56 +++++++++ 6 files changed, 268 insertions(+), 20 deletions(-) create mode 100644 .changeset/decode-replay-payloads-synchronously.md diff --git a/.changeset/decode-replay-payloads-synchronously.md b/.changeset/decode-replay-payloads-synchronously.md new file mode 100644 index 0000000000..b4faf3824b --- /dev/null +++ b/.changeset/decode-replay-payloads-synchronously.md @@ -0,0 +1,5 @@ +--- +"@workflow/core": patch +--- + +Decode replay payloads synchronously with Node AES-GCM and zstd when the runtime supports it. diff --git a/packages/core/src/encryption.test.ts b/packages/core/src/encryption.test.ts index f3544fc9e8..937463ce82 100644 --- a/packages/core/src/encryption.test.ts +++ b/packages/core/src/encryption.test.ts @@ -1,6 +1,12 @@ import { RuntimeDecryptionError } from '@workflow/errors'; import { describe, expect, it } from 'vitest'; -import { type CryptoKey, decrypt, encrypt, importKey } from './encryption.js'; +import { + type CryptoKey, + decrypt, + decryptSync, + encrypt, + importKey, +} from './encryption.js'; const RAW_KEY = new Uint8Array(32).fill(7); const OTHER_RAW_KEY = new Uint8Array(32).fill(8); @@ -27,6 +33,16 @@ describe('encryption', () => { const decoded = await decrypt(key, ciphertext); expect(new TextDecoder().decode(decoded)).toBe('hello, workflow'); }); + + it('decryptSync() returns the plaintext without a promise', async () => { + const key = await getKey(); + const plaintext = new TextEncoder().encode('synchronous replay'); + const ciphertext = await encrypt(key, plaintext); + + const decoded = decryptSync(key, ciphertext); + expect(decoded).toBeInstanceOf(Uint8Array); + expect(new TextDecoder().decode(decoded)).toBe('synchronous replay'); + }); }); describe('importKey', () => { @@ -92,6 +108,19 @@ describe('encryption', () => { expect(cause?.name).toBe('OperationError'); }); + it('keeps RuntimeDecryptionError on synchronous auth failure', async () => { + const key = await getKey(); + const ciphertext = await encrypt( + key, + new TextEncoder().encode('tamper me') + ); + ciphertext[ciphertext.length - 1] ^= 0xff; + + expect(() => decryptSync(key, ciphertext)).toThrowError( + RuntimeDecryptionError + ); + }); + it('does not record a formatPrefix at the low-level layer', async () => { // This function only ever sees the stripped AES payload // (`[nonce][ciphertext+tag]`), never the outer `encr` envelope marker. diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index 7e0675f9ce..3841d52da3 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -23,6 +23,46 @@ import { RuntimeDecryptionError, WorkflowRuntimeError } from '@workflow/errors'; // so consumers can reference it without adding `dom` lib. export type CryptoKey = import('node:crypto').webcrypto.CryptoKey; +/** + * Raw key material retained alongside keys imported by this module. + * + * Node's synchronous cipher API cannot consume a Web Crypto `CryptoKey`, and + * our keys are deliberately non-extractable. Keeping the original bytes in a + * WeakMap gives the Node replay path access to the same key without making it + * extractable or extending its lifetime beyond the `CryptoKey`. Browser/edge + * callers continue to use Web Crypto and never consult this map. + */ +const importedKeyMaterial = new WeakMap(); + +interface NodeDecipher { + setAAD(data: Uint8Array): NodeDecipher; + setAuthTag(tag: Uint8Array): NodeDecipher; + update(data: Uint8Array): Uint8Array; + final(): Uint8Array; +} + +interface NodeCrypto { + createDecipheriv( + algorithm: string, + key: Uint8Array, + iv: Uint8Array, + options?: { authTagLength?: number } + ): NodeDecipher; +} + +/** Resolve node:crypto without a static import, preserving browser bundles. */ +function getNodeCrypto(): NodeCrypto | undefined { + try { + return ( + globalThis as { + process?: { getBuiltinModule?: (id: string) => NodeCrypto }; + } + ).process?.getBuiltinModule?.('node:crypto'); + } catch { + return undefined; + } +} + /** AES-GCM nonce length in bytes. */ export const NONCE_LENGTH = 12; /** AES-GCM authentication tag length in bits. */ @@ -55,7 +95,7 @@ export async function importKey( `Encryption key must be exactly ${KEY_LENGTH} bytes, got ${raw.byteLength}` ); } - return globalThis.crypto.subtle.importKey( + const key = await globalThis.crypto.subtle.importKey( 'raw', raw, 'AES-GCM', @@ -65,6 +105,82 @@ export async function importKey( // a strict subset of `KeyUsage[]`, so this cast is sound. usages as ('encrypt' | 'decrypt')[] ); + // Copy the caller's bytes: a caller may reuse/mutate its input buffer after + // importKey(), while a CryptoKey's material is immutable. + importedKeyMaterial.set(key, raw.slice()); + return key; +} + +/** + * Decrypt AES-256-GCM synchronously when running on Node and the key was + * imported by this module. + * + * Returns `undefined` when the portable Web Crypto fallback is required (for + * example in a browser, or for an externally-created CryptoKey). Authentication + * failures throw the same RuntimeDecryptionError shape as {@link decrypt}. + */ +export function decryptSync( + key: CryptoKey, + data: Uint8Array, + aad?: Uint8Array +): Uint8Array | undefined { + const material = importedKeyMaterial.get(key); + const nodeCrypto = getNodeCrypto(); + if (!material || !nodeCrypto) return undefined; + if (!key.usages.includes('decrypt')) { + throw new RuntimeDecryptionError( + 'AES-256-GCM decryption failed: CryptoKey does not support decrypt', + { + context: { operation: 'decrypt', byteLength: data.byteLength }, + } + ); + } + + const minLength = NONCE_LENGTH + TAG_BYTES; + if (data.byteLength < minLength) { + throw new RuntimeDecryptionError( + `Encrypted data too short: expected at least ${minLength} bytes, got ${data.byteLength}`, + { + context: { + operation: 'decrypt', + byteLength: data.byteLength, + }, + } + ); + } + + const nonce = data.subarray(0, NONCE_LENGTH); + const ciphertextEnd = data.byteLength - TAG_BYTES; + const ciphertext = data.subarray(NONCE_LENGTH, ciphertextEnd); + const authTag = data.subarray(ciphertextEnd); + try { + const decipher = nodeCrypto.createDecipheriv( + 'aes-256-gcm', + material, + nonce, + { authTagLength: TAG_BYTES } + ); + if (aad) decipher.setAAD(aad); + decipher.setAuthTag(authTag); + const head = decipher.update(ciphertext); + const tail = decipher.final(); + if (tail.byteLength === 0) return head; + const plaintext = new Uint8Array(head.byteLength + tail.byteLength); + plaintext.set(head, 0); + plaintext.set(tail, head.byteLength); + return plaintext; + } catch (cause) { + throw new RuntimeDecryptionError( + `AES-256-GCM decryption failed: ${cause instanceof Error ? cause.message : String(cause)}`, + { + cause, + context: { + operation: 'decrypt', + byteLength: data.byteLength, + }, + } + ); + } } /** diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index e626cd3f91..684180edb8 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -38,10 +38,12 @@ import { type CompressionStats, compress, decompress, + decompressReplayPayload, } from './serialization/compression.js'; import { aesKeyOf, decrypt, + decryptReplayPayload, deriveRunPayloadKeys, type EncryptionKeyParam, encrypt, @@ -3427,8 +3429,8 @@ export interface PreparedReplayPayload { /** * Swappable implementation of the host-side preparation stage. Supporting - * both direct and promised results lets a future synchronous Node decryptor use - * the same cache contract as today's asynchronous Web Crypto implementation. + * both direct and promised results covers synchronous Node AES/zstd as well as + * the portable Web Crypto and sealed-envelope fallbacks. */ export type ReplayPayloadPreparer = ( value: unknown, @@ -3439,18 +3441,49 @@ export type ReplayPayloadPreparer = ( * Decrypt and decompress persisted data without parsing it into JavaScript. * Legacy non-binary values pass through unchanged for their consumer to revive. */ -export const prepareReplayPayload: ReplayPayloadPreparer = async ( - value, - key -) => { +function prepareReplayPayloadWithStats( + value: unknown, + key: PayloadKey | undefined, + compressionStats?: CompressionStats +): PreparedReplayPayload | Promise { + const finish = (prepared: unknown): PreparedReplayPayload => ({ + data: prepared, + }); + const decompressPrepared = ( + decrypted: unknown + ): PreparedReplayPayload | Promise => { + const prepared = decompressReplayPayload(decrypted, compressionStats); + return prepared instanceof Promise + ? prepared.then(finish) + : finish(prepared); + }; + + const decrypted = decryptReplayPayload(value, key); + return decrypted instanceof Promise + ? decrypted.then(decompressPrepared) + : decompressPrepared(decrypted); +} + +// Replay preparation is event-at-a-time and may run inside the response +// decoder. Per-payload compression attributes would add a detached O(N) +// microtask tail and repeatedly overwrite one span, so the replay fast path +// records only its aggregate preparation span. +export const prepareReplayPayload: ReplayPayloadPreparer = (value, key) => + prepareReplayPayloadWithStats(value, key); + +async function prepareReplayPayloadWithTelemetry( + value: unknown, + key: PayloadKey | undefined +): Promise { const compressionStats: CompressionStats = {}; - const prepared = await decompress( - await decrypt(value, key), + const prepared = await prepareReplayPayloadWithStats( + value, + key, compressionStats ); await recordCompression(compressionStats, 'deserialize'); - return { data: prepared }; -}; + return prepared; +} /** * Parse a prepared workflow argument or successful step/hook payload using the @@ -3577,7 +3610,7 @@ export async function hydrateWorkflowArguments( prepared?: PreparedReplayPayload ): Promise { return deserializePreparedReplayPayload( - prepared ?? (await prepareReplayPayload(value, key)), + prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)), global, extraRevivers ); @@ -3874,7 +3907,7 @@ export async function hydrateStepError( prepared?: PreparedReplayPayload ): Promise { return deserializePreparedStepError( - prepared ?? (await prepareReplayPayload(value, key)), + prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)), global, extraRevivers ); @@ -3999,7 +4032,7 @@ export async function hydrateStepReturnValue( prepared?: PreparedReplayPayload ): Promise { return deserializePreparedReplayPayload( - prepared ?? (await prepareReplayPayload(value, key)), + prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)), global, extraRevivers ); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index 2a99303be4..6b4bbb0405 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -309,10 +309,10 @@ export async function compress( * Non-compressed data (including non-binary legacy data) is returned * unchanged, so this is safe to apply unconditionally on read paths. */ -export async function decompress( +export function decompressReplayPayload( data: Uint8Array | unknown, stats?: CompressionStats -): Promise { +): Uint8Array | unknown | Promise { if (!(data instanceof Uint8Array)) return data; const prefix = peekFormatPrefix(data); @@ -332,15 +332,24 @@ export async function decompress( ); } const { payload } = decodeFormatPrefix(data); - const inflated = await gunzipBytes(payload); - recordStats(stats, 'gzip', inflated.length, data.length); - return inflated; + return gunzipBytes(payload).then((inflated) => { + recordStats(stats, 'gzip', inflated.length, data.length); + return inflated; + }); } recordStats(stats, 'none', data.length, data.length); return data; } +/** Portable always-Promise facade retained for existing callers. */ +export async function decompress( + data: Uint8Array | unknown, + stats?: CompressionStats +): Promise { + return decompressReplayPayload(data, stats); +} + /** * Check if data is compressed (has a 'zstd' or 'gzip' format prefix). */ diff --git a/packages/core/src/serialization/encryption.ts b/packages/core/src/serialization/encryption.ts index e3942d0ce2..06988a6edf 100644 --- a/packages/core/src/serialization/encryption.ts +++ b/packages/core/src/serialization/encryption.ts @@ -10,6 +10,7 @@ import { RuntimeDecryptionError } from '@workflow/errors'; import { decrypt as aesGcmDecrypt, + decryptSync as aesGcmDecryptSync, encrypt as aesGcmEncrypt, type CryptoKey, importKey as importAesKey, @@ -320,3 +321,58 @@ export async function decrypt( throw error; } } + +/** + * Replay-specialized decrypt path. + * + * Symmetric payloads use Node's synchronous AES-GCM implementation when the + * runtime owns the imported key material, avoiding one Web Crypto job and + * promise settlement per event. Sealed envelopes and portable runtimes retain + * the asynchronous implementation. The union is intentional: callers that + * care about the common synchronous path can avoid wrapping it in an async + * function, while existing callers may continue to `await` either shape. + */ +export function decryptReplayPayload( + data: Uint8Array | unknown, + key: PayloadKey | undefined +): Uint8Array | unknown | Promise { + if (!(data instanceof Uint8Array)) return data; + + const format = peekFormatPrefix(data); + if (format === SerializationFormat.SEALED) { + return openSealedEnvelope(data, key); + } + if (format !== SerializationFormat.ENCRYPTED) return data; + + const aesKey = aesKeyOf(key); + if (!aesKey) { + throw new RuntimeDecryptionError( + 'Encrypted data encountered but no encryption key is available. ' + + 'Encryption is not configured or no key was provided for this run.', + { + context: { + operation: 'decrypt', + byteLength: data.byteLength, + formatPrefix: 'encr', + }, + } + ); + } + + const { payload } = decodeFormatPrefix(data); + try { + const syncResult = aesGcmDecryptSync(aesKey, payload); + if (syncResult) return syncResult; + return aesGcmDecrypt(aesKey, payload).catch((error) => { + if (RuntimeDecryptionError.is(error) && error.context) { + error.context.formatPrefix = format; + } + throw error; + }); + } catch (error) { + if (RuntimeDecryptionError.is(error) && error.context) { + error.context.formatPrefix = format; + } + throw error; + } +} From c7487857b14c469bd32fb18fbfa324deba4277b6 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:53:38 -0700 Subject: [PATCH 02/16] [core] Simplify synchronous replay decoding --- packages/core/src/encryption.ts | 5 +- .../core/src/serialization/compression.ts | 19 +++---- packages/core/src/serialization/encryption.ts | 51 +++++-------------- 3 files changed, 23 insertions(+), 52 deletions(-) diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index 3841d52da3..e9807cc1fe 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -51,7 +51,7 @@ interface NodeCrypto { } /** Resolve node:crypto without a static import, preserving browser bundles. */ -function getNodeCrypto(): NodeCrypto | undefined { +const nodeCrypto = (() => { try { return ( globalThis as { @@ -61,7 +61,7 @@ function getNodeCrypto(): NodeCrypto | undefined { } catch { return undefined; } -} +})(); /** AES-GCM nonce length in bytes. */ export const NONCE_LENGTH = 12; @@ -125,7 +125,6 @@ export function decryptSync( aad?: Uint8Array ): Uint8Array | undefined { const material = importedKeyMaterial.get(key); - const nodeCrypto = getNodeCrypto(); if (!material || !nodeCrypto) return undefined; if (!key.usages.includes('decrypt')) { throw new RuntimeDecryptionError( diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index 6b4bbb0405..0f59f540c7 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -101,7 +101,7 @@ interface NodeZlib { * this module stays bundler-safe for browser/edge targets (where it returns * undefined and we fall back to gzip). */ -function getNodeZlib(): NodeZlib | undefined { +const nodeZlib = (() => { try { return ( globalThis as { @@ -111,13 +111,12 @@ function getNodeZlib(): NodeZlib | undefined { } catch { return undefined; } -} +})(); function isZstdAvailable(): boolean { - const z = getNodeZlib(); return ( - typeof z?.zstdCompressSync === 'function' && - typeof z?.zstdDecompressSync === 'function' + typeof nodeZlib?.zstdCompressSync === 'function' && + typeof nodeZlib?.zstdDecompressSync === 'function' ); } @@ -177,17 +176,15 @@ async function gunzipBytes(data: Uint8Array): Promise { } function zstdBytes(data: Uint8Array): Uint8Array { - const z = getNodeZlib(); - const level = z?.constants?.ZSTD_c_compressionLevel; + const level = nodeZlib?.constants?.ZSTD_c_compressionLevel; const opts = level !== undefined ? { params: { [level]: ZSTD_LEVEL } } : undefined; // biome-ignore lint/style/noNonNullAssertion: guarded by isZstdAvailable() - return new Uint8Array(z!.zstdCompressSync!(data, opts)); + return new Uint8Array(nodeZlib!.zstdCompressSync!(data, opts)); } function unzstdBytes(data: Uint8Array): Uint8Array { - const z = getNodeZlib(); - if (!z?.zstdDecompressSync) { + if (!nodeZlib?.zstdDecompressSync) { throw new Error( 'Compressed (zstd) workflow data encountered but node:zlib zstd ' + 'support is not available in this runtime (requires Node.js 22.15+). ' + @@ -195,7 +192,7 @@ function unzstdBytes(data: Uint8Array): Uint8Array { '(serialization-format.ts).' ); } - return new Uint8Array(z.zstdDecompressSync(data)); + return new Uint8Array(nodeZlib.zstdDecompressSync(data)); } /** diff --git a/packages/core/src/serialization/encryption.ts b/packages/core/src/serialization/encryption.ts index 06988a6edf..33803e65fd 100644 --- a/packages/core/src/serialization/encryption.ts +++ b/packages/core/src/serialization/encryption.ts @@ -266,12 +266,7 @@ async function openSealedEnvelope( try { return await openSealed(key.keyPair, payload, key.aad); } catch (error) { - // The sealed-box layer only sees the stripped payload, so it cannot - // record the outer envelope prefix. Enrich it here before rethrowing. - if (RuntimeDecryptionError.is(error) && error.context) { - error.context.formatPrefix = SerializationFormat.SEALED; - } - throw error; + throw addFormatPrefix(error, SerializationFormat.SEALED); } } @@ -311,17 +306,17 @@ export async function decrypt( try { return await aesGcmDecrypt(aesKey, payload); } catch (error) { - // The low-level AES layer only sees the stripped payload, so it cannot - // record the outer envelope prefix. This layer peeked it (`encr`), so - // enrich the diagnostic context with the real format prefix before - // rethrowing. - if (RuntimeDecryptionError.is(error) && error.context) { - error.context.formatPrefix = format; - } - throw error; + throw addFormatPrefix(error, format); } } +function addFormatPrefix(error: unknown, format: string): unknown { + if (RuntimeDecryptionError.is(error) && error.context) { + error.context.formatPrefix = format; + } + return error; +} + /** * Replay-specialized decrypt path. * @@ -340,39 +335,19 @@ export function decryptReplayPayload( const format = peekFormatPrefix(data); if (format === SerializationFormat.SEALED) { - return openSealedEnvelope(data, key); + return decrypt(data, key); } if (format !== SerializationFormat.ENCRYPTED) return data; const aesKey = aesKeyOf(key); - if (!aesKey) { - throw new RuntimeDecryptionError( - 'Encrypted data encountered but no encryption key is available. ' + - 'Encryption is not configured or no key was provided for this run.', - { - context: { - operation: 'decrypt', - byteLength: data.byteLength, - formatPrefix: 'encr', - }, - } - ); - } + if (!aesKey) return decrypt(data, key); const { payload } = decodeFormatPrefix(data); try { const syncResult = aesGcmDecryptSync(aesKey, payload); if (syncResult) return syncResult; - return aesGcmDecrypt(aesKey, payload).catch((error) => { - if (RuntimeDecryptionError.is(error) && error.context) { - error.context.formatPrefix = format; - } - throw error; - }); + return decrypt(data, key); } catch (error) { - if (RuntimeDecryptionError.is(error) && error.context) { - error.context.formatPrefix = format; - } - throw error; + throw addFormatPrefix(error, format); } } From 2de8ca585b6fe7850c2085adc6b8ba03abd6591f Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:57:12 -0700 Subject: [PATCH 03/16] [core] Simplify synchronous replay payload APIs --- packages/core/src/encryption.ts | 47 ++++++------- .../core/src/replay-payload-cache.test.ts | 4 +- packages/core/src/replay-payload-cache.ts | 8 ++- packages/core/src/serialization.ts | 33 +++++---- packages/core/src/serialization/client.ts | 8 +-- .../src/serialization/compression.test.ts | 4 +- .../core/src/serialization/compression.ts | 18 ++--- packages/core/src/serialization/encryption.ts | 69 ++++++++++--------- .../src/serialization/serialization.test.ts | 14 ++++ packages/core/src/serialization/step.ts | 8 +-- 10 files changed, 106 insertions(+), 107 deletions(-) diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index e9807cc1fe..da04c0aaff 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -34,30 +34,12 @@ export type CryptoKey = import('node:crypto').webcrypto.CryptoKey; */ const importedKeyMaterial = new WeakMap(); -interface NodeDecipher { - setAAD(data: Uint8Array): NodeDecipher; - setAuthTag(tag: Uint8Array): NodeDecipher; - update(data: Uint8Array): Uint8Array; - final(): Uint8Array; -} - -interface NodeCrypto { - createDecipheriv( - algorithm: string, - key: Uint8Array, - iv: Uint8Array, - options?: { authTagLength?: number } - ): NodeDecipher; -} - /** Resolve node:crypto without a static import, preserving browser bundles. */ const nodeCrypto = (() => { try { - return ( - globalThis as { - process?: { getBuiltinModule?: (id: string) => NodeCrypto }; - } - ).process?.getBuiltinModule?.('node:crypto'); + return typeof process === 'undefined' + ? undefined + : process.getBuiltinModule('node:crypto'); } catch { return undefined; } @@ -115,17 +97,26 @@ export async function importKey( * Decrypt AES-256-GCM synchronously when running on Node and the key was * imported by this module. * - * Returns `undefined` when the portable Web Crypto fallback is required (for - * example in a browser, or for an externally-created CryptoKey). Authentication - * failures throw the same RuntimeDecryptionError shape as {@link decrypt}. + * The key must have been created by this module's {@link importKey}, and the + * Node synchronous crypto API must be available. Authentication failures throw + * the same RuntimeDecryptionError shape as {@link decrypt}. */ export function decryptSync( key: CryptoKey, data: Uint8Array, aad?: Uint8Array -): Uint8Array | undefined { +): Uint8Array { const material = importedKeyMaterial.get(key); - if (!material || !nodeCrypto) return undefined; + if (!material) { + throw new WorkflowRuntimeError( + 'Synchronous AES-256-GCM decryption requires a key created by importKey()' + ); + } + if (!nodeCrypto) { + throw new WorkflowRuntimeError( + 'Synchronous AES-256-GCM decryption requires the Node.js crypto module' + ); + } if (!key.usages.includes('decrypt')) { throw new RuntimeDecryptionError( 'AES-256-GCM decryption failed: CryptoKey does not support decrypt', @@ -163,7 +154,9 @@ export function decryptSync( decipher.setAuthTag(authTag); const head = decipher.update(ciphertext); const tail = decipher.final(); - if (tail.byteLength === 0) return head; + if (tail.byteLength === 0) { + return new Uint8Array(head.buffer, head.byteOffset, head.byteLength); + } const plaintext = new Uint8Array(head.byteLength + tail.byteLength); plaintext.set(head, 0); plaintext.set(tail, head.byteLength); diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 0d3d930619..2eb0858572 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -186,12 +186,12 @@ describe('ReplayPayloadCache', () => { await cache.prepareEventPayload('evnt_legacy', 'result', legacy); await cache.prepareEventPayload('evnt_legacy', 'result', legacy); - expect(preparer).toHaveBeenCalledTimes(2); + expect(preparer).not.toHaveBeenCalled(); const events = makeEvents([legacy, legacy, legacy]); events[2] = { ...events[2], eventData: undefined } as unknown as Event; await cache.prewarm(makeRun(legacy), events); - expect(preparer).toHaveBeenCalledTimes(2); + expect(preparer).not.toHaveBeenCalled(); }); it('memoizes primitive step results, including undefined', async () => { diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 31144a6add..6c8150cbae 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -173,7 +173,9 @@ export class ReplayPayloadCache { cacheKey: string, value: unknown ): Promise { - if (!(value instanceof Uint8Array)) return this.runPreparation(value); + if (!(value instanceof Uint8Array)) { + return Promise.resolve({ data: value }); + } const preparation = this.ensurePreparation(cacheKey, value); void preparation.catch(() => { @@ -198,7 +200,9 @@ export class ReplayPayloadCache { } /** Normalize synchronous and asynchronous preparers to one promise contract. */ - private async runPreparation(value: unknown): Promise { + private async runPreparation( + value: Uint8Array + ): Promise { return this.preparer(value, this.encryptionKey); } diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index 684180edb8..a38d034745 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -38,7 +38,6 @@ import { type CompressionStats, compress, decompress, - decompressReplayPayload, } from './serialization/compression.js'; import { aesKeyOf, @@ -3433,35 +3432,35 @@ export interface PreparedReplayPayload { * the portable Web Crypto and sealed-envelope fallbacks. */ export type ReplayPayloadPreparer = ( - value: unknown, + value: Uint8Array, key: PayloadKey | undefined ) => PreparedReplayPayload | Promise; /** * Decrypt and decompress persisted data without parsing it into JavaScript. - * Legacy non-binary values pass through unchanged for their consumer to revive. + * Legacy non-binary values are handled before this binary preparation boundary. */ function prepareReplayPayloadWithStats( - value: unknown, + value: Uint8Array, key: PayloadKey | undefined, compressionStats?: CompressionStats ): PreparedReplayPayload | Promise { - const finish = (prepared: unknown): PreparedReplayPayload => ({ + const finish = (prepared: Uint8Array): PreparedReplayPayload => ({ data: prepared, }); const decompressPrepared = ( - decrypted: unknown + decrypted: Uint8Array ): PreparedReplayPayload | Promise => { - const prepared = decompressReplayPayload(decrypted, compressionStats); + const prepared = decompress(decrypted, compressionStats); return prepared instanceof Promise ? prepared.then(finish) : finish(prepared); }; - const decrypted = decryptReplayPayload(value, key); - return decrypted instanceof Promise - ? decrypted.then(decompressPrepared) - : decompressPrepared(decrypted); + if (peekFormatPrefix(value) === SerializationFormat.SEALED) { + return decrypt(value, key).then(decompressPrepared); + } + return decompressPrepared(decryptReplayPayload(value, key)); } // Replay preparation is event-at-a-time and may run inside the response @@ -3475,6 +3474,8 @@ async function prepareReplayPayloadWithTelemetry( value: unknown, key: PayloadKey | undefined ): Promise { + if (!(value instanceof Uint8Array)) return { data: value }; + const compressionStats: CompressionStats = {}; const prepared = await prepareReplayPayloadWithStats( value, @@ -3979,11 +3980,7 @@ export async function hydrateRunError( extraRevivers: Record any> = {} ): Promise { const compressionStats: CompressionStats = {}; - const decrypted = await decompress( - await decrypt(value, key), - compressionStats - ); - await recordCompression(compressionStats, 'deserialize'); + const decrypted = await decrypt(value, key); if (!(decrypted instanceof Uint8Array)) { // See the matching note in `hydrateStepError`: this branch is for @@ -3997,7 +3994,9 @@ export async function hydrateRunError( }); } - const { format, payload } = decodeFormatPrefix(decrypted); + const prepared = await decompress(decrypted, compressionStats); + await recordCompression(compressionStats, 'deserialize'); + const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { const str = new TextDecoder().decode(payload); diff --git a/packages/core/src/serialization/client.ts b/packages/core/src/serialization/client.ts index 5d9b0e6d78..1014bf92a8 100644 --- a/packages/core/src/serialization/client.ts +++ b/packages/core/src/serialization/client.ts @@ -54,10 +54,7 @@ export async function deserialize( encryptionKey?: PayloadKey, options?: CodecOptions ): Promise { - const decrypted = await decompress( - await decryptData(data, encryptionKey), - options?.compressionStats - ); + const decrypted = await decryptData(data, encryptionKey); if (!(decrypted instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { @@ -68,7 +65,8 @@ export async function deserialize( ); } - const { format, payload } = decodeFormatPrefix(decrypted); + const prepared = await decompress(decrypted, options?.compressionStats); + const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { return devalueCodec.deserialize(payload, 'client', options); diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts index 355183b7ef..9e26cf2d80 100644 --- a/packages/core/src/serialization/compression.test.ts +++ b/packages/core/src/serialization/compression.test.ts @@ -99,11 +99,9 @@ describe('compression layer (compress/decompress)', () => { expect(result).toBe(original); }); - it('decompress passes non-compressed and non-binary data through', async () => { + it('decompress passes non-compressed bytes through', async () => { const plain = textEncoder.encode('devl"hello"'); expect(await decompress(plain)).toBe(plain); - const legacy = [1, 2, 3]; - expect(await decompress(legacy)).toBe(legacy); }); }); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index 0f59f540c7..c6981969e2 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -303,14 +303,12 @@ export async function compress( * Dispatches on the prefix ('zstd' or 'gzip') and inflates the inner * payload (which carries its own format prefix, e.g. 'devl'). * - * Non-compressed data (including non-binary legacy data) is returned - * unchanged, so this is safe to apply unconditionally on read paths. + * Non-compressed data is returned unchanged. */ -export function decompressReplayPayload( - data: Uint8Array | unknown, +export function decompress( + data: Uint8Array, stats?: CompressionStats -): Uint8Array | unknown | Promise { - if (!(data instanceof Uint8Array)) return data; +): Uint8Array | Promise { const prefix = peekFormatPrefix(data); if (prefix === SerializationFormat.ZSTD) { @@ -339,14 +337,6 @@ export function decompressReplayPayload( return data; } -/** Portable always-Promise facade retained for existing callers. */ -export async function decompress( - data: Uint8Array | unknown, - stats?: CompressionStats -): Promise { - return decompressReplayPayload(data, stats); -} - /** * Check if data is compressed (has a 'zstd' or 'gzip' format prefix). */ diff --git a/packages/core/src/serialization/encryption.ts b/packages/core/src/serialization/encryption.ts index 33803e65fd..fcf36f9427 100644 --- a/packages/core/src/serialization/encryption.ts +++ b/packages/core/src/serialization/encryption.ts @@ -270,6 +270,34 @@ async function openSealedEnvelope( } } +function requireAesDecryptionKey( + data: Uint8Array, + key: PayloadKey | undefined +): CryptoKey { + const aesKey = aesKeyOf(key); + if (aesKey) return aesKey; + + throw new RuntimeDecryptionError( + 'Encrypted data encountered but no encryption key is available. ' + + 'Encryption is not configured or no key was provided for this run.', + { + context: { + operation: 'decrypt', + byteLength: data.byteLength, + formatPrefix: 'encr', + }, + } + ); +} + +export function decrypt( + data: Uint8Array, + key: PayloadKey | undefined +): Promise; +export function decrypt( + data: unknown, + key: PayloadKey | undefined +): Promise; export async function decrypt( data: Uint8Array | unknown, key: PayloadKey | undefined @@ -286,21 +314,7 @@ export async function decrypt( // If the data is not encrypted, return it unchanged. if (format !== SerializationFormat.ENCRYPTED) return data; - // If the data is encrypted but no symmetric key is available, fail fast. - const aesKey = aesKeyOf(key); - if (!aesKey) { - throw new RuntimeDecryptionError( - 'Encrypted data encountered but no encryption key is available. ' + - 'Encryption is not configured or no key was provided for this run.', - { - context: { - operation: 'decrypt', - byteLength: data.byteLength, - formatPrefix: 'encr', - }, - } - ); - } + const aesKey = requireAesDecryptionKey(data, key); const { payload } = decodeFormatPrefix(data); try { @@ -320,33 +334,24 @@ function addFormatPrefix(error: unknown, format: string): unknown { /** * Replay-specialized decrypt path. * - * Symmetric payloads use Node's synchronous AES-GCM implementation when the - * runtime owns the imported key material, avoiding one Web Crypto job and - * promise settlement per event. Sealed envelopes and portable runtimes retain - * the asynchronous implementation. The union is intentional: callers that - * care about the common synchronous path can avoid wrapping it in an async - * function, while existing callers may continue to `await` either shape. + * This byte-only function handles plaintext and symmetric envelopes without a + * promise. The replay preparation boundary dispatches sealed envelopes to the + * asynchronous X25519 decryptor before calling this function. */ export function decryptReplayPayload( - data: Uint8Array | unknown, + data: Uint8Array, key: PayloadKey | undefined -): Uint8Array | unknown | Promise { - if (!(data instanceof Uint8Array)) return data; - +): Uint8Array { const format = peekFormatPrefix(data); if (format === SerializationFormat.SEALED) { - return decrypt(data, key); + throw new Error('Sealed replay payloads require asynchronous decryption'); } if (format !== SerializationFormat.ENCRYPTED) return data; - const aesKey = aesKeyOf(key); - if (!aesKey) return decrypt(data, key); - + const aesKey = requireAesDecryptionKey(data, key); const { payload } = decodeFormatPrefix(data); try { - const syncResult = aesGcmDecryptSync(aesKey, payload); - if (syncResult) return syncResult; - return decrypt(data, key); + return aesGcmDecryptSync(aesKey, payload); } catch (error) { throw addFormatPrefix(error, format); } diff --git a/packages/core/src/serialization/serialization.test.ts b/packages/core/src/serialization/serialization.test.ts index 4a9a07afbb..856d1e17c4 100644 --- a/packages/core/src/serialization/serialization.test.ts +++ b/packages/core/src/serialization/serialization.test.ts @@ -9,6 +9,7 @@ import { devalueCodec } from './codec-devalue.js'; import { aesKeyOf, decrypt, + decryptReplayPayload, encrypt, isRunPayloadKeys, isSealTarget, @@ -307,6 +308,19 @@ describe('encrypt', () => { }); describe('decrypt', () => { + it('decrypts replay payloads synchronously', async () => { + const key = await makeKey(); + const data = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + new Uint8Array([1, 2, 3]) + ) as Uint8Array; + const encrypted = (await encrypt(data, key)) as Uint8Array; + + const decrypted = decryptReplayPayload(encrypted, key); + expect(decrypted).toBeInstanceOf(Uint8Array); + expect(decrypted).toEqual(data); + }); + it('should return non-binary data unchanged', async () => { const data = [1, 2, 3]; const result = await decrypt(data, undefined); diff --git a/packages/core/src/serialization/step.ts b/packages/core/src/serialization/step.ts index 7e8c8730d7..08f5cbfabc 100644 --- a/packages/core/src/serialization/step.ts +++ b/packages/core/src/serialization/step.ts @@ -54,10 +54,7 @@ export async function deserialize( encryptionKey?: PayloadKey, options?: CodecOptions ): Promise { - const decrypted = await decompress( - await decryptData(data, encryptionKey), - options?.compressionStats - ); + const decrypted = await decryptData(data, encryptionKey); if (!(decrypted instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { @@ -68,7 +65,8 @@ export async function deserialize( ); } - const { format, payload } = decodeFormatPrefix(decrypted); + const prepared = await decompress(decrypted, options?.compressionStats); + const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { return devalueCodec.deserialize(payload, 'step', options); From e9a16a556ad1d8f5c5f94ba5ef7379c8412a3c3e Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:32:05 -0700 Subject: [PATCH 04/16] [core] Deepen payload codec boundaries --- .../decode-replay-payloads-synchronously.md | 3 +- packages/cli/src/lib/inspect/hydration.ts | 23 ++-- packages/core/src/encryption.ts | 114 ++++++---------- packages/core/src/private.ts | 4 +- packages/core/src/replay-payload-cache.ts | 4 +- packages/core/src/runtime/helpers.ts | 6 +- .../core/src/runtime/quickjs-entrypoint.ts | 16 ++- packages/core/src/runtime/run.ts | 8 +- packages/core/src/runtime/step-executor.ts | 4 +- .../core/src/serialization-format.test.ts | 12 +- packages/core/src/serialization-format.ts | 126 ++++-------------- packages/core/src/serialization.test.ts | 64 ++------- packages/core/src/serialization.ts | 114 +++++----------- packages/core/src/serialization/client.ts | 16 +-- .../src/serialization/compression.test.ts | 10 +- .../core/src/serialization/compression.ts | 77 ++++++----- packages/core/src/serialization/encryption.ts | 29 ++-- packages/core/src/serialization/format.ts | 31 ++--- packages/core/src/serialization/index.ts | 1 + .../src/serialization/serialization.test.ts | 51 +------ packages/core/src/serialization/step.ts | 16 +-- packages/core/src/serialization/types.ts | 16 ++- .../core/src/serialization/workflow-vm.ts | 2 +- packages/core/src/serialization/workflow.ts | 10 +- packages/core/src/step/context-storage.ts | 4 +- packages/core/src/workflow.ts | 8 +- 26 files changed, 265 insertions(+), 504 deletions(-) diff --git a/.changeset/decode-replay-payloads-synchronously.md b/.changeset/decode-replay-payloads-synchronously.md index b4faf3824b..54d0633684 100644 --- a/.changeset/decode-replay-payloads-synchronously.md +++ b/.changeset/decode-replay-payloads-synchronously.md @@ -1,5 +1,6 @@ --- "@workflow/core": patch +"@workflow/cli": patch --- -Decode replay payloads synchronously with Node AES-GCM and zstd when the runtime supports it. +Simplify payload codecs and decode replay payloads synchronously with Node AES-GCM and zstd. diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index eb043aa7d2..20b35adf37 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -6,7 +6,7 @@ */ import { inspect } from 'node:util'; -import { getCommonRevivers, maybeDecrypt } from '@workflow/core/serialization'; +import { decrypt, getCommonRevivers } from '@workflow/core/serialization'; import { ClassInstanceRef, extractClassName, @@ -31,6 +31,13 @@ export type EncryptionKeyResolver = | ((runId: string) => Promise) | null; +async function decryptPayload( + value: unknown, + key: Parameters[1] +): Promise { + return value instanceof Uint8Array ? decrypt(value, key) : value; +} + // Re-export types and utilities that consumers need export { CLASS_INSTANCE_REF_TYPE, @@ -313,8 +320,8 @@ function getRevivers(): Revivers { * Pre-process a resource's data fields: if the resolver is provided and * the field is encrypted, decrypt it before generic hydration. * - * Uses core's `maybeDecrypt()` which handles the 'encr' prefix stripping - * and AES-GCM decryption transparently. + * Binary envelopes go through the core decryptor; legacy values remain + * unchanged for the generic hydrator. * * When the resolver is null (no --decrypt flag), encrypted fields pass * through as Uint8Array and are replaced with EncryptedDataRef in post-processing. @@ -347,18 +354,18 @@ async function maybeDecryptFields< const k = rawKey ? await deriveRunPayloadKeys(rawKey) : undefined; // Decrypt input/output/error fields (WorkflowRun, Step) - result.input = await maybeDecrypt(result.input, k); - result.output = await maybeDecrypt(result.output, k); - (result as any).error = await maybeDecrypt((result as any).error, k); + result.input = await decryptPayload(result.input, k); + result.output = await decryptPayload(result.output, k); + (result as any).error = await decryptPayload((result as any).error, k); // Decrypt metadata field (Hook) - result.metadata = await maybeDecrypt(result.metadata, k); + result.metadata = await decryptPayload(result.metadata, k); // Decrypt eventData fields (Event) if (result.eventData && typeof result.eventData === 'object') { const eventData = { ...result.eventData }; for (const field of getEventDataRefFields(result.eventType ?? '')) { - eventData[field] = await maybeDecrypt(eventData[field], k); + eventData[field] = await decryptPayload(eventData[field], k); } result.eventData = eventData; } diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index da04c0aaff..65f15826d9 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -13,9 +13,8 @@ import { RuntimeDecryptionError, WorkflowRuntimeError } from '@workflow/errors'; * operations within the same run. This avoids repeated `importKey()` * calls on every encrypt/decrypt invocation. * - * Wire format: `[nonce (12 bytes)][ciphertext + auth tag]` - * The `encr` format prefix is NOT part of this module — it's added/stripped - * by the serialization layer in `maybeEncrypt`/`maybeDecrypt`. + * Wire format: `[nonce (12 bytes)][ciphertext + auth tag]`. The serialization + * encryption module owns the outer `encr` format prefix. */ // CryptoKey is a global type in browsers and Node.js 20+, but TypeScript's @@ -24,15 +23,14 @@ import { RuntimeDecryptionError, WorkflowRuntimeError } from '@workflow/errors'; export type CryptoKey = import('node:crypto').webcrypto.CryptoKey; /** - * Raw key material retained alongside keys imported by this module. + * Node key handles retained alongside keys imported by this module. * - * Node's synchronous cipher API cannot consume a Web Crypto `CryptoKey`, and - * our keys are deliberately non-extractable. Keeping the original bytes in a - * WeakMap gives the Node replay path access to the same key without making it - * extractable or extending its lifetime beyond the `CryptoKey`. Browser/edge - * callers continue to use Web Crypto and never consult this map. + * Node's synchronous cipher API needs a native key handle. Creating it while + * the raw bytes are already available avoids trying to convert a deliberately + * non-extractable `CryptoKey` later. Browser/edge callers never populate or + * consult this map. */ -const importedKeyMaterial = new WeakMap(); +const nodeKeys = new WeakMap(); /** Resolve node:crypto without a static import, preserving browser bundles. */ const nodeCrypto = (() => { @@ -87,12 +85,34 @@ export async function importKey( // a strict subset of `KeyUsage[]`, so this cast is sound. usages as ('encrypt' | 'decrypt')[] ); - // Copy the caller's bytes: a caller may reuse/mutate its input buffer after - // importKey(), while a CryptoKey's material is immutable. - importedKeyMaterial.set(key, raw.slice()); + if (nodeCrypto) { + nodeKeys.set(key, nodeCrypto.createSecretKey(raw)); + } return key; } +function assertValidAesGcmEnvelope(data: Uint8Array): void { + const minLength = NONCE_LENGTH + TAG_BYTES; + if (data.byteLength < minLength) { + throw new RuntimeDecryptionError( + `Encrypted data too short: expected at least ${minLength} bytes, got ${data.byteLength}`, + { + context: { operation: 'decrypt', byteLength: data.byteLength }, + } + ); + } +} + +function wrapDecryptionError(cause: unknown, byteLength: number): never { + throw new RuntimeDecryptionError( + `AES-256-GCM decryption failed: ${cause instanceof Error ? cause.message : String(cause)}`, + { + cause, + context: { operation: 'decrypt', byteLength }, + } + ); +} + /** * Decrypt AES-256-GCM synchronously when running on Node and the key was * imported by this module. @@ -106,8 +126,8 @@ export function decryptSync( data: Uint8Array, aad?: Uint8Array ): Uint8Array { - const material = importedKeyMaterial.get(key); - if (!material) { + const nodeKey = nodeKeys.get(key); + if (!nodeKey) { throw new WorkflowRuntimeError( 'Synchronous AES-256-GCM decryption requires a key created by importKey()' ); @@ -126,27 +146,15 @@ export function decryptSync( ); } - const minLength = NONCE_LENGTH + TAG_BYTES; - if (data.byteLength < minLength) { - throw new RuntimeDecryptionError( - `Encrypted data too short: expected at least ${minLength} bytes, got ${data.byteLength}`, - { - context: { - operation: 'decrypt', - byteLength: data.byteLength, - }, - } - ); - } - - const nonce = data.subarray(0, NONCE_LENGTH); + assertValidAesGcmEnvelope(data); const ciphertextEnd = data.byteLength - TAG_BYTES; + const nonce = data.subarray(0, NONCE_LENGTH); const ciphertext = data.subarray(NONCE_LENGTH, ciphertextEnd); const authTag = data.subarray(ciphertextEnd); try { const decipher = nodeCrypto.createDecipheriv( 'aes-256-gcm', - material, + nodeKey, nonce, { authTagLength: TAG_BYTES } ); @@ -162,16 +170,7 @@ export function decryptSync( plaintext.set(tail, head.byteLength); return plaintext; } catch (cause) { - throw new RuntimeDecryptionError( - `AES-256-GCM decryption failed: ${cause instanceof Error ? cause.message : String(cause)}`, - { - cause, - context: { - operation: 'decrypt', - byteLength: data.byteLength, - }, - } - ); + wrapDecryptionError(cause, data.byteLength); } } @@ -255,20 +254,9 @@ export async function decrypt( data: Uint8Array, aad?: Uint8Array ): Promise { - const minLength = NONCE_LENGTH + TAG_LENGTH / 8; // nonce + auth tag - if (data.byteLength < minLength) { - throw new RuntimeDecryptionError( - `Encrypted data too short: expected at least ${minLength} bytes, got ${data.byteLength}`, - { - context: { - operation: 'decrypt', - byteLength: data.byteLength, - }, - } - ); - } + assertValidAesGcmEnvelope(data); const nonce = data.subarray(0, NONCE_LENGTH); - const ciphertext = data.subarray(NONCE_LENGTH); + const ciphertextAndTag = data.subarray(NONCE_LENGTH); let plaintext: ArrayBuffer; try { plaintext = await globalThis.crypto.subtle.decrypt( @@ -279,26 +267,10 @@ export async function decrypt( ...(aad ? { additionalData: aad } : {}), }, key, - ciphertext + ciphertextAndTag ); } catch (cause) { - // The most common shape we see in the wild is a DOMException with - // `name: 'OperationError'` and message "The operation failed for - // an operation-specific reason" — this is what Web Crypto throws - // when the GCM auth tag does not verify. Re-throw as - // RuntimeDecryptionError, attaching diagnostic context (byte length) - // that the bare DOMException lacks. - const causeMsg = cause instanceof Error ? cause.message : String(cause); - throw new RuntimeDecryptionError( - `AES-256-GCM decryption failed: ${causeMsg}`, - { - cause, - context: { - operation: 'decrypt', - byteLength: data.byteLength, - }, - } - ); + wrapDecryptionError(cause, data.byteLength); } return new Uint8Array(plaintext); } diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index fccc2b3996..2df0d8eaba 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -8,7 +8,7 @@ import type { EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import type { ReplayPayloadCache } from './replay-payload-cache.js'; import type { Serializable } from './schemas.js'; -import type { PayloadKey } from './serialization/encryption.js'; +import type { DecryptionKey } from './serialization/encryption.js'; export type StepFunction< Args extends Serializable[] = any[], @@ -132,7 +132,7 @@ export function getStepFunction(stepId: string): StepFunction | undefined { export interface WorkflowOrchestratorContext { runId: string; - encryptionKey: PayloadKey | undefined; + encryptionKey: DecryptionKey | undefined; worldCapabilities?: WorldCapabilities; globalThis: typeof globalThis; /** diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 6c8150cbae..8f9cfac990 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -1,5 +1,5 @@ import type { Event, WorkflowRun } from '@workflow/world'; -import type { PayloadKey } from './serialization/encryption.js'; +import type { DecryptionKey } from './serialization/encryption.js'; import { type PreparedReplayPayload, prepareReplayPayload, @@ -43,7 +43,7 @@ export class ReplayPayloadCache { private nextUnscannedEventIndex = 0; constructor( - private readonly encryptionKey: PayloadKey | undefined, + private readonly encryptionKey: DecryptionKey | undefined, private readonly preparer: ReplayPayloadPreparer = prepareReplayPayload ) {} diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 03c08f88c3..f24944478f 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -27,8 +27,8 @@ import { monotonicFactory } from 'ulid'; import { runtimeLogger } from '../logger.js'; import { bytesToBase64, deriveRunKeyPair } from '../sealed-box.js'; import { + type DecryptionKey, deriveRunPayloadKeys, - type PayloadKey, } from '../serialization/encryption.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { getSpanKind, trace } from '../telemetry.js'; @@ -1247,8 +1247,8 @@ export function getQueueOverhead(message: { requestedAt?: Date }) { export function memoizeEncryptionKey( world: World, runOrId: WorkflowRun | string -): () => Promise { - let cached: Promise | undefined; +): () => Promise { + let cached: Promise | undefined; return () => { if (!cached) { cached = (async () => { diff --git a/packages/core/src/runtime/quickjs-entrypoint.ts b/packages/core/src/runtime/quickjs-entrypoint.ts index bd81fb6981..f3dead89cb 100644 --- a/packages/core/src/runtime/quickjs-entrypoint.ts +++ b/packages/core/src/runtime/quickjs-entrypoint.ts @@ -37,11 +37,7 @@ import { encrypt as encryptSerializedData, type RunPayloadKeys, } from '../serialization/encryption.js'; -import { - dehydrateRunError, - hydrateRunError, - maybeEncrypt, -} from '../serialization.js'; +import { dehydrateRunError, hydrateRunError } from '../serialization.js'; import { remapErrorStack, stripInlineSourceMap } from '../source-map.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { serializeTraceCarrier } from '../telemetry.js'; @@ -320,6 +316,12 @@ async function dispatchPendingOps(params: { // EntityConflictError, which we swallow below. This drops one // network round-trip per pending hook. try { + if ( + hook.metadata !== undefined && + !(hook.metadata instanceof Uint8Array) + ) { + throw new TypeError('QuickJS hook metadata must be binary'); + } const encryptedMetadata = typeof hook.metadata === 'undefined' ? undefined @@ -1899,10 +1901,10 @@ export async function runWorkflowWithQuickJS(params: { message: (rehydrateErr as Error)?.message, } ); - dehydratedError = (await maybeEncrypt( + dehydratedError = await encryptSerializedData( result.failed.valueBytes, encryptionKey - )) as Uint8Array; + ); } } else { if (errorStack) { diff --git a/packages/core/src/runtime/run.ts b/packages/core/src/runtime/run.ts index 1663df9b8d..201eb5a4f8 100644 --- a/packages/core/src/runtime/run.ts +++ b/packages/core/src/runtime/run.ts @@ -12,8 +12,8 @@ import { type World, } from '@workflow/world'; import { + type DecryptionKey, deriveRunPayloadKeys, - type PayloadKey, } from '../serialization/encryption.js'; import { getExternalRevivers, @@ -118,7 +118,7 @@ export class Run { * reused for returnValue, getReadable(), etc. * @internal */ - #encryptionKeyPromise: Promise | null = null; + #encryptionKeyPromise: Promise | null = null; /** * When true, run_created failed and the run may not exist yet (the @@ -140,7 +140,7 @@ export class Run { * to be resolved once. * @internal */ - #getEncryptionKey(): Promise { + #getEncryptionKey(): Promise { if (!this.#encryptionKeyPromise) { this.#encryptionKeyPromise = (async () => { const world = await this.#lazyWorldPromise; @@ -158,7 +158,7 @@ export class Run { * unobserved run lookup. * @internal */ - #getEncryptionKeyLazily(): () => Promise { + #getEncryptionKeyLazily(): () => Promise { return () => this.#getEncryptionKey(); } diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 4b2ccc7c07..93959c5aae 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -29,7 +29,7 @@ import { } from '@workflow/world'; import { runtimeLogger, stepLogger } from '../logger.js'; import { getStepFunction } from '../private.js'; -import type { PayloadKey } from '../serialization/encryption.js'; +import type { DecryptionKey } from '../serialization/encryption.js'; import { cancelAbortReaders, dehydrateStepError, @@ -103,7 +103,7 @@ export interface StepExecutorParams { rootRunId?: string; stepId: string; stepName: string; - encryptionKey?: PayloadKey; + encryptionKey?: DecryptionKey; /** * The workflow run's specVersion, used to gate payload compression. * Step outputs/errors are only compressed when the run is marked as diff --git a/packages/core/src/serialization-format.test.ts b/packages/core/src/serialization-format.test.ts index 5a19584a8f..bd52868167 100644 --- a/packages/core/src/serialization-format.test.ts +++ b/packages/core/src/serialization-format.test.ts @@ -62,7 +62,7 @@ describe('encodeWithFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded).toBeInstanceOf(Uint8Array); expect(encoded.length).toBe(4 + 3); // "devl" (4 bytes) + payload (3 bytes) @@ -71,14 +71,6 @@ describe('encodeWithFormatPrefix', () => { expect(prefix).toBe('devl'); expect(Array.from(encoded.subarray(4))).toEqual([1, 2, 3]); }); - - it('should pass through non-Uint8Array values', () => { - const result = encodeWithFormatPrefix( - SerializationFormat.DEVALUE_V1, - 'hello' - ); - expect(result).toBe('hello'); - }); }); describe('decodeFormatPrefix', () => { @@ -87,7 +79,7 @@ describe('decodeFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); const { format, payload: decoded } = decodeFormatPrefix(encoded); expect(format).toBe('devl'); diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index 831caaae6d..06848c8de1 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -8,6 +8,12 @@ import { getEventDataRefFields } from '@workflow/world'; import { parse, unflatten } from 'devalue'; +import { + decodeFormatPrefix as decodePrefix, + encodeWithFormatPrefix, + peekFormatPrefix, +} from './serialization/format.js'; +import { SerializationFormat } from './serialization/types.js'; // --------------------------------------------------------------------------- // Key material (browser-safe re-exports) @@ -40,93 +46,29 @@ export { // Format prefix constants and encoding/decoding // --------------------------------------------------------------------------- -export const SerializationFormat = { - /** devalue stringify/parse with TextEncoder/TextDecoder */ - DEVALUE_V1: 'devl', - /** Encrypted payload (inner payload has its own format prefix after decryption) */ - ENCRYPTED: 'encr', - /** - * Sealed payload — asymmetrically encrypted to a run's X25519 public key - * (inner payload has its own format prefix after opening). - * - * Written by cross-run writers that hold only the recipient run's public - * key. Opening it requires the run's private scalar rather than the - * symmetric per-run key, so o11y display treats it as ciphertext via - * {@link isEncryptedData} but {@link hydrateDataWithKey} does not attempt - * an AES-GCM decrypt on it. - */ - SEALED: 'encp', - /** Gzip-compressed payload (inner payload has its own format prefix after decompression) */ - GZIP: 'gzip', - /** Zstandard-compressed payload (inner payload has its own format prefix after decompression) */ - ZSTD: 'zstd', -} as const; +export { encodeWithFormatPrefix, SerializationFormat }; export type SerializationFormatType = (typeof SerializationFormat)[keyof typeof SerializationFormat]; -/** Length of the format prefix in bytes */ -const FORMAT_PREFIX_LENGTH = 4; - -const formatEncoder = new TextEncoder(); -const formatDecoder = new TextDecoder(); - -/** - * Encode a payload with a format prefix. - */ -export function encodeWithFormatPrefix( - format: SerializationFormatType, - payload: Uint8Array | unknown -): Uint8Array | unknown { - if (!(payload instanceof Uint8Array)) { - return payload; - } - - const prefixBytes = formatEncoder.encode(format); - if (prefixBytes.length !== FORMAT_PREFIX_LENGTH) { - throw new Error( - `Format identifier must be exactly ${FORMAT_PREFIX_LENGTH} ASCII characters, got "${format}" (${prefixBytes.length} bytes)` - ); - } - - const result = new Uint8Array(FORMAT_PREFIX_LENGTH + payload.length); - result.set(prefixBytes, 0); - result.set(payload, FORMAT_PREFIX_LENGTH); - return result; -} - /** * Decode a format-prefixed payload. */ -export function decodeFormatPrefix(data: Uint8Array | unknown): { +export function decodeFormatPrefix(data: Uint8Array): { format: SerializationFormatType; payload: Uint8Array; } { - if (!(data instanceof Uint8Array)) { - return { - format: SerializationFormat.DEVALUE_V1, - payload: new TextEncoder().encode(JSON.stringify(data)), - }; - } - - if (data.length < FORMAT_PREFIX_LENGTH) { - throw new Error( - `Data too short to contain format prefix: expected at least ${FORMAT_PREFIX_LENGTH} bytes, got ${data.length}` - ); - } - - const prefixBytes = data.subarray(0, FORMAT_PREFIX_LENGTH); - const format = formatDecoder.decode(prefixBytes); - - const knownFormats = Object.values(SerializationFormat) as string[]; - if (!knownFormats.includes(format)) { + const decoded = decodePrefix(data); + const knownFormats = Object.values(SerializationFormat); + if (!knownFormats.includes(decoded.format as SerializationFormatType)) { throw new Error( - `Unknown serialization format: "${format}". Known formats: ${knownFormats.join(', ')}` + `Unknown serialization format: "${decoded.format}". Known formats: ${knownFormats.join(', ')}` ); } - - const payload = data.subarray(FORMAT_PREFIX_LENGTH); - return { format: format as SerializationFormatType, payload }; + return decoded as { + format: SerializationFormatType; + payload: Uint8Array; + }; } // --------------------------------------------------------------------------- @@ -190,10 +132,8 @@ export function isExpiredStub(data: unknown): boolean { * Browser-safe — does not depend on the full serialization module. */ export function isEncryptedData(data: unknown): boolean { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { - return false; - } - const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); + if (!(data instanceof Uint8Array)) return false; + const prefix = peekFormatPrefix(data); return ( prefix === SerializationFormat.ENCRYPTED || prefix === SerializationFormat.SEALED @@ -207,11 +147,10 @@ export function isEncryptedData(data: unknown): boolean { * Browser-safe — does not depend on the full serialization module. */ export function isSealedData(data: unknown): boolean { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { - return false; - } - const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); - return prefix === SerializationFormat.SEALED; + return ( + data instanceof Uint8Array && + peekFormatPrefix(data) === SerializationFormat.SEALED + ); } /** @@ -219,31 +158,22 @@ export function isSealedData(data: unknown): boolean { * Browser-safe — does not depend on the full serialization module. */ export function isCompressedData(data: unknown): boolean { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { - return false; - } - const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); + if (!(data instanceof Uint8Array)) return false; + const prefix = peekFormatPrefix(data); return ( prefix === SerializationFormat.GZIP || prefix === SerializationFormat.ZSTD ); } -interface NodeZlibDecode { - gunzipSync?: (data: Uint8Array) => Uint8Array; - zstdDecompressSync?: (data: Uint8Array) => Uint8Array; -} - /** * Resolve `node:zlib` via `process.getBuiltinModule` — no static Node * dependency, invisible to browser bundlers. Returns undefined off Node. */ -function getNodeZlib(): NodeZlibDecode | undefined { +function getNodeZlib() { try { - return ( - globalThis as { - process?: { getBuiltinModule?: (id: string) => NodeZlibDecode }; - } - ).process?.getBuiltinModule?.('node:zlib'); + return typeof process === 'undefined' + ? undefined + : process.getBuiltinModule('node:zlib'); } catch { return undefined; } diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index 6a9cbdfc23..aab4a45554 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -14,6 +14,7 @@ import { getStepFunction, registerStepFunction } from './private.js'; import { bytesToBase64, deriveRunKeyPair } from './sealed-box.js'; import { decrypt as decryptEnvelope, + encrypt as encryptEnvelope, runPayloadKeys, sealTo, } from './serialization/encryption.js'; @@ -38,8 +39,6 @@ import { hydrateWorkflowArguments, hydrateWorkflowReturnValue, isEncrypted, - maybeDecrypt, - maybeEncrypt, SerializationFormat, } from './serialization.js'; import { hydrateData } from './serialization-format.js'; @@ -4892,42 +4891,7 @@ describe('format prefix system', () => { }); }); -describe('decodeFormatPrefix legacy compatibility', () => { - it('should handle legacy object data (non-Uint8Array)', () => { - const legacyData = { message: 'hello', count: 42 }; - const result = decodeFormatPrefix(legacyData); - - expect(result.format).toBe(SerializationFormat.DEVALUE_V1); - expect(result.payload).toBeInstanceOf(Uint8Array); - - // The payload should be JSON-encoded - const decoded = new TextDecoder().decode(result.payload); - expect(JSON.parse(decoded)).toEqual(legacyData); - }); - - it('should handle legacy array data (non-Uint8Array)', () => { - const legacyData = [1, 2, 'three', { nested: true }]; - const result = decodeFormatPrefix(legacyData); - - expect(result.format).toBe(SerializationFormat.DEVALUE_V1); - expect(result.payload).toBeInstanceOf(Uint8Array); - - const decoded = new TextDecoder().decode(result.payload); - expect(JSON.parse(decoded)).toEqual(legacyData); - }); - - it('should handle legacy undefined data (non-Uint8Array)', () => { - const legacyData = undefined; - const result = decodeFormatPrefix(legacyData); - - expect(result.format).toBe(SerializationFormat.DEVALUE_V1); - expect(result.payload).toBeInstanceOf(Uint8Array); - - // JSON.stringify(undefined) returns undefined (not a string), - // which when encoded produces an empty Uint8Array - expect(result.payload.length).toBe(0); - }); - +describe('decodeFormatPrefix', () => { it('should still correctly handle v2 Uint8Array data', () => { // Create valid v2 data with 'devl' prefix const payload = new TextEncoder().encode('["test"]'); @@ -6102,7 +6066,7 @@ describe('encrypt/decrypt primitives', () => { }); }); -describe('maybeEncrypt / maybeDecrypt', () => { +describe('serialized payload encryption', () => { const testKeyRaw = new Uint8Array([ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, @@ -6115,30 +6079,24 @@ describe('maybeEncrypt / maybeDecrypt', () => { it('should pass through data unchanged when key is undefined', async () => { const data = new Uint8Array([1, 2, 3, 4]); - const result = await maybeEncrypt(data, undefined); + const result = await encryptEnvelope(data, undefined); expect(result).toBe(data); // Same reference }); it('should encrypt and add "encr" prefix when key is provided', async () => { const data = new Uint8Array([1, 2, 3, 4]); - const result = await maybeEncrypt(data, testKey); + const result = await encryptEnvelope(data, testKey); expect(result).not.toBe(data); expect(isEncrypted(result)).toBe(true); }); - it('should round-trip through maybeEncrypt/maybeDecrypt', async () => { + it('should round-trip through encrypt/decrypt', async () => { const data = new Uint8Array([10, 20, 30, 40, 50]); - const encrypted = await maybeEncrypt(data, testKey); - const decrypted = await maybeDecrypt(encrypted, testKey); + const encrypted = await encryptEnvelope(data, testKey); + const decrypted = await decryptEnvelope(encrypted, testKey); expect(decrypted).toEqual(data); }); - it('should pass through non-Uint8Array values in maybeDecrypt', async () => { - const legacyData = [1, 'hello', { key: 'value' }]; - const result = await maybeDecrypt(legacyData, testKey); - expect(result).toBe(legacyData); // Same reference - }); - it('should pass through unencrypted Uint8Array in maybeDecrypt', async () => { // Data with 'devl' prefix (not encrypted) const prefix = new TextEncoder().encode('devl'); @@ -6147,14 +6105,14 @@ describe('maybeEncrypt / maybeDecrypt', () => { data.set(prefix, 0); data.set(payload, prefix.length); - const result = await maybeDecrypt(data, testKey); + const result = await decryptEnvelope(data, testKey); expect(result).toBe(data); // Same reference — not encrypted, passed through }); it('should throw when encrypted data has no key', async () => { const data = new Uint8Array([1, 2, 3]); - const encrypted = await maybeEncrypt(data, testKey); - await expect(maybeDecrypt(encrypted, undefined)).rejects.toThrow( + const encrypted = await encryptEnvelope(data, testKey); + await expect(decryptEnvelope(encrypted, undefined)).rejects.toThrow( 'Encrypted data encountered but no encryption key' ); }); diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index a38d034745..9c51bcfaf2 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -41,6 +41,7 @@ import { } from './serialization/compression.js'; import { aesKeyOf, + type DecryptionKey, decrypt, decryptReplayPayload, deriveRunPayloadKeys, @@ -128,6 +129,7 @@ export { decompress, type EncryptionKeyParam, // Sealed-box ('encp') key variants — see serialization/encryption.ts. + type DecryptionKey, type PayloadKey, type RunPayloadKeys, type SealTarget, @@ -303,7 +305,7 @@ export function getSerializeStream( let prefixed = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); // Encrypt the frame payload if a key is provided. // The length header remains in the clear so the deserializer can @@ -323,9 +325,9 @@ export function getSerializeStream( prefixed = encodeWithFormatPrefix( SerializationFormat.SEALED, await sealSession.seal(prefixed) - ) as Uint8Array; + ); } else if (keyState.key) { - prefixed = (await encrypt(prefixed, keyState.key)) as Uint8Array; + prefixed = await encrypt(prefixed, keyState.key); } // Write length-prefixed frame: [4-byte length][prefixed data] @@ -420,10 +422,8 @@ export function getDeserializeStream( // A sealed frame needs the run's keypair; a symmetric frame needs an // AES key. Report the shortfall precisely — "no key at all" and "the // wrong kind of key" have very different causes. - const usable = sealed - ? isRunPayloadKeys(keyState.key) - : aesKeyOf(keyState.key) !== undefined; - if (!usable) { + const aesKey = aesKeyOf(keyState.key); + if (sealed ? !isRunPayloadKeys(keyState.key) : !aesKey) { controller.error( new RuntimeDecryptionError( sealed @@ -450,7 +450,7 @@ export function getDeserializeStream( // with the one-shot path. decrypted = sealed ? await openSession!.open(decodeFormatPrefix(frameData).payload) - : ((await decrypt(frameData, keyState.key)) as Uint8Array); + : await decrypt(frameData, aesKey); } catch (error) { // The low-level crypto layer only sees the stripped payload, so it // cannot record the outer envelope prefix. We peeked it here, so @@ -3379,37 +3379,6 @@ function getStepRevivers( }; } -// ============================================================================ -// Encryption Helpers -// ============================================================================ -// These delegate to the modular `encrypt`/`decrypt` from `./serialization/encryption.js` -// but are kept as named exports for backwards compatibility with existing consumers. - -/** - * Encrypt data if the world supports encryption. - * Returns original data if encryption is not available. - * - * @deprecated Use `encrypt` from `./serialization/encryption.js` instead. - */ -export async function maybeEncrypt( - data: Uint8Array, - key: PayloadKey | undefined -): Promise { - return (await encrypt(data, key)) as Uint8Array; -} - -/** - * Decrypt data if it has the 'encr' prefix. - * - * @deprecated Use `decrypt` from `./serialization/encryption.js` instead. - */ -export async function maybeDecrypt( - data: Uint8Array | unknown, - key: PayloadKey | undefined -): Promise { - return decrypt(data, key); -} - /** * Replay hydration has two stages: * @@ -3433,7 +3402,7 @@ export interface PreparedReplayPayload { */ export type ReplayPayloadPreparer = ( value: Uint8Array, - key: PayloadKey | undefined + key: DecryptionKey | undefined ) => PreparedReplayPayload | Promise; /** @@ -3442,7 +3411,7 @@ export type ReplayPayloadPreparer = ( */ function prepareReplayPayloadWithStats( value: Uint8Array, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, compressionStats?: CompressionStats ): PreparedReplayPayload | Promise { const finish = (prepared: Uint8Array): PreparedReplayPayload => ({ @@ -3472,7 +3441,7 @@ export const prepareReplayPayload: ReplayPayloadPreparer = (value, key) => async function prepareReplayPayloadWithTelemetry( value: unknown, - key: PayloadKey | undefined + key: DecryptionKey | undefined ): Promise { if (!(value instanceof Uint8Array)) return { data: value }; @@ -3567,7 +3536,7 @@ export async function dehydrateWorkflowArguments( v1Compat = false, framedByteStreams = false, compression = false -): Promise { +): Promise { if (v1Compat) { const str = stringify( value, @@ -3603,9 +3572,9 @@ export async function dehydrateWorkflowArguments( * payload skips host-side decrypt/decompress but always performs VM revival. */ export async function hydrateWorkflowArguments( - value: Uint8Array | unknown, + value: unknown, _runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload @@ -3636,7 +3605,7 @@ export async function dehydrateWorkflowReturnValue( * runtime/suspension-handler.ts passes one for step-input dehydration. */ guestCodeStatsOut?: GuestCodeStats -): Promise { +): Promise { if (v1Compat) { const str = stringify(value, getWorkflowReducers(global)); return revive(str); @@ -3671,9 +3640,9 @@ export async function dehydrateWorkflowReturnValue( * of a completed workflow run. */ export async function hydrateWorkflowReturnValue( - value: Uint8Array | unknown, + value: unknown, runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, ops: Promise[] = [], global: Record = globalThis, extraRevivers: Record any> = {} @@ -3706,7 +3675,7 @@ export async function dehydrateStepArguments( compression = false, /** See `dehydrateWorkflowReturnValue`. */ guestCodeStatsOut?: GuestCodeStats -): Promise { +): Promise { if (v1Compat) { const str = stringify(value, getWorkflowReducers(global)); return revive(str); @@ -3738,9 +3707,9 @@ export async function dehydrateStepArguments( * from the database at the start of the step execution. */ export async function hydrateStepArguments( - value: Uint8Array | unknown, + value: unknown, runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, ops: Promise[] = [], global: Record = globalThis, extraRevivers: Record any> = {}, @@ -3791,7 +3760,7 @@ export async function dehydrateStepReturnValue( // the backgrounded `run_started`. Threaded into the step reducers' stream // sink. Undefined outside turbo / on the await path. runReadyBarrier?: Promise -): Promise { +): Promise { if (v1Compat) { const str = stringify( value, @@ -3865,7 +3834,7 @@ export async function dehydrateStepError( const serialized = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); // Compress before encrypting — encrypted bytes don't compress. const compressionStats: CompressionStats = {}; const compressed = await compress( @@ -3873,10 +3842,7 @@ export async function dehydrateStepError( compression, compressionStats ); - const encrypted = (await maybeEncrypt( - compressed as Uint8Array, - key - )) as Uint8Array; + const encrypted = await encrypt(compressed, key); await recordCompression(compressionStats, 'serialize'); return encrypted; } catch (error) { @@ -3900,9 +3866,9 @@ export async function dehydrateStepError( * @returns The hydrated thrown value, ready to reject the step promise */ export async function hydrateStepError( - value: Uint8Array | unknown, + value: unknown, _runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload @@ -3938,7 +3904,7 @@ export async function dehydrateRunError( const serialized = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); // Compress before encrypting — encrypted bytes don't compress. const compressionStats: CompressionStats = {}; const compressed = await compress( @@ -3946,10 +3912,7 @@ export async function dehydrateRunError( compression, compressionStats ); - const encrypted = (await maybeEncrypt( - compressed as Uint8Array, - key - )) as Uint8Array; + const encrypted = await encrypt(compressed, key); await recordCompression(compressionStats, 'serialize'); return encrypted; } catch (error) { @@ -3972,28 +3935,23 @@ export async function dehydrateRunError( * @returns The hydrated thrown value, ready to be consumed by the client */ export async function hydrateRunError( - value: Uint8Array | unknown, + value: unknown, runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, ops: Promise[] = [], global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - const compressionStats: CompressionStats = {}; - const decrypted = await decrypt(value, key); - - if (!(decrypted instanceof Uint8Array)) { - // See the matching note in `hydrateStepError`: this branch is for - // devalue flattened arrays from legacy callers; current SDK versions - // always emit a Uint8Array, and a misshapen value here intentionally - // throws via `unflatten` so the surrounding try/catch in o11y helpers - // surfaces the issue rather than masking it. - return unflatten(decrypted as any[], { + if (!(value instanceof Uint8Array)) { + return unflatten(value as any[], { ...getExternalRevivers(global, ops, runId, key), ...extraRevivers, }); } + const compressionStats: CompressionStats = {}; + const decrypted = await decrypt(value, key); + const prepared = await decompress(decrypted, compressionStats); await recordCompression(compressionStats, 'deserialize'); const { format, payload } = decodeFormatPrefix(prepared); @@ -4023,9 +3981,9 @@ export async function hydrateRunError( * of a `step_completed` event. */ export async function hydrateStepReturnValue( - value: Uint8Array | unknown, + value: unknown, _runId: string, - key: PayloadKey | undefined, + key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload diff --git a/packages/core/src/serialization/client.ts b/packages/core/src/serialization/client.ts index 1014bf92a8..03eb59c36f 100644 --- a/packages/core/src/serialization/client.ts +++ b/packages/core/src/serialization/client.ts @@ -10,6 +10,7 @@ import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; import { compress, decompress } from './compression.js'; import { + type DecryptionKey, decrypt as decryptData, encrypt as encryptData, type PayloadKey, @@ -25,13 +26,13 @@ export async function serialize( value: unknown, encryptionKey?: PayloadKey, options?: CodecOptions -): Promise { +): Promise { try { const payload = devalueCodec.serialize(value, 'client', options); const prefixed = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); // Compress before encrypting — encrypted bytes don't compress. const compressed = await compress( prefixed, @@ -50,21 +51,20 @@ export async function serialize( * Deserialize a value for the client environment (e.g. workflow return value). */ export async function deserialize( - data: Uint8Array | unknown, - encryptionKey?: PayloadKey, + data: unknown, + encryptionKey?: DecryptionKey, options?: CodecOptions ): Promise { - const decrypted = await decryptData(data, encryptionKey); - - if (!(decrypted instanceof Uint8Array)) { + if (!(data instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { - return devalueCodec.deserializeLegacy(decrypted, 'client', options); + return devalueCodec.deserializeLegacy(data, 'client', options); } throw new Error( 'Cannot deserialize non-binary data without legacy support' ); } + const decrypted = await decryptData(data, encryptionKey); const prepared = await decompress(decrypted, options?.compressionStats); const { format, payload } = decodeFormatPrefix(prepared); diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts index 9e26cf2d80..665730f8f2 100644 --- a/packages/core/src/serialization/compression.test.ts +++ b/packages/core/src/serialization/compression.test.ts @@ -163,12 +163,6 @@ describe('CompressionStats telemetry sink', () => { expect(stats.storedBytes).toBe(original.length); }); - it('does not record for non-binary (legacy) data', async () => { - const stats: CompressionStats = {}; - await compress({ not: 'binary' }, true, stats); - expect(stats.recorded).toBeFalsy(); - }); - it('records the inflate on the read path', async () => { const original = devlBytes(JSON.stringify(makeCompressibleValue())); const compressed = (await compress(original, true)) as Uint8Array; @@ -350,7 +344,9 @@ describe('codec selection (zstd preferred, gzip fallback)', () => { // Read path still inflates gzip and reports the codec. const readStats: CompressionStats = {}; - const inflated = (await decompress(compressed, readStats)) as Uint8Array; + const result = decompress(compressed, readStats); + expect(result).not.toBeInstanceOf(Promise); + const inflated = await result; expect(inflated).toEqual(original); expect(readStats.codec).toBe('gzip'); }); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index c6981969e2..948a85c003 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -90,12 +90,6 @@ function codecOverrideFromEnv(): 'gzip' | 'zstd' | undefined { } } -interface NodeZlib { - zstdCompressSync?: (data: Uint8Array, opts?: unknown) => Uint8Array; - zstdDecompressSync?: (data: Uint8Array) => Uint8Array; - constants?: Record; -} - /** * Resolve `node:zlib` via `process.getBuiltinModule` — no static import, so * this module stays bundler-safe for browser/edge targets (where it returns @@ -103,11 +97,9 @@ interface NodeZlib { */ const nodeZlib = (() => { try { - return ( - globalThis as { - process?: { getBuiltinModule?: (id: string) => NodeZlib }; - } - ).process?.getBuiltinModule?.('node:zlib'); + return typeof process === 'undefined' + ? undefined + : process.getBuiltinModule('node:zlib'); } catch { return undefined; } @@ -123,9 +115,13 @@ function isZstdAvailable(): boolean { /** * gzip via the web-standard `CompressionStream` (Node 18+, browsers, edge). */ -function isGzipAvailable(): boolean { +function canCompressGzip(): boolean { + return typeof CompressionStream === 'function'; +} + +function canDecompressGzip(): boolean { return ( - typeof CompressionStream === 'function' && + typeof nodeZlib?.gunzipSync === 'function' || typeof DecompressionStream === 'function' ); } @@ -150,12 +146,9 @@ async function pipeThroughTransform( writePromise.catch(() => {}); const chunks: Uint8Array[] = []; let total = 0; - const reader = transform.readable.getReader(); - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - total += value.length; + for await (const chunk of transform.readable) { + chunks.push(chunk); + total += chunk.length; } await writePromise; const out = new Uint8Array(total); @@ -175,6 +168,19 @@ async function gunzipBytes(data: Uint8Array): Promise { return pipeThroughTransform(data, new DecompressionStream('gzip')); } +function gunzip(data: Uint8Array): Uint8Array | Promise { + if (nodeZlib?.gunzipSync) { + return new Uint8Array(nodeZlib.gunzipSync(data)); + } + if (typeof DecompressionStream === 'function') { + return gunzipBytes(data); + } + throw new Error( + 'Compressed (gzip) workflow data encountered but no gzip decoder is ' + + 'available in this runtime.' + ); +} + function zstdBytes(data: Uint8Array): Uint8Array { const level = nodeZlib?.constants?.ZSTD_c_compressionLevel; const opts = @@ -243,10 +249,10 @@ function recordStats( */ function selectWriteCodec(): 'zstd' | 'gzip' | 'none' { const override = codecOverrideFromEnv(); - if (override === 'gzip') return isGzipAvailable() ? 'gzip' : 'none'; + if (override === 'gzip') return canCompressGzip() ? 'gzip' : 'none'; // Default and explicit 'zstd' both prefer zstd, then fall back to gzip. if (isZstdAvailable()) return 'zstd'; - if (isGzipAvailable()) return 'gzip'; + if (canCompressGzip()) return 'gzip'; return 'none'; } @@ -265,12 +271,10 @@ function selectWriteCodec(): 'zstd' | 'gzip' | 'none' { * when compression is disabled, unavailable, or not worthwhile. */ export async function compress( - data: Uint8Array | unknown, + data: Uint8Array, enabled: boolean, stats?: CompressionStats -): Promise { - if (!(data instanceof Uint8Array)) return data; - // From here `data` is binary, so every return path records stats. +): Promise { if ( !enabled || data.length < COMPRESSION_MIN_BYTES || @@ -319,18 +323,19 @@ export function decompress( } if (prefix === SerializationFormat.GZIP) { - if (!isGzipAvailable()) { - throw new Error( - 'Compressed (gzip) workflow data encountered but DecompressionStream ' + - 'is not available in this runtime. Node.js 18+, browsers, and edge ' + - 'runtimes all support it.' - ); + if (!canDecompressGzip()) { + throw new Error('Compressed (gzip) workflow data cannot be decoded'); } const { payload } = decodeFormatPrefix(data); - return gunzipBytes(payload).then((inflated) => { - recordStats(stats, 'gzip', inflated.length, data.length); - return inflated; - }); + const inflated = gunzip(payload); + if (inflated instanceof Promise) { + return inflated.then((result) => { + recordStats(stats, 'gzip', result.length, data.length); + return result; + }); + } + recordStats(stats, 'gzip', inflated.length, data.length); + return inflated; } recordStats(stats, 'none', data.length, data.length); @@ -340,7 +345,7 @@ export function decompress( /** * Check if data is compressed (has a 'zstd' or 'gzip' format prefix). */ -export function isCompressed(data: Uint8Array | unknown): boolean { +export function isCompressed(data: Uint8Array): boolean { const prefix = peekFormatPrefix(data); return ( prefix === SerializationFormat.ZSTD || prefix === SerializationFormat.GZIP diff --git a/packages/core/src/serialization/encryption.ts b/packages/core/src/serialization/encryption.ts index fcf36f9427..531833795f 100644 --- a/packages/core/src/serialization/encryption.ts +++ b/packages/core/src/serialization/encryption.ts @@ -208,10 +208,10 @@ export async function resolveEncryptionKey( * @returns The encrypted data with its format prefix, or the original data if no key */ export async function encrypt( - data: Uint8Array | unknown, + data: Uint8Array, key: PayloadKey | undefined -): Promise { - if (!key || !(data instanceof Uint8Array)) return data; +): Promise { + if (!key) return data; if (isSealTarget(key)) { const sealed = await sealToPublicKey(key.recipientPublicKey, data, key.aad); @@ -243,7 +243,7 @@ export async function encrypt( */ async function openSealedEnvelope( data: Uint8Array, - key: PayloadKey | undefined + key: DecryptionKey | undefined ): Promise { // Sealed payloads need the private scalar. Anything else — no key, a bare // symmetric key, or a write-only seal target — cannot open them. @@ -272,7 +272,7 @@ async function openSealedEnvelope( function requireAesDecryptionKey( data: Uint8Array, - key: PayloadKey | undefined + key: DecryptionKey | undefined ): CryptoKey { const aesKey = aesKeyOf(key); if (aesKey) return aesKey; @@ -290,21 +290,10 @@ function requireAesDecryptionKey( ); } -export function decrypt( - data: Uint8Array, - key: PayloadKey | undefined -): Promise; -export function decrypt( - data: unknown, - key: PayloadKey | undefined -): Promise; export async function decrypt( - data: Uint8Array | unknown, - key: PayloadKey | undefined -): Promise { - // Non-binary data is returned as-is. - if (!(data instanceof Uint8Array)) return data; - + data: Uint8Array, + key: DecryptionKey | undefined +): Promise { const format = peekFormatPrefix(data); if (format === SerializationFormat.SEALED) { @@ -340,7 +329,7 @@ function addFormatPrefix(error: unknown, format: string): unknown { */ export function decryptReplayPayload( data: Uint8Array, - key: PayloadKey | undefined + key: DecryptionKey | undefined ): Uint8Array { const format = peekFormatPrefix(data); if (format === SerializationFormat.SEALED) { diff --git a/packages/core/src/serialization/format.ts b/packages/core/src/serialization/format.ts index 28f1bc134d..4cb286b111 100644 --- a/packages/core/src/serialization/format.ts +++ b/packages/core/src/serialization/format.ts @@ -36,12 +36,8 @@ const formatDecoder = new TextDecoder(); */ export function encodeWithFormatPrefix( format: FormatPrefix, - payload: Uint8Array | unknown -): Uint8Array | unknown { - if (!(payload instanceof Uint8Array)) { - return payload; - } - + payload: Uint8Array +): Uint8Array { const prefixBytes = formatEncoder.encode(format); const result = new Uint8Array(FORMAT_PREFIX_LENGTH + payload.length); result.set(prefixBytes, 0); @@ -59,10 +55,8 @@ export function encodeWithFormatPrefix( * @param data - The format-prefixed data * @returns The format prefix, or null */ -export function peekFormatPrefix( - data: Uint8Array | unknown -): FormatPrefix | null { - if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { +export function peekFormatPrefix(data: Uint8Array): FormatPrefix | null { + if (data.length < FORMAT_PREFIX_LENGTH) { return null; } const prefixBytes = data.subarray(0, FORMAT_PREFIX_LENGTH); @@ -73,8 +67,11 @@ export function peekFormatPrefix( /** * Check if data is encrypted (has 'encr' format prefix). */ -export function isEncrypted(data: Uint8Array | unknown): boolean { - return peekFormatPrefix(data) === SerializationFormat.ENCRYPTED; +export function isEncrypted(data: unknown): boolean { + return ( + data instanceof Uint8Array && + peekFormatPrefix(data) === SerializationFormat.ENCRYPTED + ); } /** @@ -92,18 +89,10 @@ export function isEncrypted(data: Uint8Array | unknown): boolean { * @returns An object with the format prefix and payload * @throws Error if the data is too short or has an invalid prefix */ -export function decodeFormatPrefix(data: Uint8Array | unknown): { +export function decodeFormatPrefix(data: Uint8Array): { format: FormatPrefix; payload: Uint8Array; } { - // Compat for legacy specVersion 1 runs that don't have a format prefix - if (!(data instanceof Uint8Array)) { - return { - format: SerializationFormat.DEVALUE_V1, - payload: new TextEncoder().encode(JSON.stringify(data)), - }; - } - if (data.length < FORMAT_PREFIX_LENGTH) { throw new Error( `Data too short to contain format prefix: expected at least ${FORMAT_PREFIX_LENGTH} bytes, got ${data.length}` diff --git a/packages/core/src/serialization/index.ts b/packages/core/src/serialization/index.ts index 8fe3d68213..d2e0ec0869 100644 --- a/packages/core/src/serialization/index.ts +++ b/packages/core/src/serialization/index.ts @@ -20,6 +20,7 @@ export { // Re-export composable encryption export { type CryptoKey, + type DecryptionKey, decrypt, type EncryptionKeyParam, encrypt, diff --git a/packages/core/src/serialization/serialization.test.ts b/packages/core/src/serialization/serialization.test.ts index 856d1e17c4..6b16bc36ce 100644 --- a/packages/core/src/serialization/serialization.test.ts +++ b/packages/core/src/serialization/serialization.test.ts @@ -117,7 +117,7 @@ describe('encodeWithFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded.length).toBe(4 + 3); // First 4 bytes should be 'devl' @@ -126,28 +126,12 @@ describe('encodeWithFormatPrefix', () => { expect(Array.from(encoded.subarray(4))).toEqual([1, 2, 3]); }); - it('should return non-Uint8Array values unchanged', () => { - const str = 'hello'; - expect(encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, str)).toBe( - str - ); - - const num = 42; - expect(encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, num)).toBe( - num - ); - - expect( - encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, null) - ).toBeNull(); - }); - it('should handle empty payload', () => { const payload = new Uint8Array(0); const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded.length).toBe(4); }); @@ -157,7 +141,7 @@ describe('encodeWithFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); expect(encoded.length).toBe(4 + 100000); }); }); @@ -168,22 +152,13 @@ describe('decodeFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); const decoded = decodeFormatPrefix(encoded); expect(decoded.format).toBe('devl'); expect(decoded.payload).toEqual(new Uint8Array([1, 2, 3])); }); - it('should handle legacy non-binary data', () => { - const legacyData = [1, 'hello', { a: 2 }]; - const decoded = decodeFormatPrefix(legacyData); - expect(decoded.format).toBe('devl'); - expect(decoded.payload).toEqual( - new TextEncoder().encode(JSON.stringify(legacyData)) - ); - }); - it('should throw for data too short', () => { expect(() => decodeFormatPrefix(new Uint8Array([1, 2, 3]))).toThrow( /Data too short to contain format prefix/ @@ -204,7 +179,7 @@ describe('decodeFormatPrefix', () => { const encoded = encodeWithFormatPrefix( SerializationFormat.ENCRYPTED, payload - ) as Uint8Array; + ); const decoded = decodeFormatPrefix(encoded); expect(decoded.format).toBe('encr'); @@ -223,13 +198,6 @@ describe('peekFormatPrefix', () => { expect(peekFormatPrefix(encoded)).toBe('devl'); }); - it('should return null for non-binary data', () => { - expect(peekFormatPrefix('not binary')).toBeNull(); - expect(peekFormatPrefix(42)).toBeNull(); - expect(peekFormatPrefix(null)).toBeNull(); - expect(peekFormatPrefix(undefined)).toBeNull(); - }); - it('should return null for data too short', () => { expect(peekFormatPrefix(new Uint8Array([1]))).toBeNull(); expect(peekFormatPrefix(new Uint8Array([1, 2, 3]))).toBeNull(); @@ -287,19 +255,12 @@ describe('encrypt', () => { expect(result).toBe(data); }); - it('should return non-Uint8Array data unchanged even with key', async () => { - const key = await makeKey(); - const data = 'string data'; - const result = await encrypt(data, key); - expect(result).toBe(data); - }); - it('should encrypt and add encr prefix when key provided', async () => { const key = await makeKey(); const data = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, new Uint8Array([1, 2, 3]) - ) as Uint8Array; + ); const encrypted = await encrypt(data, key); expect(encrypted).toBeInstanceOf(Uint8Array); diff --git a/packages/core/src/serialization/step.ts b/packages/core/src/serialization/step.ts index 08f5cbfabc..94d5a9acf6 100644 --- a/packages/core/src/serialization/step.ts +++ b/packages/core/src/serialization/step.ts @@ -10,6 +10,7 @@ import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; import { compress, decompress } from './compression.js'; import { + type DecryptionKey, decrypt as decryptData, encrypt as encryptData, type PayloadKey, @@ -25,13 +26,13 @@ export async function serialize( value: unknown, encryptionKey?: PayloadKey, options?: CodecOptions -): Promise { +): Promise { try { const payload = devalueCodec.serialize(value, 'step', options); const prefixed = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, payload - ) as Uint8Array; + ); // Compress before encrypting — encrypted bytes don't compress. const compressed = await compress( prefixed, @@ -50,21 +51,20 @@ export async function serialize( * Deserialize a value for the step execution environment. */ export async function deserialize( - data: Uint8Array | unknown, - encryptionKey?: PayloadKey, + data: unknown, + encryptionKey?: DecryptionKey, options?: CodecOptions ): Promise { - const decrypted = await decryptData(data, encryptionKey); - - if (!(decrypted instanceof Uint8Array)) { + if (!(data instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { - return devalueCodec.deserializeLegacy(decrypted, 'step', options); + return devalueCodec.deserializeLegacy(data, 'step', options); } throw new Error( 'Cannot deserialize non-binary data without legacy support' ); } + const decrypted = await decryptData(data, encryptionKey); const prepared = await decompress(decrypted, options?.compressionStats); const { format, payload } = decodeFormatPrefix(prepared); diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 617c4cd205..51a1b5467b 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -24,14 +24,20 @@ export function isFormatPrefix(value: string): value is FormatPrefix { return value.length === 4 && /^[a-z0-9]{4}$/.test(value); } +function formatPrefix( + value: Value +): Value & FormatPrefix { + return value as Value & FormatPrefix; +} + /** * Well-known format prefix constants. Codecs may define additional ones. */ export const SerializationFormat = { /** devalue stringify/parse with TextEncoder/TextDecoder */ - DEVALUE_V1: 'devl' as FormatPrefix, + DEVALUE_V1: formatPrefix('devl'), /** Encrypted payload (inner payload has its own format prefix) */ - ENCRYPTED: 'encr' as FormatPrefix, + ENCRYPTED: formatPrefix('encr'), /** * Sealed payload — asymmetrically encrypted to a run's X25519 public key * (inner payload has its own format prefix). @@ -41,11 +47,11 @@ export const SerializationFormat = { * cannot decrypt. A run's own payloads continue to use {@link ENCRYPTED}. * See `sealed-box.ts` for the construction. */ - SEALED: 'encp' as FormatPrefix, + SEALED: formatPrefix('encp'), /** Gzip-compressed payload (inner payload has its own format prefix) */ - GZIP: 'gzip' as FormatPrefix, + GZIP: formatPrefix('gzip'), /** Zstandard-compressed payload (inner payload has its own format prefix) */ - ZSTD: 'zstd' as FormatPrefix, + ZSTD: formatPrefix('zstd'), } as const; // ---- Serializable Types ---- diff --git a/packages/core/src/serialization/workflow-vm.ts b/packages/core/src/serialization/workflow-vm.ts index 7ebf7c938f..4bad5fd921 100644 --- a/packages/core/src/serialization/workflow-vm.ts +++ b/packages/core/src/serialization/workflow-vm.ts @@ -49,7 +49,7 @@ export function serialize(value: unknown): Uint8Array { * @param data - Uint8Array with format prefix, or legacy non-binary data * @returns The deserialized value */ -export function deserialize(data: Uint8Array | unknown): unknown { +export function deserialize(data: unknown): unknown { // Legacy: non-binary data if (!(data instanceof Uint8Array)) { if (devalueVmCodec.deserializeLegacy) { diff --git a/packages/core/src/serialization/workflow.ts b/packages/core/src/serialization/workflow.ts index 344fec0151..a0791ba733 100644 --- a/packages/core/src/serialization/workflow.ts +++ b/packages/core/src/serialization/workflow.ts @@ -27,10 +27,7 @@ import { SerializationFormat } from './types.js'; export function serialize(value: unknown, options?: CodecOptions): Uint8Array { try { const payload = devalueCodec.serialize(value, 'workflow', options); - return encodeWithFormatPrefix( - SerializationFormat.DEVALUE_V1, - payload - ) as Uint8Array; + return encodeWithFormatPrefix(SerializationFormat.DEVALUE_V1, payload); } catch (error) { const { message, hint } = formatSerializationError('workflow value', error); throw new SerializationError(message, { hint, cause: error }); @@ -44,10 +41,7 @@ export function serialize(value: unknown, options?: CodecOptions): Uint8Array { * @param options - Optional global, extra revivers for VM-context deserialization * @returns The deserialized value */ -export function deserialize( - data: Uint8Array | unknown, - options?: CodecOptions -): unknown { +export function deserialize(data: unknown, options?: CodecOptions): unknown { if (!(data instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { return devalueCodec.deserializeLegacy(data, 'workflow', options); diff --git a/packages/core/src/step/context-storage.ts b/packages/core/src/step/context-storage.ts index 91f41e099e..d19a1216e0 100644 --- a/packages/core/src/step/context-storage.ts +++ b/packages/core/src/step/context-storage.ts @@ -1,6 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { FlushableStreamState } from '../flushable-stream.js'; -import type { PayloadKey } from '../serialization/encryption.js'; +import type { DecryptionKey } from '../serialization/encryption.js'; import type { WorkflowMetadata } from '../workflow/get-workflow-metadata.js'; import type { StepMetadata } from './get-step-metadata.js'; @@ -57,7 +57,7 @@ export type StepContext = { */ preCompletionOps: Promise[]; closureVars?: Record; - encryptionKey?: PayloadKey; + encryptionKey?: DecryptionKey; writables?: Map; /** * Turbo mode only: a promise that resolves once the backgrounded diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 98b8728269..3bcd3f08fa 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -26,7 +26,7 @@ import { getPortLazy } from './runtime/get-port-lazy.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; -import type { PayloadKey } from './serialization/encryption.js'; +import type { DecryptionKey } from './serialization/encryption.js'; import { dehydrateWorkflowReturnValue, hydrateWorkflowArguments, @@ -123,7 +123,7 @@ interface WorkflowSessionOptions { readonly workflowCode: string; readonly workflowRun: WorkflowRun; readonly events: Event[]; - readonly encryptionKey: PayloadKey | undefined; + readonly encryptionKey: DecryptionKey | undefined; readonly replayPayloadCache: ReplayPayloadCache; readonly runReadyBarrier?: Promise; readonly worldCapabilities?: WorldCapabilities; @@ -271,7 +271,7 @@ export async function runWorkflow( workflowCode: string, workflowRun: WorkflowRun, events: Event[], - encryptionKey: PayloadKey | undefined, + encryptionKey: DecryptionKey | undefined, /** * Optional per-run cache for replay payload preparation and immutable final * values. Owned by the inline execution loop for this invocation. @@ -291,7 +291,7 @@ export async function runWorkflow( * capabilities are treated as unsupported. */ worldCapabilities?: WorldCapabilities -): Promise { +): Promise { const result = await replayWorkflow({ workflowCode, workflowRun, From 363133018acd85dfa90226a8fee7e1c6e8adc3a0 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:55:53 -0700 Subject: [PATCH 05/16] [core] Consolidate payload codecs --- .../decode-replay-payloads-synchronously.md | 1 + packages/core/src/encryption.ts | 6 +- packages/core/src/serialization-format.ts | 114 +----------- packages/core/src/serialization.test.ts | 2 +- packages/core/src/serialization.ts | 146 +++++---------- packages/core/src/serialization/client.ts | 23 ++- .../src/serialization/compression.test.ts | 10 +- .../core/src/serialization/compression.ts | 170 +++++++++++------- packages/core/src/serialization/encryption.ts | 21 +-- packages/core/src/serialization/payload.ts | 26 +++ packages/core/src/serialization/replay.ts | 40 +++++ .../src/serialization/serialization.test.ts | 6 - packages/core/src/serialization/step.ts | 23 ++- .../world-vercel/src/serialized-data.test.ts | 33 ++++ packages/world-vercel/src/serialized-data.ts | 27 +-- 15 files changed, 302 insertions(+), 346 deletions(-) create mode 100644 packages/core/src/serialization/payload.ts create mode 100644 packages/core/src/serialization/replay.ts create mode 100644 packages/world-vercel/src/serialized-data.test.ts diff --git a/.changeset/decode-replay-payloads-synchronously.md b/.changeset/decode-replay-payloads-synchronously.md index 54d0633684..de58d45d3b 100644 --- a/.changeset/decode-replay-payloads-synchronously.md +++ b/.changeset/decode-replay-payloads-synchronously.md @@ -1,6 +1,7 @@ --- "@workflow/core": patch "@workflow/cli": patch +"@workflow/world-vercel": patch --- Simplify payload codecs and decode replay payloads synchronously with Node AES-GCM and zstd. diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index 65f15826d9..d9ad2629fa 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -27,8 +27,10 @@ export type CryptoKey = import('node:crypto').webcrypto.CryptoKey; * * Node's synchronous cipher API needs a native key handle. Creating it while * the raw bytes are already available avoids trying to convert a deliberately - * non-extractable `CryptoKey` later. Browser/edge callers never populate or - * consult this map. + * non-extractable `CryptoKey` later. Passing the `CryptoKey` directly to + * `createDecipheriv`, or converting it later with `KeyObject.from`, is + * deprecated by Node for non-extractable keys. Browser/edge callers never + * populate or consult this map. */ const nodeKeys = new WeakMap(); diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index 06848c8de1..28947f53eb 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -8,6 +8,11 @@ import { getEventDataRefFields } from '@workflow/world'; import { parse, unflatten } from 'devalue'; +import { + decompress, + decompressSync, + registerZstdDecoder, +} from './serialization/compression.js'; import { decodeFormatPrefix as decodePrefix, encodeWithFormatPrefix, @@ -47,6 +52,7 @@ export { // --------------------------------------------------------------------------- export { encodeWithFormatPrefix, SerializationFormat }; +export { registerZstdDecoder }; export type SerializationFormatType = (typeof SerializationFormat)[keyof typeof SerializationFormat]; @@ -165,109 +171,6 @@ export function isCompressedData(data: unknown): boolean { ); } -/** - * Resolve `node:zlib` via `process.getBuiltinModule` — no static Node - * dependency, invisible to browser bundlers. Returns undefined off Node. - */ -function getNodeZlib() { - try { - return typeof process === 'undefined' - ? undefined - : process.getBuiltinModule('node:zlib'); - } catch { - return undefined; - } -} - -/** - * Synchronously decompress a `gzip`/`zstd` payload when running on Node.js. - * - * Returns `undefined` when sync decompression isn't available (e.g. in the - * browser, or zstd on Node < 22.15) — callers fall back to leaving the data - * un-hydrated (the async `hydrateDataWithKey` path handles decompression in - * browsers via `DecompressionStream` / a registered zstd decoder). - */ -function decompressSyncIfAvailable( - format: string, - payload: Uint8Array -): Uint8Array | undefined { - try { - const zlib = getNodeZlib(); - if (format === SerializationFormat.GZIP && zlib?.gunzipSync) { - return new Uint8Array(zlib.gunzipSync(payload)); - } - if (format === SerializationFormat.ZSTD && zlib?.zstdDecompressSync) { - return new Uint8Array(zlib.zstdDecompressSync(payload)); - } - } catch { - // Fall through — treat as unavailable - } - return undefined; -} - -/** - * Browser zstd decoder, registered by the o11y host (web-shared) since the - * Web `DecompressionStream` has no zstd support. Node decodes via `node:zlib` - * and never needs this. See `registerZstdDecoder`. - */ -let zstdBrowserDecoder: - | ((payload: Uint8Array) => Promise) - | undefined; - -/** - * Register a browser zstd decoder (e.g. a WASM-backed one). The web o11y UI - * calls this at init so `hydrateDataWithKey` can inflate zstd payloads after - * client-side decryption. Node readers use `node:zlib` and ignore this. - */ -export function registerZstdDecoder( - decoder: (payload: Uint8Array) => Promise -): void { - zstdBrowserDecoder = decoder; -} - -/** - * Asynchronously decompress a `gzip`/`zstd` payload. - * - gzip: web-standard `DecompressionStream` (Node 18+, browsers, edge). - * - zstd: `node:zlib` when on Node, else the registered browser decoder. - */ -async function decompressAsync( - format: string, - payload: Uint8Array -): Promise { - if (format === SerializationFormat.ZSTD) { - const sync = decompressSyncIfAvailable(format, payload); - if (sync) return sync; - if (zstdBrowserDecoder) return zstdBrowserDecoder(payload); - throw new Error( - 'zstd-compressed workflow data encountered but no zstd decoder is ' + - 'available. Node.js 22.15+ decodes natively; in the browser register ' + - 'one via registerZstdDecoder (the web o11y package does this).' - ); - } - - const transform = new DecompressionStream('gzip'); - const writer = transform.writable.getWriter(); - const writePromise = writer.write(payload).then(() => writer.close()); - writePromise.catch(() => {}); - const chunks: Uint8Array[] = []; - let total = 0; - const reader = transform.readable.getReader(); - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - total += value.length; - } - await writePromise; - const out = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - out.set(chunk, offset); - offset += chunk.length; - } - return out; -} - // --------------------------------------------------------------------------- // Revivers type (shared across all environments) // --------------------------------------------------------------------------- @@ -320,7 +223,7 @@ export function hydrateData(value: unknown, revivers: Revivers): unknown { // pass the data through untouched (like encrypted data) so async // consumers can route it through `hydrateDataWithKey`, which // decompresses via DecompressionStream / a registered zstd decoder. - const inflated = decompressSyncIfAvailable(format, payload); + const inflated = decompressSync(value); if (inflated === undefined) { return value; } @@ -379,8 +282,7 @@ export async function hydrateDataWithKey( // web-standard DecompressionStream (works in browsers); zstd uses // node:zlib on Node or the registered WASM decoder in the browser. // The inflated bytes carry their own format prefix (e.g. 'devl'). - const { format, payload } = decodeFormatPrefix(data); - data = await decompressAsync(format, payload); + data = await decompress(data); } // Delegate the (decrypted/decompressed) result to sync hydrateData return hydrateData(data, revivers); diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index aab4a45554..d354a2d635 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -6097,7 +6097,7 @@ describe('serialized payload encryption', () => { expect(decrypted).toEqual(data); }); - it('should pass through unencrypted Uint8Array in maybeDecrypt', async () => { + it('should pass through an unencrypted Uint8Array', async () => { // Data with 'devl' prefix (not encrypted) const prefix = new TextEncoder().encode('devl'); const payload = new TextEncoder().encode('test'); diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index 9c51bcfaf2..25102dfa98 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -34,16 +34,11 @@ import { decodeRunPublicKey, } from './sealed-box.js'; import * as clientModule from './serialization/client.js'; -import { - type CompressionStats, - compress, - decompress, -} from './serialization/compression.js'; +import type { CompressionStats } from './serialization/compression.js'; import { aesKeyOf, type DecryptionKey, decrypt, - decryptReplayPayload, deriveRunPayloadKeys, type EncryptionKeyParam, encrypt, @@ -71,6 +66,7 @@ import { isInstanceOfPrototype, readProperty, } from './serialization/hardened.js'; +import { decodePayload, encodePayload } from './serialization/payload.js'; import { getClassReducers, getClassRevivers, @@ -84,6 +80,10 @@ import { getStepFunctionReducer, getStepFunctionReviver, } from './serialization/reducers/step-function.js'; +import { + type PreparedReplayPayload, + prepareReplayPayload, +} from './serialization/replay.js'; import * as stepModule from './serialization/step.js'; import { type FormatPrefix, @@ -125,8 +125,6 @@ export { isEncrypted, encrypt, decrypt, - compress, - decompress, type EncryptionKeyParam, // Sealed-box ('encp') key variants — see serialization/encryption.ts. type DecryptionKey, @@ -141,6 +139,8 @@ export { aesKeyOf, }; +export { compress, decompress } from './serialization/compression.js'; + // Re-export the legacy SerializationFormatType for backwards compatibility. // New code should use FormatPrefix from './serialization/types.js'. export type SerializationFormatType = @@ -3379,78 +3379,20 @@ function getStepRevivers( }; } -/** - * Replay hydration has two stages: - * - * 1. Host-side preparation decrypts and decompresses persisted data. That work - * is independent of a workflow VM and can be cached across replay VMs. - * 2. Deserialization revives the prepared representation against the current - * VM's globals. It must run again for every VM to produce fresh object graphs - * and correctly scoped Workflow objects. - * - * `data` is the boundary between those stages. For current-format payloads it - * is still format-prefixed serialized bytes, not a live JavaScript value. - */ -export interface PreparedReplayPayload { - readonly data: unknown; -} - -/** - * Swappable implementation of the host-side preparation stage. Supporting - * both direct and promised results covers synchronous Node AES/zstd as well as - * the portable Web Crypto and sealed-envelope fallbacks. - */ -export type ReplayPayloadPreparer = ( - value: Uint8Array, - key: DecryptionKey | undefined -) => PreparedReplayPayload | Promise; - -/** - * Decrypt and decompress persisted data without parsing it into JavaScript. - * Legacy non-binary values are handled before this binary preparation boundary. - */ -function prepareReplayPayloadWithStats( - value: Uint8Array, - key: DecryptionKey | undefined, - compressionStats?: CompressionStats -): PreparedReplayPayload | Promise { - const finish = (prepared: Uint8Array): PreparedReplayPayload => ({ - data: prepared, - }); - const decompressPrepared = ( - decrypted: Uint8Array - ): PreparedReplayPayload | Promise => { - const prepared = decompress(decrypted, compressionStats); - return prepared instanceof Promise - ? prepared.then(finish) - : finish(prepared); - }; - - if (peekFormatPrefix(value) === SerializationFormat.SEALED) { - return decrypt(value, key).then(decompressPrepared); - } - return decompressPrepared(decryptReplayPayload(value, key)); -} - -// Replay preparation is event-at-a-time and may run inside the response -// decoder. Per-payload compression attributes would add a detached O(N) -// microtask tail and repeatedly overwrite one span, so the replay fast path -// records only its aggregate preparation span. -export const prepareReplayPayload: ReplayPayloadPreparer = (value, key) => - prepareReplayPayloadWithStats(value, key); +export { + type PreparedReplayPayload, + prepareReplayPayload, + type ReplayPayloadPreparer, +} from './serialization/replay.js'; async function prepareReplayPayloadWithTelemetry( value: unknown, key: DecryptionKey | undefined ): Promise { - if (!(value instanceof Uint8Array)) return { data: value }; + if (!(value instanceof Uint8Array)) return { legacy: value }; const compressionStats: CompressionStats = {}; - const prepared = await prepareReplayPayloadWithStats( - value, - key, - compressionStats - ); + const prepared = await prepareReplayPayload(value, key, compressionStats); await recordCompression(compressionStats, 'deserialize'); return prepared; } @@ -3465,7 +3407,8 @@ export function deserializePreparedReplayPayload( global: Record = globalThis, extraRevivers: Record any> = {} ): any { - return workflowModule.deserialize(prepared.data, { + const data = prepared instanceof Uint8Array ? prepared : prepared.legacy; + return workflowModule.deserialize(data, { global, extraRevivers: { ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), @@ -3483,7 +3426,7 @@ export function deserializePreparedStepError( global: Record = globalThis, extraRevivers: Record any> = {} ): unknown { - const { data } = prepared; + const data = prepared instanceof Uint8Array ? prepared : prepared.legacy; if (!(data instanceof Uint8Array)) { return unflatten(data as any[], { ...getWorkflowRevivers(global), @@ -3571,18 +3514,19 @@ export async function dehydrateWorkflowArguments( * arguments from the database at the start of workflow execution. A prepared * payload skips host-side decrypt/decompress but always performs VM revival. */ -export async function hydrateWorkflowArguments( +export function hydrateWorkflowArguments( value: unknown, _runId: string, key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload -): Promise { - return deserializePreparedReplayPayload( - prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)), - global, - extraRevivers +): any | Promise { + if (prepared) { + return deserializePreparedReplayPayload(prepared, global, extraRevivers); + } + return prepareReplayPayloadWithTelemetry(value, key).then((payload) => + deserializePreparedReplayPayload(payload, global, extraRevivers) ); } @@ -3835,14 +3779,13 @@ export async function dehydrateStepError( SerializationFormat.DEVALUE_V1, payload ); - // Compress before encrypting — encrypted bytes don't compress. const compressionStats: CompressionStats = {}; - const compressed = await compress( + const encrypted = await encodePayload( serialized, + key, compression, compressionStats ); - const encrypted = await encrypt(compressed, key); await recordCompression(compressionStats, 'serialize'); return encrypted; } catch (error) { @@ -3865,18 +3808,19 @@ export async function dehydrateStepError( * @param prepared - Optional cached decrypt/decompress result * @returns The hydrated thrown value, ready to reject the step promise */ -export async function hydrateStepError( +export function hydrateStepError( value: unknown, _runId: string, key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload -): Promise { - return deserializePreparedStepError( - prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)), - global, - extraRevivers +): unknown | Promise { + if (prepared) { + return deserializePreparedStepError(prepared, global, extraRevivers); + } + return prepareReplayPayloadWithTelemetry(value, key).then((payload) => + deserializePreparedStepError(payload, global, extraRevivers) ); } @@ -3905,14 +3849,13 @@ export async function dehydrateRunError( SerializationFormat.DEVALUE_V1, payload ); - // Compress before encrypting — encrypted bytes don't compress. const compressionStats: CompressionStats = {}; - const compressed = await compress( + const encrypted = await encodePayload( serialized, + key, compression, compressionStats ); - const encrypted = await encrypt(compressed, key); await recordCompression(compressionStats, 'serialize'); return encrypted; } catch (error) { @@ -3950,9 +3893,7 @@ export async function hydrateRunError( } const compressionStats: CompressionStats = {}; - const decrypted = await decrypt(value, key); - - const prepared = await decompress(decrypted, compressionStats); + const prepared = await decodePayload(value, key, compressionStats); await recordCompression(compressionStats, 'deserialize'); const { format, payload } = decodeFormatPrefix(prepared); @@ -3980,18 +3921,19 @@ export async function hydrateRunError( * Called from the workflow handler when replaying the event log * of a `step_completed` event. */ -export async function hydrateStepReturnValue( +export function hydrateStepReturnValue( value: unknown, _runId: string, key: DecryptionKey | undefined, global: Record = globalThis, extraRevivers: Record any> = {}, prepared?: PreparedReplayPayload -): Promise { - return deserializePreparedReplayPayload( - prepared ?? (await prepareReplayPayloadWithTelemetry(value, key)), - global, - extraRevivers +): any | Promise { + if (prepared) { + return deserializePreparedReplayPayload(prepared, global, extraRevivers); + } + return prepareReplayPayloadWithTelemetry(value, key).then((payload) => + deserializePreparedReplayPayload(payload, global, extraRevivers) ); } diff --git a/packages/core/src/serialization/client.ts b/packages/core/src/serialization/client.ts index 03eb59c36f..b23586000a 100644 --- a/packages/core/src/serialization/client.ts +++ b/packages/core/src/serialization/client.ts @@ -8,15 +8,10 @@ import { SerializationError } from '@workflow/errors'; import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; -import { compress, decompress } from './compression.js'; -import { - type DecryptionKey, - decrypt as decryptData, - encrypt as encryptData, - type PayloadKey, -} from './encryption.js'; +import type { DecryptionKey, PayloadKey } from './encryption.js'; import { formatSerializationError, rethrowIfRuntimeError } from './errors.js'; import { decodeFormatPrefix, encodeWithFormatPrefix } from './format.js'; +import { decodePayload, encodePayload } from './payload.js'; import { SerializationFormat } from './types.js'; /** @@ -33,13 +28,12 @@ export async function serialize( SerializationFormat.DEVALUE_V1, payload ); - // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress( + return await encodePayload( prefixed, - options?.compression === true, + encryptionKey, + options?.compression ?? false, options?.compressionStats ); - return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); const { message, hint } = formatSerializationError('client value', error); @@ -64,8 +58,11 @@ export async function deserialize( ); } - const decrypted = await decryptData(data, encryptionKey); - const prepared = await decompress(decrypted, options?.compressionStats); + const prepared = await decodePayload( + data, + encryptionKey, + options?.compressionStats + ); const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts index 665730f8f2..bdf2dd5820 100644 --- a/packages/core/src/serialization/compression.test.ts +++ b/packages/core/src/serialization/compression.test.ts @@ -325,20 +325,24 @@ describe('codec selection (zstd preferred, gzip fallback)', () => { textEncoder.encode(JSON.stringify(makeCompressibleValue())) ) as Uint8Array; const stats: CompressionStats = {}; - const compressed = await compress(original, true, stats); + const compression = compress(original, true, stats); + expect(compression).not.toBeInstanceOf(Promise); + const compressed = await compression; expect(isCompressed(compressed)).toBe(true); expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.ZSTD); expect(stats.codec).toBe('zstd'); }); - it('WORKFLOW_COMPRESSION_CODEC=gzip forces the portable codec', async () => { + it('WORKFLOW_COMPRESSION_CODEC=gzip forces gzip', async () => { process.env.WORKFLOW_COMPRESSION_CODEC = 'gzip'; const original = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, textEncoder.encode(JSON.stringify(makeCompressibleValue())) ) as Uint8Array; const stats: CompressionStats = {}; - const compressed = await compress(original, true, stats); + const compression = compress(original, true, stats); + expect(compression).not.toBeInstanceOf(Promise); + const compressed = await compression; expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.GZIP); expect(stats.codec).toBe('gzip'); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index 948a85c003..256127cb70 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -10,8 +10,7 @@ * compression runs at every step boundary so the write CPU is a per-step * tax. zstd requires `node:zlib` >= 22.15 (Web `CompressionStream` has no * zstd), so on a runtime without it we fall back to gzip via the portable - * `CompressionStream`. `WORKFLOW_COMPRESSION_CODEC=gzip` forces the - * portable codec. + * `CompressionStream`. `WORKFLOW_COMPRESSION_CODEC=gzip` forces gzip. * * Read side: dispatch on the format prefix, so both 'zstd' and 'gzip' * payloads are always decodable regardless of which codec wrote them. @@ -77,9 +76,8 @@ function isCompressionDisabledByEnv(): boolean { } /** - * Optional codec override (`WORKFLOW_COMPRESSION_CODEC=gzip|zstd`). Lets an - * operator pin the portable codec (gzip) — useful for A/B comparisons or - * runtimes where zstd read support isn't yet everywhere. + * Optional codec override (`WORKFLOW_COMPRESSION_CODEC=gzip|zstd`). Useful + * for A/B comparisons or runtimes where zstd read support isn't everywhere. */ function codecOverrideFromEnv(): 'gzip' | 'zstd' | undefined { try { @@ -105,6 +103,17 @@ const nodeZlib = (() => { } })(); +let zstdBrowserDecoder: + | ((payload: Uint8Array) => Promise) + | undefined; + +/** Register the zstd decoder used by browser observability clients. */ +export function registerZstdDecoder( + decoder: (payload: Uint8Array) => Promise +): void { + zstdBrowserDecoder = decoder; +} + function isZstdAvailable(): boolean { return ( typeof nodeZlib?.zstdCompressSync === 'function' && @@ -116,13 +125,9 @@ function isZstdAvailable(): boolean { * gzip via the web-standard `CompressionStream` (Node 18+, browsers, edge). */ function canCompressGzip(): boolean { - return typeof CompressionStream === 'function'; -} - -function canDecompressGzip(): boolean { return ( - typeof nodeZlib?.gunzipSync === 'function' || - typeof DecompressionStream === 'function' + typeof nodeZlib?.gzipSync === 'function' || + typeof CompressionStream === 'function' ); } @@ -160,45 +165,51 @@ async function pipeThroughTransform( return out; } -async function gzipBytes(data: Uint8Array): Promise { +function gzipBytes(data: Uint8Array): Promise { return pipeThroughTransform(data, new CompressionStream('gzip')); } -async function gunzipBytes(data: Uint8Array): Promise { - return pipeThroughTransform(data, new DecompressionStream('gzip')); +function gzip(data: Uint8Array): Uint8Array | Promise { + return nodeZlib?.gzipSync + ? new Uint8Array(nodeZlib.gzipSync(data)) + : gzipBytes(data); } -function gunzip(data: Uint8Array): Uint8Array | Promise { - if (nodeZlib?.gunzipSync) { - return new Uint8Array(nodeZlib.gunzipSync(data)); - } - if (typeof DecompressionStream === 'function') { - return gunzipBytes(data); - } - throw new Error( - 'Compressed (gzip) workflow data encountered but no gzip decoder is ' + - 'available in this runtime.' - ); +function gunzipBytes(data: Uint8Array): Promise { + return pipeThroughTransform(data, new DecompressionStream('gzip')); } function zstdBytes(data: Uint8Array): Uint8Array { + const compress = nodeZlib?.zstdCompressSync; + if (!compress) { + throw new Error('zstd compression is not available in this runtime'); + } const level = nodeZlib?.constants?.ZSTD_c_compressionLevel; const opts = level !== undefined ? { params: { [level]: ZSTD_LEVEL } } : undefined; - // biome-ignore lint/style/noNonNullAssertion: guarded by isZstdAvailable() - return new Uint8Array(nodeZlib!.zstdCompressSync!(data, opts)); + return new Uint8Array(compress(data, opts)); } -function unzstdBytes(data: Uint8Array): Uint8Array { - if (!nodeZlib?.zstdDecompressSync) { - throw new Error( - 'Compressed (zstd) workflow data encountered but node:zlib zstd ' + - 'support is not available in this runtime (requires Node.js 22.15+). ' + - 'In the browser, register a zstd decoder via registerZstdDecoder ' + - '(serialization-format.ts).' - ); +function decompressZstd(payload: Uint8Array): Uint8Array | Promise { + if (nodeZlib?.zstdDecompressSync) { + return new Uint8Array(nodeZlib.zstdDecompressSync(payload)); } - return new Uint8Array(nodeZlib.zstdDecompressSync(data)); + if (zstdBrowserDecoder) return zstdBrowserDecoder(payload); + throw new Error( + 'Compressed (zstd) workflow data encountered but no zstd decoder is ' + + 'available. Node.js 22.15+ decodes natively; in the browser ' + + 'register one via registerZstdDecoder.' + ); +} + +function decompressGzip(payload: Uint8Array): Uint8Array | Promise { + if (nodeZlib?.gunzipSync) { + return new Uint8Array(nodeZlib.gunzipSync(payload)); + } + if (typeof DecompressionStream === 'function') return gunzipBytes(payload); + throw new Error( + 'Compressed (gzip) workflow data encountered but no gzip decoder is available.' + ); } /** @@ -270,11 +281,11 @@ function selectWriteCodec(): 'zstd' | 'gzip' | 'none' { * @returns The compressed data with a codec prefix, or the original data * when compression is disabled, unavailable, or not worthwhile. */ -export async function compress( +export function compress( data: Uint8Array, enabled: boolean, stats?: CompressionStats -): Promise { +): Uint8Array | Promise { if ( !enabled || data.length < COMPRESSION_MIN_BYTES || @@ -290,16 +301,23 @@ export async function compress( return data; } - const compressed = codec === 'zstd' ? zstdBytes(data) : await gzipBytes(data); - const format = - codec === 'zstd' ? SerializationFormat.ZSTD : SerializationFormat.GZIP; - const wrappedLength = 4 + compressed.length; // format prefix + payload - if (wrappedLength >= data.length * (1 - COMPRESSION_MIN_SAVINGS_RATIO)) { - recordStats(stats, 'none', data.length, data.length); - return data; - } - recordStats(stats, codec, data.length, wrappedLength); - return encodeWithFormatPrefix(format, compressed); + const finish = (compressed: Uint8Array): Uint8Array => { + const wrappedLength = 4 + compressed.length; + if (wrappedLength >= data.length * (1 - COMPRESSION_MIN_SAVINGS_RATIO)) { + recordStats(stats, 'none', data.length, data.length); + return data; + } + recordStats(stats, codec, data.length, wrappedLength); + return encodeWithFormatPrefix( + codec === 'zstd' ? SerializationFormat.ZSTD : SerializationFormat.GZIP, + compressed + ); + }; + + const compressed = codec === 'zstd' ? zstdBytes(data) : gzip(data); + return compressed instanceof Promise + ? compressed.then(finish) + : finish(compressed); } /** @@ -315,30 +333,46 @@ export function decompress( ): Uint8Array | Promise { const prefix = peekFormatPrefix(data); - if (prefix === SerializationFormat.ZSTD) { - const { payload } = decodeFormatPrefix(data); - const inflated = unzstdBytes(payload); - recordStats(stats, 'zstd', inflated.length, data.length); - return inflated; + const codec = + prefix === SerializationFormat.ZSTD + ? 'zstd' + : prefix === SerializationFormat.GZIP + ? 'gzip' + : undefined; + if (!codec) { + recordStats(stats, 'none', data.length, data.length); + return data; } + const { payload } = decodeFormatPrefix(data); + const inflated = + codec === 'zstd' ? decompressZstd(payload) : decompressGzip(payload); + const finish = (value: Uint8Array): Uint8Array => { + recordStats(stats, codec, value.length, data.length); + return value; + }; + return inflated instanceof Promise ? inflated.then(finish) : finish(inflated); +} + +/** + * Decompress without starting a portable asynchronous codec. Observability's + * synchronous hydration path uses this to leave browser data untouched until + * its async hydration path is requested. + */ +export function decompressSync(data: Uint8Array): Uint8Array | undefined { + const prefix = peekFormatPrefix(data); + if (prefix === SerializationFormat.ZSTD) { + const decompress = nodeZlib?.zstdDecompressSync; + return decompress + ? new Uint8Array(decompress(decodeFormatPrefix(data).payload)) + : undefined; + } if (prefix === SerializationFormat.GZIP) { - if (!canDecompressGzip()) { - throw new Error('Compressed (gzip) workflow data cannot be decoded'); - } - const { payload } = decodeFormatPrefix(data); - const inflated = gunzip(payload); - if (inflated instanceof Promise) { - return inflated.then((result) => { - recordStats(stats, 'gzip', result.length, data.length); - return result; - }); - } - recordStats(stats, 'gzip', inflated.length, data.length); - return inflated; + const decompress = nodeZlib?.gunzipSync; + return decompress + ? new Uint8Array(decompress(decodeFormatPrefix(data).payload)) + : undefined; } - - recordStats(stats, 'none', data.length, data.length); return data; } diff --git a/packages/core/src/serialization/encryption.ts b/packages/core/src/serialization/encryption.ts index 531833795f..9818bc3f6c 100644 --- a/packages/core/src/serialization/encryption.ts +++ b/packages/core/src/serialization/encryption.ts @@ -181,8 +181,9 @@ export function aesKeyOf(key: PayloadKey | undefined): CryptoKey | undefined { } /** - * Encryption key parameter type. Accepts a resolved key, undefined (no encryption), - * a promise, or a resolver that can defer fetching the key until data needs it. + * A stream may receive a resolved key, an in-flight lookup, or a lazy lookup + * that should not start until the first frame arrives. It resolves this input + * only once. */ export type EncryptionKeyParam = | PayloadKey @@ -225,18 +226,6 @@ export async function encrypt( return encodeWithFormatPrefix(SerializationFormat.ENCRYPTED, encrypted); } -/** - * Decrypt a format-prefixed payload if it's encrypted or sealed. - * - * Strips the `encr`/`encp` format prefix and recovers the inner payload. - * Opening a sealed (`encp`) payload requires the run's X25519 keypair, so the - * caller must supply {@link RunPayloadKeys} — a bare symmetric key cannot do - * it, and neither can a {@link SealTarget} (which is write-only by design). - * - * @param data - The potentially encrypted data - * @param key - Encryption key (undefined to skip decryption) - * @returns The decrypted inner payload, or the original data if not encrypted - */ /** * Open a sealed (`encp`) envelope. Split out from {@link decrypt} to keep * each scheme's error handling readable on its own. @@ -290,6 +279,10 @@ function requireAesDecryptionKey( ); } +/** + * Decrypt a format-prefixed payload if it is encrypted or sealed. + * Sealed payloads require the owning run's X25519 keypair. + */ export async function decrypt( data: Uint8Array, key: DecryptionKey | undefined diff --git a/packages/core/src/serialization/payload.ts b/packages/core/src/serialization/payload.ts new file mode 100644 index 0000000000..d62b03cbf9 --- /dev/null +++ b/packages/core/src/serialization/payload.ts @@ -0,0 +1,26 @@ +import { type CompressionStats, compress, decompress } from './compression.js'; +import { + type DecryptionKey, + decrypt, + encrypt, + type PayloadKey, +} from './encryption.js'; + +/** Apply the storage layers in their only valid order: compress, then encrypt. */ +export async function encodePayload( + data: Uint8Array, + key: PayloadKey | undefined, + compression: boolean, + stats?: CompressionStats +): Promise { + return encrypt(await compress(data, compression, stats), key); +} + +/** Remove the storage layers in reverse order: decrypt, then decompress. */ +export async function decodePayload( + data: Uint8Array, + key: DecryptionKey | undefined, + stats?: CompressionStats +): Promise { + return decompress(await decrypt(data, key), stats); +} diff --git a/packages/core/src/serialization/replay.ts b/packages/core/src/serialization/replay.ts new file mode 100644 index 0000000000..dda10d456f --- /dev/null +++ b/packages/core/src/serialization/replay.ts @@ -0,0 +1,40 @@ +import { type CompressionStats, decompress } from './compression.js'; +import { + type DecryptionKey, + decrypt, + decryptReplayPayload, +} from './encryption.js'; +import { peekFormatPrefix } from './format.js'; +import { SerializationFormat } from './types.js'; + +/** Legacy payloads need no byte preparation but still share the hydrate API. */ +export interface LegacyReplayPayload { + readonly legacy: unknown; +} + +/** Host-owned bytes, or a tagged legacy value from before binary envelopes. */ +export type PreparedReplayPayload = Uint8Array | LegacyReplayPayload; + +export type ReplayPayloadPreparer = ( + value: Uint8Array, + key: DecryptionKey | undefined +) => Uint8Array | Promise; + +/** + * Decrypt and decompress persisted bytes without creating VM-owned values. + * + * AES-GCM, zstd, and Node gzip complete synchronously. Sealed envelopes and + * portable browser gzip return a Promise because their underlying codecs do. + */ +export function prepareReplayPayload( + value: Uint8Array, + key: DecryptionKey | undefined, + compressionStats?: CompressionStats +): Uint8Array | Promise { + const decompressPayload = (decrypted: Uint8Array) => + decompress(decrypted, compressionStats); + + return peekFormatPrefix(value) === SerializationFormat.SEALED + ? decrypt(value, key).then(decompressPayload) + : decompressPayload(decryptReplayPayload(value, key)); +} diff --git a/packages/core/src/serialization/serialization.test.ts b/packages/core/src/serialization/serialization.test.ts index 6b16bc36ce..e753dea885 100644 --- a/packages/core/src/serialization/serialization.test.ts +++ b/packages/core/src/serialization/serialization.test.ts @@ -282,12 +282,6 @@ describe('decrypt', () => { expect(decrypted).toEqual(data); }); - it('should return non-binary data unchanged', async () => { - const data = [1, 2, 3]; - const result = await decrypt(data, undefined); - expect(result).toBe(data); - }); - it('should return non-encrypted binary data unchanged', async () => { const data = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, diff --git a/packages/core/src/serialization/step.ts b/packages/core/src/serialization/step.ts index 94d5a9acf6..efcb03e878 100644 --- a/packages/core/src/serialization/step.ts +++ b/packages/core/src/serialization/step.ts @@ -8,15 +8,10 @@ import { SerializationError } from '@workflow/errors'; import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; -import { compress, decompress } from './compression.js'; -import { - type DecryptionKey, - decrypt as decryptData, - encrypt as encryptData, - type PayloadKey, -} from './encryption.js'; +import type { DecryptionKey, PayloadKey } from './encryption.js'; import { formatSerializationError, rethrowIfRuntimeError } from './errors.js'; import { decodeFormatPrefix, encodeWithFormatPrefix } from './format.js'; +import { decodePayload, encodePayload } from './payload.js'; import { SerializationFormat } from './types.js'; /** @@ -33,13 +28,12 @@ export async function serialize( SerializationFormat.DEVALUE_V1, payload ); - // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress( + return await encodePayload( prefixed, - options?.compression === true, + encryptionKey, + options?.compression ?? false, options?.compressionStats ); - return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); const { message, hint } = formatSerializationError('step value', error); @@ -64,8 +58,11 @@ export async function deserialize( ); } - const decrypted = await decryptData(data, encryptionKey); - const prepared = await decompress(decrypted, options?.compressionStats); + const prepared = await decodePayload( + data, + encryptionKey, + options?.compressionStats + ); const { format, payload } = decodeFormatPrefix(prepared); if (format === SerializationFormat.DEVALUE_V1) { diff --git a/packages/world-vercel/src/serialized-data.test.ts b/packages/world-vercel/src/serialized-data.test.ts new file mode 100644 index 0000000000..47a05af53e --- /dev/null +++ b/packages/world-vercel/src/serialized-data.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import { + hasSerializedDataFormatPrefix, + normalizeSerializedData, +} from './serialized-data.js'; + +const encoder = new TextEncoder(); + +function envelope(format: string, payload: Uint8Array): Uint8Array { + const data = new Uint8Array(4 + payload.length); + data.set(encoder.encode(format)); + data.set(payload, 4); + return data; +} + +describe('serialized data normalization', () => { + it.each([ + 'devl', + 'encr', + 'encp', + 'gzip', + 'zstd', + ])('recognizes the %s envelope', (format) => { + expect( + hasSerializedDataFormatPrefix(envelope(format, new Uint8Array())) + ).toBe(true); + }); + + it('leaves non-compressed envelopes untouched', () => { + const encrypted = envelope('encr', new Uint8Array([1, 2, 3])); + expect(normalizeSerializedData(encrypted)).toBe(encrypted); + }); +}); diff --git a/packages/world-vercel/src/serialized-data.ts b/packages/world-vercel/src/serialized-data.ts index 8f17150fd4..1885f430ce 100644 --- a/packages/world-vercel/src/serialized-data.ts +++ b/packages/world-vercel/src/serialized-data.ts @@ -1,35 +1,25 @@ import { WorkflowWorldError } from '@workflow/errors'; const FORMAT_PREFIX_LENGTH = 4; -const DEVALUE_FORMAT_PREFIX = 'devl'; -const ENCRYPTED_FORMAT_PREFIX = 'encr'; const GZIP_FORMAT_PREFIX = 'gzip'; const ZSTD_FORMAT_PREFIX = 'zstd'; const formatDecoder = new TextDecoder(); const SERIALIZED_DATA_FORMAT_PREFIXES = new Set([ - DEVALUE_FORMAT_PREFIX, - ENCRYPTED_FORMAT_PREFIX, + 'devl', + 'encr', + 'encp', GZIP_FORMAT_PREFIX, ZSTD_FORMAT_PREFIX, ]); -interface NodeZlibDecode { - gunzipSync?: (data: Uint8Array) => Uint8Array; - zstdDecompressSync?: (data: Uint8Array) => Uint8Array; -} - -function getNodeZlib(): NodeZlibDecode | undefined { +const nodeZlib = (() => { try { - return ( - globalThis as { - process?: { getBuiltinModule?: (id: string) => NodeZlibDecode }; - } - ).process?.getBuiltinModule?.('node:zlib'); + return process.getBuiltinModule('node:zlib'); } catch { return undefined; } -} +})(); function peekFormatPrefix(value: unknown): string | null { if ( @@ -47,9 +37,10 @@ export function hasSerializedDataFormatPrefix(value: unknown): boolean { } function decompress(format: string, payload: Uint8Array): Uint8Array { - const zlib = getNodeZlib(); const decompress = - format === ZSTD_FORMAT_PREFIX ? zlib?.zstdDecompressSync : zlib?.gunzipSync; + format === ZSTD_FORMAT_PREFIX + ? nodeZlib?.zstdDecompressSync + : nodeZlib?.gunzipSync; if (!decompress) { throw new WorkflowWorldError( From 64dbc2c3693babb90f9928462b5668d64d97cbb0 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:00:21 -0700 Subject: [PATCH 06/16] [core] Avoid native codec result copies --- packages/core/src/serialization/compression.ts | 17 +++++++++++------ packages/world-vercel/src/serialized-data.ts | 3 ++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index 256127cb70..e44406e73c 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -103,6 +103,11 @@ const nodeZlib = (() => { } })(); +/** Return a plain Uint8Array view without copying a Node Buffer's bytes. */ +function asUint8Array(data: Uint8Array): Uint8Array { + return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); +} + let zstdBrowserDecoder: | ((payload: Uint8Array) => Promise) | undefined; @@ -171,7 +176,7 @@ function gzipBytes(data: Uint8Array): Promise { function gzip(data: Uint8Array): Uint8Array | Promise { return nodeZlib?.gzipSync - ? new Uint8Array(nodeZlib.gzipSync(data)) + ? asUint8Array(nodeZlib.gzipSync(data)) : gzipBytes(data); } @@ -187,12 +192,12 @@ function zstdBytes(data: Uint8Array): Uint8Array { const level = nodeZlib?.constants?.ZSTD_c_compressionLevel; const opts = level !== undefined ? { params: { [level]: ZSTD_LEVEL } } : undefined; - return new Uint8Array(compress(data, opts)); + return asUint8Array(compress(data, opts)); } function decompressZstd(payload: Uint8Array): Uint8Array | Promise { if (nodeZlib?.zstdDecompressSync) { - return new Uint8Array(nodeZlib.zstdDecompressSync(payload)); + return asUint8Array(nodeZlib.zstdDecompressSync(payload)); } if (zstdBrowserDecoder) return zstdBrowserDecoder(payload); throw new Error( @@ -204,7 +209,7 @@ function decompressZstd(payload: Uint8Array): Uint8Array | Promise { function decompressGzip(payload: Uint8Array): Uint8Array | Promise { if (nodeZlib?.gunzipSync) { - return new Uint8Array(nodeZlib.gunzipSync(payload)); + return asUint8Array(nodeZlib.gunzipSync(payload)); } if (typeof DecompressionStream === 'function') return gunzipBytes(payload); throw new Error( @@ -364,13 +369,13 @@ export function decompressSync(data: Uint8Array): Uint8Array | undefined { if (prefix === SerializationFormat.ZSTD) { const decompress = nodeZlib?.zstdDecompressSync; return decompress - ? new Uint8Array(decompress(decodeFormatPrefix(data).payload)) + ? asUint8Array(decompress(decodeFormatPrefix(data).payload)) : undefined; } if (prefix === SerializationFormat.GZIP) { const decompress = nodeZlib?.gunzipSync; return decompress - ? new Uint8Array(decompress(decodeFormatPrefix(data).payload)) + ? asUint8Array(decompress(decodeFormatPrefix(data).payload)) : undefined; } return data; diff --git a/packages/world-vercel/src/serialized-data.ts b/packages/world-vercel/src/serialized-data.ts index 1885f430ce..e0a249d50c 100644 --- a/packages/world-vercel/src/serialized-data.ts +++ b/packages/world-vercel/src/serialized-data.ts @@ -48,7 +48,8 @@ function decompress(format: string, payload: Uint8Array): Uint8Array { ); } - return new Uint8Array(decompress(payload)); + const result = decompress(payload); + return new Uint8Array(result.buffer, result.byteOffset, result.byteLength); } export function normalizeSerializedData(value: unknown): unknown { From 3b683abcbc28f161d05ccdc536c15e7b63495e06 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:03:30 -0700 Subject: [PATCH 07/16] [core] Reuse loaded encryption helpers --- packages/core/src/serialization-format.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index 28947f53eb..a485af295c 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -13,6 +13,11 @@ import { decompressSync, registerZstdDecoder, } from './serialization/compression.js'; +import { + type DecryptionKey, + decrypt, + isRunPayloadKeys, +} from './serialization/encryption.js'; import { decodeFormatPrefix as decodePrefix, encodeWithFormatPrefix, @@ -34,11 +39,8 @@ import { SerializationFormat } from './serialization/types.js'; * `encryption.ts` and `sealed-box.ts` are all free of Node dependencies. */ export { - type DecryptionKey, - decrypt as decryptEnvelope, deriveRunPayloadKeys, encrypt as encryptEnvelope, - isRunPayloadKeys, isSealTarget, type PayloadKey, type RunPayloadKeys, @@ -46,6 +48,7 @@ export { type SealTarget, sealTo, } from './serialization/encryption.js'; +export { type DecryptionKey, decrypt as decryptEnvelope, isRunPayloadKeys }; // --------------------------------------------------------------------------- // Format prefix constants and encoding/decoding @@ -259,16 +262,13 @@ export function hydrateData(value: unknown, revivers: Revivers): unknown { export async function hydrateDataWithKey( value: unknown, revivers: Revivers, - key: import('./serialization/encryption.js').DecryptionKey | undefined + key: DecryptionKey | undefined ): Promise { let data = value; if (data instanceof Uint8Array && isEncryptedData(data) && key) { // Envelope-aware decrypt: handles both `encr` (AES-GCM under the run's // symmetric key) and `encp` (sealed to the run's X25519 public key by // some other run), dispatching on the format prefix. - const { decrypt, isRunPayloadKeys } = await import( - './serialization/encryption.js' - ); // Opening a sealed payload needs the run's private scalar. If the caller // supplied only a symmetric key, leave the bytes as ciphertext so the UI // keeps showing its "Encrypted" affordance, rather than surfacing a From 55acabad6ec49c2e6028017b0f0879cfedfb9116 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:09:51 -0700 Subject: [PATCH 08/16] [core] Fix legacy replay payload tagging --- packages/core/src/replay-payload-cache.test.ts | 16 +++++++--------- packages/core/src/replay-payload-cache.ts | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 2eb0858572..04484533b5 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -57,14 +57,14 @@ function makeEvents(payloads: unknown[]): Event[] { describe('ReplayPayloadCache', () => { it('deduplicates preparation and accepts a synchronous preparer', async () => { const payload = new Uint8Array([1]); - const preparer = vi.fn((value) => ({ data: value })); + const preparer = vi.fn((value) => value); const cache = new ReplayPayloadCache(undefined, preparer); const first = cache.prepareEventPayload('evnt_one', 'result', payload); const second = cache.prepareEventPayload('evnt_one', 'result', payload); expect(first).toBe(second); - await expect(first).resolves.toEqual({ data: payload }); + await expect(first).resolves.toBe(payload); expect(preparer).toHaveBeenCalledOnce(); }); @@ -74,7 +74,7 @@ describe('ReplayPayloadCache', () => { const preparer = vi .fn() .mockRejectedValueOnce(new Error('decrypt failed')) - .mockReturnValueOnce({ data: payload }); + .mockReturnValueOnce(payload); const cache = new ReplayPayloadCache(undefined, preparer); await cache.prewarm(run, []); @@ -83,9 +83,7 @@ describe('ReplayPayloadCache', () => { ); expect(preparer).toHaveBeenCalledOnce(); - await expect(cache.prepareWorkflowInput(run)).resolves.toEqual({ - data: payload, - }); + await expect(cache.prepareWorkflowInput(run)).resolves.toBe(payload); expect(preparer).toHaveBeenCalledTimes(2); }); @@ -95,7 +93,7 @@ describe('ReplayPayloadCache', () => { const preparer = vi.fn( (value) => new Promise((resolve) => { - resolvers.push(() => resolve({ data: value })); + resolvers.push(() => resolve(value)); }) ); const cache = new ReplayPayloadCache(undefined, preparer); @@ -158,7 +156,7 @@ describe('ReplayPayloadCache', () => { // shift every later position. Resuming from that length skips exactly the // events the reload was for, which is what `resetScan` exists to prevent. const payloads = [0, 1, 2].map((value) => new Uint8Array([value])); - const preparer = vi.fn((value) => ({ data: value })); + const preparer = vi.fn((value) => value); const cache = new ReplayPayloadCache(undefined, preparer); const run = makeRun(undefined); const [first, missing, second] = makeEvents(payloads); @@ -181,7 +179,7 @@ describe('ReplayPayloadCache', () => { it('bypasses legacy values and ignores missing event data during prewarm', async () => { const legacy = [0, { value: 1 }]; - const preparer = vi.fn((value) => ({ data: value })); + const preparer = vi.fn((value) => value); const cache = new ReplayPayloadCache(undefined, preparer); await cache.prepareEventPayload('evnt_legacy', 'result', legacy); diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 8f9cfac990..2b90fb3570 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -174,7 +174,7 @@ export class ReplayPayloadCache { value: unknown ): Promise { if (!(value instanceof Uint8Array)) { - return Promise.resolve({ data: value }); + return Promise.resolve({ legacy: value }); } const preparation = this.ensurePreparation(cacheKey, value); From 94e1f8f918290983973e75d4815dde6d30928534 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:21:05 -0700 Subject: [PATCH 09/16] [core] Consolidate replay decryption --- packages/core/src/encryption.test.ts | 83 ++++++-------- packages/core/src/encryption.ts | 107 ++++++++---------- packages/core/src/runtime/helpers.test.ts | 4 +- packages/core/src/sealed-box.test.ts | 2 +- packages/core/src/serialization.test.ts | 8 +- packages/core/src/serialization/encryption.ts | 41 ++----- packages/core/src/serialization/replay.ts | 16 +-- .../src/serialization/serialization.test.ts | 22 ++-- 8 files changed, 119 insertions(+), 164 deletions(-) diff --git a/packages/core/src/encryption.test.ts b/packages/core/src/encryption.test.ts index 937463ce82..d981bebec1 100644 --- a/packages/core/src/encryption.test.ts +++ b/packages/core/src/encryption.test.ts @@ -1,12 +1,6 @@ import { RuntimeDecryptionError } from '@workflow/errors'; import { describe, expect, it } from 'vitest'; -import { - type CryptoKey, - decrypt, - decryptSync, - encrypt, - importKey, -} from './encryption.js'; +import { type CryptoKey, decrypt, encrypt, importKey } from './encryption.js'; const RAW_KEY = new Uint8Array(32).fill(7); const OTHER_RAW_KEY = new Uint8Array(32).fill(8); @@ -20,9 +14,18 @@ async function getOtherKey(): Promise { return importKey(OTHER_RAW_KEY); } +async function captureError(action: () => unknown): Promise { + try { + await action(); + } catch (error) { + return error; + } + throw new Error('Expected operation to fail'); +} + describe('encryption', () => { describe('round-trip', () => { - it('encrypt() + decrypt() returns the original plaintext', async () => { + it('decrypts synchronously on Node', async () => { const key = await getKey(); const plaintext = new TextEncoder().encode('hello, workflow'); const ciphertext = await encrypt(key, plaintext); @@ -30,18 +33,12 @@ describe('encryption', () => { // Ciphertext is longer than plaintext: 12-byte nonce + 16-byte GCM tag. expect(ciphertext.byteLength).toBe(plaintext.byteLength + 12 + 16); - const decoded = await decrypt(key, ciphertext); - expect(new TextDecoder().decode(decoded)).toBe('hello, workflow'); - }); - - it('decryptSync() returns the plaintext without a promise', async () => { - const key = await getKey(); - const plaintext = new TextEncoder().encode('synchronous replay'); - const ciphertext = await encrypt(key, plaintext); - - const decoded = decryptSync(key, ciphertext); + const decoded = decrypt(key, ciphertext); + expect(decoded).not.toBeInstanceOf(Promise); expect(decoded).toBeInstanceOf(Uint8Array); - expect(new TextDecoder().decode(decoded)).toBe('synchronous replay'); + expect(new TextDecoder().decode(decoded as Uint8Array)).toBe( + 'hello, workflow' + ); }); }); @@ -58,20 +55,15 @@ describe('encryption', () => { const key = await getKey(); // 12-byte nonce + 16-byte tag = 28 bytes minimum. 10 bytes is too short. const tooShort = new Uint8Array(10).fill(0); - const error = await decrypt(key, tooShort).catch((e) => e); + const error = await captureError(() => decrypt(key, tooShort)); expect(RuntimeDecryptionError.is(error)).toBe(true); - expect(error.message).toMatch(/Encrypted data too short/); - expect(error.context).toMatchObject({ - operation: 'decrypt', - byteLength: 10, + expect(error).toMatchObject({ + message: expect.stringMatching(/Encrypted data too short/), + context: { operation: 'decrypt', byteLength: 10 }, }); }); it('throws RuntimeDecryptionError (not a bare OperationError) on auth-tag failure', async () => { - // GCM auth-tag verification failure surfaces from Node's Web - // Crypto API as `OperationError: The operation failed for an - // operation-specific reason at AESCipherJob.onDone`. The - // encryption module must rewrap this as a RuntimeDecryptionError. const key = await getKey(); const plaintext = new TextEncoder().encode('hello, workflow'); const ciphertext = await encrypt(key, plaintext); @@ -80,16 +72,14 @@ describe('encryption', () => { const tampered = new Uint8Array(ciphertext); tampered[tampered.length - 1] ^= 0xff; - const error = await decrypt(key, tampered).catch((e) => e); + const error = await captureError(() => decrypt(key, tampered)); expect(RuntimeDecryptionError.is(error)).toBe(true); - expect(error.cause).toBeDefined(); - // The original DOMException carries name OperationError on Node 20+, - // which is what the wrapping is meant to capture as cause. - const cause = error.cause as { name?: string }; - expect(cause?.name).toBe('OperationError'); - expect(error.context).toMatchObject({ - operation: 'decrypt', - byteLength: tampered.byteLength, + expect(error).toMatchObject({ + cause: expect.anything(), + context: { + operation: 'decrypt', + byteLength: tampered.byteLength, + }, }); }); @@ -101,11 +91,9 @@ describe('encryption', () => { new TextEncoder().encode('secret') ); - const error = await decrypt(readerKey, ciphertext).catch((e) => e); + const error = await captureError(() => decrypt(readerKey, ciphertext)); expect(RuntimeDecryptionError.is(error)).toBe(true); - // Wrong key → auth tag mismatch → same OperationError as ciphertext corruption. - const cause = error.cause as { name?: string }; - expect(cause?.name).toBe('OperationError'); + expect(error).toMatchObject({ cause: expect.anything() }); }); it('keeps RuntimeDecryptionError on synchronous auth failure', async () => { @@ -116,7 +104,7 @@ describe('encryption', () => { ); ciphertext[ciphertext.length - 1] ^= 0xff; - expect(() => decryptSync(key, ciphertext)).toThrowError( + expect(() => decrypt(key, ciphertext)).toThrowError( RuntimeDecryptionError ); }); @@ -129,13 +117,14 @@ describe('encryption', () => { // The serialization layer attaches the real envelope prefix. const key = await getKey(); const bogus = new Uint8Array(28).fill(0x41); // 28 bytes, passes length check - const error = await decrypt(key, bogus).catch((e) => e); + const error = await captureError(() => decrypt(key, bogus)); expect(RuntimeDecryptionError.is(error)).toBe(true); - expect(error.context).toMatchObject({ - operation: 'decrypt', - byteLength: 28, + expect(error).toMatchObject({ + context: { operation: 'decrypt', byteLength: 28 }, + }); + expect(error).not.toMatchObject({ + context: { formatPrefix: expect.anything() }, }); - expect(error.context.formatPrefix).toBeUndefined(); }); }); diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index d9ad2629fa..54aaa34149 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -1,11 +1,11 @@ import { RuntimeDecryptionError, WorkflowRuntimeError } from '@workflow/errors'; /** - * Browser-compatible AES-256-GCM encryption module. + * Portable AES-256-GCM encryption with a synchronous Node decrypt path. * - * Uses the Web Crypto API (`globalThis.crypto.subtle`) which works in - * both modern browsers and Node.js 20+. This module is intentionally - * free of Node.js-specific imports so it can be bundled for the browser. + * Key import, encryption, and the browser fallback use Web Crypto. On Node, + * importKey also retains a native key handle so decrypt can call OpenSSL + * synchronously without making the module unsafe to bundle for browsers. * * The World interface (`getEncryptionKeyForRun`) returns a raw 32-byte * AES-256 key. Callers should use `importKey()` once to convert it to a @@ -115,51 +115,20 @@ function wrapDecryptionError(cause: unknown, byteLength: number): never { ); } -/** - * Decrypt AES-256-GCM synchronously when running on Node and the key was - * imported by this module. - * - * The key must have been created by this module's {@link importKey}, and the - * Node synchronous crypto API must be available. Authentication failures throw - * the same RuntimeDecryptionError shape as {@link decrypt}. - */ -export function decryptSync( - key: CryptoKey, +function decryptWithNode( + crypto: typeof import('node:crypto'), + nodeKey: import('node:crypto').KeyObject, data: Uint8Array, aad?: Uint8Array ): Uint8Array { - const nodeKey = nodeKeys.get(key); - if (!nodeKey) { - throw new WorkflowRuntimeError( - 'Synchronous AES-256-GCM decryption requires a key created by importKey()' - ); - } - if (!nodeCrypto) { - throw new WorkflowRuntimeError( - 'Synchronous AES-256-GCM decryption requires the Node.js crypto module' - ); - } - if (!key.usages.includes('decrypt')) { - throw new RuntimeDecryptionError( - 'AES-256-GCM decryption failed: CryptoKey does not support decrypt', - { - context: { operation: 'decrypt', byteLength: data.byteLength }, - } - ); - } - - assertValidAesGcmEnvelope(data); const ciphertextEnd = data.byteLength - TAG_BYTES; const nonce = data.subarray(0, NONCE_LENGTH); const ciphertext = data.subarray(NONCE_LENGTH, ciphertextEnd); const authTag = data.subarray(ciphertextEnd); try { - const decipher = nodeCrypto.createDecipheriv( - 'aes-256-gcm', - nodeKey, - nonce, - { authTagLength: TAG_BYTES } - ); + const decipher = crypto.createDecipheriv('aes-256-gcm', nodeKey, nonce, { + authTagLength: TAG_BYTES, + }); if (aad) decipher.setAAD(aad); decipher.setAuthTag(authTag); const head = decipher.update(ciphertext); @@ -176,6 +145,30 @@ export function decryptSync( } } +async function decryptWithWebCrypto( + key: CryptoKey, + data: Uint8Array, + aad?: Uint8Array +): Promise { + const nonce = data.subarray(0, NONCE_LENGTH); + const ciphertextAndTag = data.subarray(NONCE_LENGTH); + try { + const plaintext = await globalThis.crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: nonce, + tagLength: TAG_LENGTH, + ...(aad ? { additionalData: aad } : {}), + }, + key, + ciphertextAndTag + ); + return new Uint8Array(plaintext); + } catch (cause) { + wrapDecryptionError(cause, data.byteLength); + } +} + /** * Encrypt data using AES-256-GCM. * @@ -249,30 +242,26 @@ export async function encrypt( * @param aad - Optional additional authenticated data. Must match the bytes * passed to {@link encrypt} exactly, otherwise the GCM tag fails to verify * and a {@link RuntimeDecryptionError} is thrown. - * @returns Decrypted plaintext + * @returns Plaintext directly on Node, or a Promise from the Web Crypto + * fallback. */ -export async function decrypt( +export function decrypt( key: CryptoKey, data: Uint8Array, aad?: Uint8Array -): Promise { +): Uint8Array | Promise { assertValidAesGcmEnvelope(data); - const nonce = data.subarray(0, NONCE_LENGTH); - const ciphertextAndTag = data.subarray(NONCE_LENGTH); - let plaintext: ArrayBuffer; - try { - plaintext = await globalThis.crypto.subtle.decrypt( + if (!key.usages.includes('decrypt')) { + throw new RuntimeDecryptionError( + 'AES-256-GCM decryption failed: CryptoKey does not support decrypt', { - name: 'AES-GCM', - iv: nonce, - tagLength: TAG_LENGTH, - ...(aad ? { additionalData: aad } : {}), - }, - key, - ciphertextAndTag + context: { operation: 'decrypt', byteLength: data.byteLength }, + } ); - } catch (cause) { - wrapDecryptionError(cause, data.byteLength); } - return new Uint8Array(plaintext); + + const nodeKey = nodeKeys.get(key); + return nodeCrypto && nodeKey + ? decryptWithNode(nodeCrypto, nodeKey, data, aad) + : decryptWithWebCrypto(key, data, aad); } diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 21852fac4b..0ce474ef07 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -941,7 +941,7 @@ describe('memoizeEncryptionKey', () => { sealed ) as Uint8Array; - await expect(decrypt(prefixed, resolved)).resolves.toEqual( + expect(await decrypt(prefixed, resolved)).toEqual( new TextEncoder().encode('"hi"') ); }); @@ -955,7 +955,7 @@ describe('memoizeEncryptionKey', () => { const encrypted = await encrypt(new TextEncoder().encode('"hi"'), resolved); expect(peekFormatPrefix(encrypted)).toBe(SerializationFormat.ENCRYPTED); - await expect(decrypt(encrypted, resolved)).resolves.toEqual( + expect(await decrypt(encrypted, resolved)).toEqual( new TextEncoder().encode('"hi"') ); }); diff --git a/packages/core/src/sealed-box.test.ts b/packages/core/src/sealed-box.test.ts index 31d5b6875f..535033d594 100644 --- a/packages/core/src/sealed-box.test.ts +++ b/packages/core/src/sealed-box.test.ts @@ -144,7 +144,7 @@ describe('sealed-box', () => { const encrypted = await aesGcmEncrypt(contentKey, plaintext()); // Even holding the very content key it just used, the writer cannot // reverse the operation: the CryptoKey carries no 'decrypt' usage. - await expect(aesGcmDecrypt(contentKey, encrypted)).rejects.toThrow( + expect(() => aesGcmDecrypt(contentKey, encrypted)).toThrow( RuntimeDecryptionError ); }); diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index d354a2d635..38bd63a2b1 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -6045,7 +6045,7 @@ describe('encrypt/decrypt primitives', () => { it('should reject truncated ciphertext', async () => { const tooShort = new Uint8Array(10); // Less than nonce (12) + auth tag (16) - await expect(decrypt(testKey, tooShort)).rejects.toThrow( + expect(() => decrypt(testKey, tooShort)).toThrow( 'Encrypted data too short' ); }); @@ -6054,7 +6054,7 @@ describe('encrypt/decrypt primitives', () => { const data = new TextEncoder().encode('secret'); const encrypted = await encrypt(testKey, data); const wrongKey = await importKey(new Uint8Array(32).fill(0xff)); - await expect(decrypt(wrongKey, encrypted)).rejects.toThrow(); + expect(() => decrypt(wrongKey, encrypted)).toThrow(); }); it('should fail with tampered ciphertext', async () => { @@ -6062,7 +6062,7 @@ describe('encrypt/decrypt primitives', () => { const encrypted = await encrypt(testKey, data); // Flip a byte in the ciphertext (past the nonce) encrypted[15] ^= 0xff; - await expect(decrypt(testKey, encrypted)).rejects.toThrow(); + expect(() => decrypt(testKey, encrypted)).toThrow(); }); }); @@ -6112,7 +6112,7 @@ describe('serialized payload encryption', () => { it('should throw when encrypted data has no key', async () => { const data = new Uint8Array([1, 2, 3]); const encrypted = await encryptEnvelope(data, testKey); - await expect(decryptEnvelope(encrypted, undefined)).rejects.toThrow( + expect(() => decryptEnvelope(encrypted, undefined)).toThrow( 'Encrypted data encountered but no encryption key' ); }); diff --git a/packages/core/src/serialization/encryption.ts b/packages/core/src/serialization/encryption.ts index 9818bc3f6c..8437345a55 100644 --- a/packages/core/src/serialization/encryption.ts +++ b/packages/core/src/serialization/encryption.ts @@ -10,7 +10,6 @@ import { RuntimeDecryptionError } from '@workflow/errors'; import { decrypt as aesGcmDecrypt, - decryptSync as aesGcmDecryptSync, encrypt as aesGcmEncrypt, type CryptoKey, importKey as importAesKey, @@ -281,12 +280,13 @@ function requireAesDecryptionKey( /** * Decrypt a format-prefixed payload if it is encrypted or sealed. - * Sealed payloads require the owning run's X25519 keypair. + * Node AES decrypts synchronously. Sealed payloads and portable Web Crypto + * fallbacks return a Promise. */ -export async function decrypt( +export function decrypt( data: Uint8Array, key: DecryptionKey | undefined -): Promise { +): Uint8Array | Promise { const format = peekFormatPrefix(data); if (format === SerializationFormat.SEALED) { @@ -300,7 +300,12 @@ export async function decrypt( const { payload } = decodeFormatPrefix(data); try { - return await aesGcmDecrypt(aesKey, payload); + const decrypted = aesGcmDecrypt(aesKey, payload); + return decrypted instanceof Promise + ? decrypted.catch((error) => { + throw addFormatPrefix(error, format); + }) + : decrypted; } catch (error) { throw addFormatPrefix(error, format); } @@ -312,29 +317,3 @@ function addFormatPrefix(error: unknown, format: string): unknown { } return error; } - -/** - * Replay-specialized decrypt path. - * - * This byte-only function handles plaintext and symmetric envelopes without a - * promise. The replay preparation boundary dispatches sealed envelopes to the - * asynchronous X25519 decryptor before calling this function. - */ -export function decryptReplayPayload( - data: Uint8Array, - key: DecryptionKey | undefined -): Uint8Array { - const format = peekFormatPrefix(data); - if (format === SerializationFormat.SEALED) { - throw new Error('Sealed replay payloads require asynchronous decryption'); - } - if (format !== SerializationFormat.ENCRYPTED) return data; - - const aesKey = requireAesDecryptionKey(data, key); - const { payload } = decodeFormatPrefix(data); - try { - return aesGcmDecryptSync(aesKey, payload); - } catch (error) { - throw addFormatPrefix(error, format); - } -} diff --git a/packages/core/src/serialization/replay.ts b/packages/core/src/serialization/replay.ts index dda10d456f..492e7aa5cd 100644 --- a/packages/core/src/serialization/replay.ts +++ b/packages/core/src/serialization/replay.ts @@ -1,11 +1,5 @@ import { type CompressionStats, decompress } from './compression.js'; -import { - type DecryptionKey, - decrypt, - decryptReplayPayload, -} from './encryption.js'; -import { peekFormatPrefix } from './format.js'; -import { SerializationFormat } from './types.js'; +import { type DecryptionKey, decrypt } from './encryption.js'; /** Legacy payloads need no byte preparation but still share the hydrate API. */ export interface LegacyReplayPayload { @@ -33,8 +27,8 @@ export function prepareReplayPayload( ): Uint8Array | Promise { const decompressPayload = (decrypted: Uint8Array) => decompress(decrypted, compressionStats); - - return peekFormatPrefix(value) === SerializationFormat.SEALED - ? decrypt(value, key).then(decompressPayload) - : decompressPayload(decryptReplayPayload(value, key)); + const decrypted = decrypt(value, key); + return decrypted instanceof Promise + ? decrypted.then(decompressPayload) + : decompressPayload(decrypted); } diff --git a/packages/core/src/serialization/serialization.test.ts b/packages/core/src/serialization/serialization.test.ts index e753dea885..8aba21f45a 100644 --- a/packages/core/src/serialization/serialization.test.ts +++ b/packages/core/src/serialization/serialization.test.ts @@ -9,7 +9,6 @@ import { devalueCodec } from './codec-devalue.js'; import { aesKeyOf, decrypt, - decryptReplayPayload, encrypt, isRunPayloadKeys, isSealTarget, @@ -269,7 +268,7 @@ describe('encrypt', () => { }); describe('decrypt', () => { - it('decrypts replay payloads synchronously', async () => { + it('decrypts AES envelopes synchronously on Node', async () => { const key = await makeKey(); const data = encodeWithFormatPrefix( SerializationFormat.DEVALUE_V1, @@ -277,7 +276,8 @@ describe('decrypt', () => { ) as Uint8Array; const encrypted = (await encrypt(data, key)) as Uint8Array; - const decrypted = decryptReplayPayload(encrypted, key); + const decrypted = decrypt(encrypted, key); + expect(decrypted).not.toBeInstanceOf(Promise); expect(decrypted).toBeInstanceOf(Uint8Array); expect(decrypted).toEqual(data); }); @@ -300,7 +300,7 @@ describe('decrypt', () => { ) as Uint8Array; const encrypted = await encrypt(data, key); - await expect(decrypt(encrypted, undefined)).rejects.toThrow( + expect(() => decrypt(encrypted, undefined)).toThrow( /Encrypted data encountered but no encryption key/ ); }); @@ -333,7 +333,9 @@ describe('decrypt', () => { const tampered = new Uint8Array(encrypted); tampered[tampered.length - 1] ^= 0xff; - const error = await decrypt(tampered, key).catch((e) => e); + const error = await Promise.resolve() + .then(() => decrypt(tampered, key)) + .catch((caught) => caught); expect(RuntimeDecryptionError.is(error)).toBe(true); expect(error.context).toMatchObject({ operation: 'decrypt', @@ -423,7 +425,9 @@ describe('sealed envelopes', () => { // A bare CryptoKey is the legacy "symmetric only" shape. It has no // scalar, so it cannot open a sealed payload — and must say so clearly // rather than failing an auth tag somewhere deeper. - const error = await decrypt(sealed, aes).catch((e) => e); + const error = await Promise.resolve(decrypt(sealed, aes)).catch( + (caught) => caught + ); expect(RuntimeDecryptionError.is(error)).toBe(true); expect(error.message).toMatch(/no run keypair is available/); expect(error.context).toMatchObject({ @@ -456,9 +460,9 @@ describe('sealed envelopes', () => { // A seal target carries no symmetric capability, so it must be treated // exactly like "no key" rather than silently coerced. - const error = await decrypt(encrypted, sealTo(keyPair.publicKey)).catch( - (e) => e - ); + const error = await Promise.resolve() + .then(() => decrypt(encrypted, sealTo(keyPair.publicKey))) + .catch((caught) => caught); expect(RuntimeDecryptionError.is(error)).toBe(true); expect(error.message).toMatch(/no encryption key is available/); }); From 71df9469c0dd5a33bd31a3327ac4917e3a2ded21 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:39:11 -0700 Subject: [PATCH 10/16] [core] Consolidate replay preparation types --- packages/core/src/replay-payload-cache.test.ts | 13 ++++++------- packages/core/src/replay-payload-cache.ts | 3 +-- packages/core/src/serialization.ts | 1 - packages/core/src/serialization/replay.ts | 12 +----------- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 04484533b5..2901797ed0 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -6,7 +6,6 @@ import { dehydrateStepReturnValue, deserializePreparedReplayPayload, prepareReplayPayload, - type ReplayPayloadPreparer, } from './serialization.js'; function makeRun(input: unknown): WorkflowRun { @@ -57,7 +56,7 @@ function makeEvents(payloads: unknown[]): Event[] { describe('ReplayPayloadCache', () => { it('deduplicates preparation and accepts a synchronous preparer', async () => { const payload = new Uint8Array([1]); - const preparer = vi.fn((value) => value); + const preparer = vi.fn((value) => value); const cache = new ReplayPayloadCache(undefined, preparer); const first = cache.prepareEventPayload('evnt_one', 'result', payload); @@ -72,7 +71,7 @@ describe('ReplayPayloadCache', () => { const payload = new Uint8Array([1]); const run = makeRun(payload); const preparer = vi - .fn() + .fn() .mockRejectedValueOnce(new Error('decrypt failed')) .mockReturnValueOnce(payload); const cache = new ReplayPayloadCache(undefined, preparer); @@ -90,7 +89,7 @@ describe('ReplayPayloadCache', () => { it('prewarms workflow, step, error, and hook payloads concurrently', async () => { const payloads = [0, 1, 2, 3].map((value) => new Uint8Array([value])); const resolvers: Array<() => void> = []; - const preparer = vi.fn( + const preparer = vi.fn( (value) => new Promise((resolve) => { resolvers.push(() => resolve(value)); @@ -124,7 +123,7 @@ describe('ReplayPayloadCache', () => { false, true ); - const preparer = vi.fn(prepareReplayPayload); + const preparer = vi.fn(prepareReplayPayload); const cache = new ReplayPayloadCache(key, preparer); const prepared = await cache.prepareEventPayload( @@ -156,7 +155,7 @@ describe('ReplayPayloadCache', () => { // shift every later position. Resuming from that length skips exactly the // events the reload was for, which is what `resetScan` exists to prevent. const payloads = [0, 1, 2].map((value) => new Uint8Array([value])); - const preparer = vi.fn((value) => value); + const preparer = vi.fn((value) => value); const cache = new ReplayPayloadCache(undefined, preparer); const run = makeRun(undefined); const [first, missing, second] = makeEvents(payloads); @@ -179,7 +178,7 @@ describe('ReplayPayloadCache', () => { it('bypasses legacy values and ignores missing event data during prewarm', async () => { const legacy = [0, { value: 1 }]; - const preparer = vi.fn((value) => value); + const preparer = vi.fn((value) => value); const cache = new ReplayPayloadCache(undefined, preparer); await cache.prepareEventPayload('evnt_legacy', 'result', legacy); diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 2b90fb3570..6e8da0809f 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -3,7 +3,6 @@ import type { DecryptionKey } from './serialization/encryption.js'; import { type PreparedReplayPayload, prepareReplayPayload, - type ReplayPayloadPreparer, } from './serialization.js'; const MAX_MEMOIZED_PRIMITIVE_LENGTH = 4096; @@ -44,7 +43,7 @@ export class ReplayPayloadCache { constructor( private readonly encryptionKey: DecryptionKey | undefined, - private readonly preparer: ReplayPayloadPreparer = prepareReplayPayload + private readonly preparer: typeof prepareReplayPayload = prepareReplayPayload ) {} /** diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index 25102dfa98..656d1d1e6e 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -3382,7 +3382,6 @@ function getStepRevivers( export { type PreparedReplayPayload, prepareReplayPayload, - type ReplayPayloadPreparer, } from './serialization/replay.js'; async function prepareReplayPayloadWithTelemetry( diff --git a/packages/core/src/serialization/replay.ts b/packages/core/src/serialization/replay.ts index 492e7aa5cd..7d75a3bc07 100644 --- a/packages/core/src/serialization/replay.ts +++ b/packages/core/src/serialization/replay.ts @@ -1,18 +1,8 @@ import { type CompressionStats, decompress } from './compression.js'; import { type DecryptionKey, decrypt } from './encryption.js'; -/** Legacy payloads need no byte preparation but still share the hydrate API. */ -export interface LegacyReplayPayload { - readonly legacy: unknown; -} - /** Host-owned bytes, or a tagged legacy value from before binary envelopes. */ -export type PreparedReplayPayload = Uint8Array | LegacyReplayPayload; - -export type ReplayPayloadPreparer = ( - value: Uint8Array, - key: DecryptionKey | undefined -) => Uint8Array | Promise; +export type PreparedReplayPayload = Uint8Array | { readonly legacy: unknown }; /** * Decrypt and decompress persisted bytes without creating VM-owned values. From e4277ae63e870d94b68b5da1068f2ee902dd61c9 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:48:22 -0700 Subject: [PATCH 11/16] [cli] Remove encryption resolver wrapper type --- packages/cli/src/lib/inspect/hydration.ts | 74 +++++++++++------------ packages/cli/src/lib/inspect/output.ts | 14 +++-- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index 20b35adf37..f1f037e2b4 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -22,15 +22,6 @@ import { parseClassName } from '@workflow/utils/parse-name'; import { getEventDataRefFields } from '@workflow/world'; import chalk from 'chalk'; -/** - * A function that resolves an encryption key for a run, or null to skip - * decryption. Accepts a runId — the resolver is responsible for looking - * up the WorkflowRun internally (with caching) if the World needs it. - */ -export type EncryptionKeyResolver = - | ((runId: string) => Promise) - | null; - async function decryptPayload( value: unknown, key: Parameters[1] @@ -323,28 +314,22 @@ function getRevivers(): Revivers { * Binary envelopes go through the core decryptor; legacy values remain * unchanged for the generic hydrator. * - * When the resolver is null (no --decrypt flag), encrypted fields pass + * Without a resolver (no --decrypt flag), encrypted fields pass * through as Uint8Array and are replaced with EncryptedDataRef in post-processing. */ -async function maybeDecryptFields< - T extends { - runId?: string; - input?: any; - output?: any; - metadata?: any; - eventType?: string; - eventData?: any; - }, ->(resource: T, resolver: EncryptionKeyResolver): Promise { - if (!resolver) return resource; +async function maybeDecryptFields( + resource: T, + resolveKey?: (runId: string) => Promise +): Promise { + if (!resolveKey || !resource || typeof resource !== 'object') return resource; - const runId = (resource as any).runId as string | undefined; - if (!runId) return resource; + const source = resource as Record; + if (typeof source.runId !== 'string') return resource; - const result = { ...resource }; + const result = { ...source }; try { - const rawKey = await resolver(runId); + const rawKey = await resolveKey(source.runId); // Resolve the full key capability so `--decrypt` can open sealed // ('encp') payloads that other runs wrote to this one, not just the // run's own symmetric ('encr') payloads. @@ -356,15 +341,19 @@ async function maybeDecryptFields< // Decrypt input/output/error fields (WorkflowRun, Step) result.input = await decryptPayload(result.input, k); result.output = await decryptPayload(result.output, k); - (result as any).error = await decryptPayload((result as any).error, k); + result.error = await decryptPayload(result.error, k); // Decrypt metadata field (Hook) result.metadata = await decryptPayload(result.metadata, k); // Decrypt eventData fields (Event) if (result.eventData && typeof result.eventData === 'object') { - const eventData = { ...result.eventData }; - for (const field of getEventDataRefFields(result.eventType ?? '')) { + const eventData = { + ...(result.eventData as Record), + }; + const eventType = + typeof result.eventType === 'string' ? result.eventType : ''; + for (const field of getEventDataRefFields(eventType)) { eventData[field] = await decryptPayload(eventData[field], k); } result.eventData = eventData; @@ -382,10 +371,10 @@ async function maybeDecryptFields< // Decryption failed (bad key, corrupted ciphertext, etc.) — fall back // to showing encrypted placeholders instead of crashing the CLI. const { logger } = await import('../config/log.js'); - logger.warn(`Decryption failed for resource ${runId}: ${message}`); + logger.warn(`Decryption failed for resource ${source.runId}: ${message}`); } - return result; + return result as T; } // --------------------------------------------------------------------------- @@ -428,21 +417,30 @@ function replaceEncryptedAndExpiredWithRef(resource: T): T { /** * Hydrate the serialized data fields of a resource for CLI display. * - * When `encryptorResolver` is null (default / no --decrypt flag), encrypted + * Without `resolveKey` (the default / no --decrypt flag), encrypted * fields are shown as styled "🔒 Encrypted" placeholders via EncryptedDataRef. * - * When `encryptorResolver` is provided (--decrypt flag), encrypted fields + * When `resolveKey` is provided (--decrypt flag), encrypted fields * are decrypted before hydration so the actual user data is displayed. */ -export async function hydrateResourceIO( +export async function hydrateResourceIO< + T extends { + stepId?: string; + hookId?: string; + eventId?: string; + eventType?: string; + input?: unknown; + output?: unknown; + metadata?: unknown; + eventData?: unknown; + executionContext?: unknown; + }, +>( resource: T, - keyResolver?: EncryptionKeyResolver + resolveKey?: (runId: string) => Promise ): Promise { // Pre-process: decrypt any encrypted fields when a resolver is provided - const preprocessed = await maybeDecryptFields( - resource as any, - keyResolver ?? null - ); + const preprocessed = await maybeDecryptFields(resource, resolveKey); const hydrated = hydrateResourceIOGeneric(preprocessed, getRevivers()) as T; // Post-process: swap encrypted Uint8Arrays and expired stubs for CLI-styled objects return replaceEncryptedAndExpiredWithRef(hydrated); diff --git a/packages/cli/src/lib/inspect/output.ts b/packages/cli/src/lib/inspect/output.ts index 3785b9ca16..a4b9f0de90 100644 --- a/packages/cli/src/lib/inspect/output.ts +++ b/packages/cli/src/lib/inspect/output.ts @@ -25,7 +25,6 @@ import { isObservabilityUpgradeRequiredError, } from './errors.js'; import { - type EncryptionKeyResolver, hydrateResourceIO, isEncryptedRef, isExpiredRef, @@ -33,15 +32,18 @@ import { import { resolveTimeWindow } from './time-window.js'; /** - * Create an EncryptionKeyResolver from a World instance. - * Returns null if decrypt is false — encrypted data will show as a placeholder. + * Create a run-key lookup from a World instance. + * Returns undefined when encrypted data should remain a placeholder. * * The resolver fetches the full WorkflowRun (cached per runId) so that the * World can inspect deployment-specific fields for key resolution. */ -function createResolver(world: World, decrypt: boolean): EncryptionKeyResolver { - if (!decrypt) return null; - if (!world.getEncryptionKeyForRun) return null; +function createResolver( + world: World, + decrypt: boolean +): ((runId: string) => Promise) | undefined { + if (!decrypt) return; + if (!world.getEncryptionKeyForRun) return; const cache = new Map>(); return (runId: string) => { let cached = cache.get(runId); From 0add1ad76416e67cc4ba52473bc7d732432bf3b0 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:52:53 -0700 Subject: [PATCH 12/16] [cli] Use the decryption capability type directly --- packages/cli/src/lib/inspect/hydration.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index f1f037e2b4..93ea37b6d8 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -6,7 +6,11 @@ */ import { inspect } from 'node:util'; -import { decrypt, getCommonRevivers } from '@workflow/core/serialization'; +import { + decrypt, + type DecryptionKey, + getCommonRevivers, +} from '@workflow/core/serialization'; import { ClassInstanceRef, extractClassName, @@ -24,7 +28,7 @@ import chalk from 'chalk'; async function decryptPayload( value: unknown, - key: Parameters[1] + key: DecryptionKey | undefined ): Promise { return value instanceof Uint8Array ? decrypt(value, key) : value; } From ba88b6bb1d0693841ef0ba333db7fb55d04109f1 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:55:45 -0700 Subject: [PATCH 13/16] [cli] Sort hydration imports --- packages/cli/src/lib/inspect/hydration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index 93ea37b6d8..d206bb0571 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -7,8 +7,8 @@ import { inspect } from 'node:util'; import { - decrypt, type DecryptionKey, + decrypt, getCommonRevivers, } from '@workflow/core/serialization'; import { From 1cbde1f87b390bd72240f12857f0bff16e9ef821 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:07:58 -0700 Subject: [PATCH 14/16] refactor(serialization): centralize format handling --- .../decode-replay-payloads-synchronously.md | 2 + packages/cli/src/lib/inspect/hydration.ts | 4 +- packages/core/src/encryption.ts | 4 +- .../core/src/replay-payload-cache.test.ts | 2 +- packages/core/src/replay-payload-cache.ts | 2 +- packages/core/src/serialization-format.ts | 26 ++--- packages/core/src/serialization.ts | 5 - .../core/src/serialization/compression.ts | 35 +++---- packages/core/src/serialization/format.ts | 9 +- .../src/serialization/serialization.test.ts | 6 +- packages/core/src/serialization/types.ts | 40 ++------ packages/web-shared/src/lib/hydration.ts | 33 +++---- .../src/lib/zstd-browser-decoder.ts | 27 ++---- packages/world-vercel/src/events-v4.ts | 4 +- .../world-vercel/src/serialized-data.test.ts | 12 +-- packages/world-vercel/src/serialized-data.ts | 94 +++++++------------ packages/world/src/index.ts | 6 ++ packages/world/src/serialization-format.ts | 45 +++++++++ 18 files changed, 175 insertions(+), 181 deletions(-) create mode 100644 packages/world/src/serialization-format.ts diff --git a/.changeset/decode-replay-payloads-synchronously.md b/.changeset/decode-replay-payloads-synchronously.md index de58d45d3b..d4b145d942 100644 --- a/.changeset/decode-replay-payloads-synchronously.md +++ b/.changeset/decode-replay-payloads-synchronously.md @@ -1,6 +1,8 @@ --- "@workflow/core": patch "@workflow/cli": patch +"@workflow/web-shared": patch +"@workflow/world": patch "@workflow/world-vercel": patch --- diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index d206bb0571..b74fc9a622 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -9,6 +9,7 @@ import { inspect } from 'node:util'; import { type DecryptionKey, decrypt, + deriveRunPayloadKeys, getCommonRevivers, } from '@workflow/core/serialization'; import { @@ -337,9 +338,6 @@ async function maybeDecryptFields( // Resolve the full key capability so `--decrypt` can open sealed // ('encp') payloads that other runs wrote to this one, not just the // run's own symmetric ('encr') payloads. - const { deriveRunPayloadKeys } = await import( - '@workflow/core/serialization' - ); const k = rawKey ? await deriveRunPayloadKeys(rawKey) : undefined; // Decrypt input/output/error fields (WorkflowRun, Step) diff --git a/packages/core/src/encryption.ts b/packages/core/src/encryption.ts index 54aaa34149..4f0fb608a5 100644 --- a/packages/core/src/encryption.ts +++ b/packages/core/src/encryption.ts @@ -93,7 +93,7 @@ export async function importKey( return key; } -function assertValidAesGcmEnvelope(data: Uint8Array): void { +function assertAesGcmEnvelopeLength(data: Uint8Array): void { const minLength = NONCE_LENGTH + TAG_BYTES; if (data.byteLength < minLength) { throw new RuntimeDecryptionError( @@ -250,7 +250,7 @@ export function decrypt( data: Uint8Array, aad?: Uint8Array ): Uint8Array | Promise { - assertValidAesGcmEnvelope(data); + assertAesGcmEnvelopeLength(data); if (!key.usages.includes('decrypt')) { throw new RuntimeDecryptionError( 'AES-256-GCM decryption failed: CryptoKey does not support decrypt', diff --git a/packages/core/src/replay-payload-cache.test.ts b/packages/core/src/replay-payload-cache.test.ts index 2901797ed0..f09ec0f4ca 100644 --- a/packages/core/src/replay-payload-cache.test.ts +++ b/packages/core/src/replay-payload-cache.test.ts @@ -2,10 +2,10 @@ import type { Event, WorkflowRun } from '@workflow/world'; import { describe, expect, it, vi } from 'vitest'; import { importKey } from './encryption.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { prepareReplayPayload } from './serialization/replay.js'; import { dehydrateStepReturnValue, deserializePreparedReplayPayload, - prepareReplayPayload, } from './serialization.js'; function makeRun(input: unknown): WorkflowRun { diff --git a/packages/core/src/replay-payload-cache.ts b/packages/core/src/replay-payload-cache.ts index 6e8da0809f..084c6fd3c9 100644 --- a/packages/core/src/replay-payload-cache.ts +++ b/packages/core/src/replay-payload-cache.ts @@ -3,7 +3,7 @@ import type { DecryptionKey } from './serialization/encryption.js'; import { type PreparedReplayPayload, prepareReplayPayload, -} from './serialization.js'; +} from './serialization/replay.js'; const MAX_MEMOIZED_PRIMITIVE_LENGTH = 4096; type ReplayPayloadField = 'result' | 'error' | 'payload'; diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index a485af295c..12f8b9ccc6 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -11,7 +11,7 @@ import { parse, unflatten } from 'devalue'; import { decompress, decompressSync, - registerZstdDecoder, + type ZstdDecoder, } from './serialization/compression.js'; import { type DecryptionKey, @@ -21,6 +21,7 @@ import { import { decodeFormatPrefix as decodePrefix, encodeWithFormatPrefix, + isEncrypted, peekFormatPrefix, } from './serialization/format.js'; import { SerializationFormat } from './serialization/types.js'; @@ -55,11 +56,18 @@ export { type DecryptionKey, decrypt as decryptEnvelope, isRunPayloadKeys }; // --------------------------------------------------------------------------- export { encodeWithFormatPrefix, SerializationFormat }; -export { registerZstdDecoder }; export type SerializationFormatType = (typeof SerializationFormat)[keyof typeof SerializationFormat]; +export interface HydrateDataOptions { + /** + * Runtime-specific zstd decoder, such as the browser observability WASM + * adapter. + */ + zstdDecoder?: ZstdDecoder; +} + /** * Decode a format-prefixed payload. */ @@ -141,12 +149,7 @@ export function isExpiredStub(data: unknown): boolean { * Browser-safe — does not depend on the full serialization module. */ export function isEncryptedData(data: unknown): boolean { - if (!(data instanceof Uint8Array)) return false; - const prefix = peekFormatPrefix(data); - return ( - prefix === SerializationFormat.ENCRYPTED || - prefix === SerializationFormat.SEALED - ); + return isEncrypted(data); } /** @@ -262,7 +265,8 @@ export function hydrateData(value: unknown, revivers: Revivers): unknown { export async function hydrateDataWithKey( value: unknown, revivers: Revivers, - key: DecryptionKey | undefined + key: DecryptionKey | undefined, + options?: HydrateDataOptions ): Promise { let data = value; if (data instanceof Uint8Array && isEncryptedData(data) && key) { @@ -280,9 +284,9 @@ export async function hydrateDataWithKey( if (data instanceof Uint8Array && isCompressedData(data)) { // Decompress: strip the codec prefix and inflate. gzip uses the // web-standard DecompressionStream (works in browsers); zstd uses - // node:zlib on Node or the registered WASM decoder in the browser. + // node:zlib on Node or an explicitly supplied WASM decoder in the browser. // The inflated bytes carry their own format prefix (e.g. 'devl'). - data = await decompress(data); + data = await decompress(data, undefined, options); } // Delegate the (decrypted/decompressed) result to sync hydrateData return hydrateData(data, revivers); diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index 656d1d1e6e..c7df7e3275 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -3379,11 +3379,6 @@ function getStepRevivers( }; } -export { - type PreparedReplayPayload, - prepareReplayPayload, -} from './serialization/replay.js'; - async function prepareReplayPayloadWithTelemetry( value: unknown, key: DecryptionKey | undefined diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index e44406e73c..ac0dd0c87f 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -14,9 +14,8 @@ * * Read side: dispatch on the format prefix, so both 'zstd' and 'gzip' * payloads are always decodable regardless of which codec wrote them. - * (The browser o11y read path decodes zstd via a registered WASM decoder — - * see `serialization-format.ts`; this module's `decompress` is the Node - * runtime/replay path and uses `node:zlib`.) + * (The browser o11y read path passes a WASM decoder explicitly — see + * `serialization-format.ts`; the Node runtime/replay path uses `node:zlib`.) * * Layering order with encryption: compression is applied BEFORE * encryption (encr(zstd(devl))) — encrypted bytes are high-entropy and @@ -108,15 +107,13 @@ function asUint8Array(data: Uint8Array): Uint8Array { return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } -let zstdBrowserDecoder: - | ((payload: Uint8Array) => Promise) - | undefined; +/** Runtime-specific zstd decoder used when native Node support is unavailable. */ +export type ZstdDecoder = ( + payload: Uint8Array +) => Uint8Array | Promise; -/** Register the zstd decoder used by browser observability clients. */ -export function registerZstdDecoder( - decoder: (payload: Uint8Array) => Promise -): void { - zstdBrowserDecoder = decoder; +export interface DecompressionOptions { + zstdDecoder?: ZstdDecoder; } function isZstdAvailable(): boolean { @@ -195,15 +192,18 @@ function zstdBytes(data: Uint8Array): Uint8Array { return asUint8Array(compress(data, opts)); } -function decompressZstd(payload: Uint8Array): Uint8Array | Promise { +function decompressZstd( + payload: Uint8Array, + decoder?: ZstdDecoder +): Uint8Array | Promise { if (nodeZlib?.zstdDecompressSync) { return asUint8Array(nodeZlib.zstdDecompressSync(payload)); } - if (zstdBrowserDecoder) return zstdBrowserDecoder(payload); + if (decoder) return decoder(payload); throw new Error( 'Compressed (zstd) workflow data encountered but no zstd decoder is ' + 'available. Node.js 22.15+ decodes natively; in the browser ' + - 'register one via registerZstdDecoder.' + 'pass one in DecompressionOptions.' ); } @@ -334,7 +334,8 @@ export function compress( */ export function decompress( data: Uint8Array, - stats?: CompressionStats + stats?: CompressionStats, + options?: DecompressionOptions ): Uint8Array | Promise { const prefix = peekFormatPrefix(data); @@ -351,7 +352,9 @@ export function decompress( const { payload } = decodeFormatPrefix(data); const inflated = - codec === 'zstd' ? decompressZstd(payload) : decompressGzip(payload); + codec === 'zstd' + ? decompressZstd(payload, options?.zstdDecoder) + : decompressGzip(payload); const finish = (value: Uint8Array): Uint8Array => { recordStats(stats, codec, value.length, data.length); return value; diff --git a/packages/core/src/serialization/format.ts b/packages/core/src/serialization/format.ts index 4cb286b111..19ff1472bf 100644 --- a/packages/core/src/serialization/format.ts +++ b/packages/core/src/serialization/format.ts @@ -65,12 +65,15 @@ export function peekFormatPrefix(data: Uint8Array): FormatPrefix | null { } /** - * Check if data is encrypted (has 'encr' format prefix). + * Check if data is encrypted, whether symmetrically (`encr`) or sealed to a + * run's public key (`encp`). Use the exact prefix when the scheme matters. */ export function isEncrypted(data: unknown): boolean { + if (!(data instanceof Uint8Array)) return false; + const prefix = peekFormatPrefix(data); return ( - data instanceof Uint8Array && - peekFormatPrefix(data) === SerializationFormat.ENCRYPTED + prefix === SerializationFormat.ENCRYPTED || + prefix === SerializationFormat.SEALED ); } diff --git a/packages/core/src/serialization/serialization.test.ts b/packages/core/src/serialization/serialization.test.ts index 8aba21f45a..11073be72d 100644 --- a/packages/core/src/serialization/serialization.test.ts +++ b/packages/core/src/serialization/serialization.test.ts @@ -374,9 +374,9 @@ describe('sealed envelopes', () => { )) as Uint8Array; expect(peekFormatPrefix(sealed)).toBe(SerializationFormat.SEALED); - // Sealed data must not be mistaken for symmetrically encrypted data by - // anything dispatching on the prefix. - expect(isEncrypted(sealed)).toBe(false); + // Both symmetric and sealed envelopes are encrypted. Callers that need to + // distinguish the schemes dispatch on the exact prefix. + expect(isEncrypted(sealed)).toBe(true); }); it('round-trips: cross-run writer seals, owning run opens', async () => { diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 51a1b5467b..8d3f042030 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -3,6 +3,12 @@ */ import type { RuntimeDecryptionErrorContext } from '@workflow/errors'; +import { + SerializationFormat, + type SerializationFormatType, +} from '@workflow/world/serialization-format.js'; + +export { SerializationFormat }; // ---- Format Prefix ---- @@ -13,7 +19,9 @@ import type { RuntimeDecryptionErrorContext } from '@workflow/errors'; * at runtime. The `SerializationFormat` object provides well-known * constants, but codecs may define additional prefixes. */ -export type FormatPrefix = string & { readonly __brand: 'FormatPrefix' }; +export type FormatPrefix = + | SerializationFormatType + | (string & { readonly __brand: 'FormatPrefix' }); /** * Runtime type guard for format prefix strings. @@ -24,36 +32,6 @@ export function isFormatPrefix(value: string): value is FormatPrefix { return value.length === 4 && /^[a-z0-9]{4}$/.test(value); } -function formatPrefix( - value: Value -): Value & FormatPrefix { - return value as Value & FormatPrefix; -} - -/** - * Well-known format prefix constants. Codecs may define additional ones. - */ -export const SerializationFormat = { - /** devalue stringify/parse with TextEncoder/TextDecoder */ - DEVALUE_V1: formatPrefix('devl'), - /** Encrypted payload (inner payload has its own format prefix) */ - ENCRYPTED: formatPrefix('encr'), - /** - * Sealed payload — asymmetrically encrypted to a run's X25519 public key - * (inner payload has its own format prefix). - * - * Used for *cross-run* writes (hook payloads, forwarded stream frames), - * where the writer holds only the recipient run's public key and therefore - * cannot decrypt. A run's own payloads continue to use {@link ENCRYPTED}. - * See `sealed-box.ts` for the construction. - */ - SEALED: formatPrefix('encp'), - /** Gzip-compressed payload (inner payload has its own format prefix) */ - GZIP: formatPrefix('gzip'), - /** Zstandard-compressed payload (inner payload has its own format prefix) */ - ZSTD: formatPrefix('zstd'), -} as const; - // ---- Serializable Types ---- /** diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index 25297fb15f..26f6e10612 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -479,29 +479,19 @@ export async function hydrateResourceIOWithKey( * Async hydration for web display. * * This follows the same resource-field mapping as {@link hydrateResourceIO}, - * but can also inflate compressed browser payloads through the registered - * zstd WASM decoder. When a key is provided, encrypted fields are decrypted - * first and then inflated/hydrated. + * but can also inflate compressed browser payloads through the zstd WASM + * decoder. When a key is provided, encrypted fields are decrypted first and + * then inflated/hydrated. */ export async function hydrateResourceIOAsync( resource: T, key?: Uint8Array ): Promise { - const { hydrateDataWithKey, deriveRunPayloadKeys } = await import( - '@workflow/core/serialization-format' - ); - // Payloads may be zstd-compressed (the Web DecompressionStream has no zstd); - // register the WASM-backed browser decoder before hydrating. Idempotent and - // lazy — the WASM is only compiled when a zstd payload is actually decoded. - const { ensureZstdDecoderRegistered } = await import( - './zstd-browser-decoder.js' - ); - ensureZstdDecoderRegistered(); - // Resolve the *full* key capability, not just the symmetric key: a run's - // event log can contain sealed ('encp') payloads that another run wrote to - // it (a cross-deployment hook resumption, say), and opening those needs the - // run's X25519 scalar in addition to its AES key. Both are derived from the - // same 32 bytes the key-retrieval endpoint returns. + const [serialization, { decompressZstdInBrowser }] = await Promise.all([ + import('@workflow/core/serialization-format'), + import('./zstd-browser-decoder.js'), + ]); + const { hydrateDataWithKey, deriveRunPayloadKeys } = serialization; // Resolve the *full* key capability, not just the symmetric key: a run's // event log can contain sealed ('encp') payloads that another run wrote to // it (a cross-deployment hook resumption, say), and opening those needs the @@ -509,16 +499,19 @@ export async function hydrateResourceIOAsync( // 32 bytes the key-retrieval endpoint returns. const cryptoKey = key ? await deriveRunPayloadKeys(key) : undefined; const revivers = getRevivers(); + const hydrationOptions = { zstdDecoder: decompressZstdInBrowser }; async function hydrateField(value: unknown): Promise { // Already-hydrated: encrypted marker with stored bytes if (isEncryptedMarker(value)) { const raw = (value as any).__encryptedData as Uint8Array; - return cryptoKey ? hydrateDataWithKey(raw, revivers, cryptoKey) : value; + return cryptoKey + ? hydrateDataWithKey(raw, revivers, cryptoKey, hydrationOptions) + : value; } // Raw Uint8Array: may be encrypted, compressed, or plain devalue. if (value instanceof Uint8Array) { - return hydrateDataWithKey(value, revivers, cryptoKey); + return hydrateDataWithKey(value, revivers, cryptoKey, hydrationOptions); } // Not serialized — return as-is. return value; diff --git a/packages/web-shared/src/lib/zstd-browser-decoder.ts b/packages/web-shared/src/lib/zstd-browser-decoder.ts index 05cf339dfd..57fd551333 100644 --- a/packages/web-shared/src/lib/zstd-browser-decoder.ts +++ b/packages/web-shared/src/lib/zstd-browser-decoder.ts @@ -2,18 +2,14 @@ * Browser zstd decoder for the o11y read path. * * The Web `DecompressionStream` has no zstd support, so `@workflow/core`'s - * `hydrateDataWithKey` delegates zstd inflation to a decoder registered via - * `registerZstdDecoder`. This module supplies that decoder, backed by the - * `@tootallnate/zstd-wasm` single-file WASM decoder. + * `hydrateDataWithKey` delegates zstd inflation to the decoder exported by + * this module, backed by the `@tootallnate/zstd-wasm` single-file WASM decoder. * * The package leaves WASM sourcing to the caller; we resolve the shipped * `zstd.wasm` as a bundler asset (`new URL(..., import.meta.url)`, the same * pattern the trace-viewer Worker uses) and compile it once, lazily — the * WASM is fetched only the first time a zstd payload is actually decoded. */ -import { registerZstdDecoder } from '@workflow/core/serialization-format'; - -let registered = false; let modulePromise: Promise | undefined; function loadWasmModule(): Promise { @@ -27,16 +23,13 @@ function loadWasmModule(): Promise { } /** - * Register the browser zstd decoder with `@workflow/core` (idempotent). - * Call this before hydrating payloads that may be zstd-compressed; the - * actual WASM compile + decode happens lazily on first use. + * Decode zstd in browsers. The WASM compile + decode remains lazy and the + * compiled module is reused for subsequent payloads. */ -export function ensureZstdDecoderRegistered(): void { - if (registered) return; - registered = true; - registerZstdDecoder(async (payload) => { - const { decompressBytes } = await import('@tootallnate/zstd-wasm'); - const wasmModule = await loadWasmModule(); - return decompressBytes(wasmModule, payload); - }); +export async function decompressZstdInBrowser( + payload: Uint8Array +): Promise { + const { decompressBytes } = await import('@tootallnate/zstd-wasm'); + const wasmModule = await loadWasmModule(); + return decompressBytes(wasmModule, payload); } diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 2b64833217..4187bb7be3 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -34,6 +34,7 @@ import { type ListEventsByCorrelationIdParams, type ListEventsParams, type PaginatedResponse, + peekSerializationFormat, StructuredErrorSchema, WaitSchema, WorkflowRunSchema, @@ -59,7 +60,6 @@ import { recordClientSpanStatus, withHttpClientSpan, } from './http-core.js'; -import { hasSerializedDataFormatPrefix } from './serialized-data.js'; import { deserializeStep, StepWireSchema } from './steps.js'; import { ErrorType, @@ -381,7 +381,7 @@ const legacyStructuredErrorEventTypes = new Set([ ]); function decodeLegacyStructuredError(payload: Uint8Array): unknown { - if (hasSerializedDataFormatPrefix(payload)) return payload; + if (peekSerializationFormat(payload) !== null) return payload; try { const parsed = StructuredErrorSchema.safeParse(decode(payload.slice())); diff --git a/packages/world-vercel/src/serialized-data.test.ts b/packages/world-vercel/src/serialized-data.test.ts index 47a05af53e..a19a357acb 100644 --- a/packages/world-vercel/src/serialized-data.test.ts +++ b/packages/world-vercel/src/serialized-data.test.ts @@ -1,8 +1,6 @@ +import { peekSerializationFormat } from '@workflow/world/serialization-format.js'; import { describe, expect, it } from 'vitest'; -import { - hasSerializedDataFormatPrefix, - normalizeSerializedData, -} from './serialized-data.js'; +import { normalizeSerializedData } from './serialized-data.js'; const encoder = new TextEncoder(); @@ -21,9 +19,9 @@ describe('serialized data normalization', () => { 'gzip', 'zstd', ])('recognizes the %s envelope', (format) => { - expect( - hasSerializedDataFormatPrefix(envelope(format, new Uint8Array())) - ).toBe(true); + expect(peekSerializationFormat(envelope(format, new Uint8Array()))).toBe( + format + ); }); it('leaves non-compressed envelopes untouched', () => { diff --git a/packages/world-vercel/src/serialized-data.ts b/packages/world-vercel/src/serialized-data.ts index e0a249d50c..c723578a21 100644 --- a/packages/world-vercel/src/serialized-data.ts +++ b/packages/world-vercel/src/serialized-data.ts @@ -1,46 +1,18 @@ +import * as nodeZlib from 'node:zlib'; import { WorkflowWorldError } from '@workflow/errors'; +import { + peekSerializationFormat, + SerializationFormat, +} from '@workflow/world/serialization-format.js'; -const FORMAT_PREFIX_LENGTH = 4; -const GZIP_FORMAT_PREFIX = 'gzip'; -const ZSTD_FORMAT_PREFIX = 'zstd'; -const formatDecoder = new TextDecoder(); - -const SERIALIZED_DATA_FORMAT_PREFIXES = new Set([ - 'devl', - 'encr', - 'encp', - GZIP_FORMAT_PREFIX, - ZSTD_FORMAT_PREFIX, -]); - -const nodeZlib = (() => { - try { - return process.getBuiltinModule('node:zlib'); - } catch { - return undefined; - } -})(); - -function peekFormatPrefix(value: unknown): string | null { - if ( - !(value instanceof Uint8Array) || - value.byteLength < FORMAT_PREFIX_LENGTH - ) { - return null; - } - return formatDecoder.decode(value.subarray(0, FORMAT_PREFIX_LENGTH)); -} - -export function hasSerializedDataFormatPrefix(value: unknown): boolean { - const format = peekFormatPrefix(value); - return format !== null && SERIALIZED_DATA_FORMAT_PREFIXES.has(format); -} - -function decompress(format: string, payload: Uint8Array): Uint8Array { +function decompress( + format: typeof SerializationFormat.GZIP | typeof SerializationFormat.ZSTD, + payload: Uint8Array +): Uint8Array { const decompress = - format === ZSTD_FORMAT_PREFIX - ? nodeZlib?.zstdDecompressSync - : nodeZlib?.gunzipSync; + format === SerializationFormat.ZSTD + ? nodeZlib.zstdDecompressSync + : nodeZlib.gunzipSync; if (!decompress) { throw new WorkflowWorldError( @@ -53,44 +25,48 @@ function decompress(format: string, payload: Uint8Array): Uint8Array { } export function normalizeSerializedData(value: unknown): unknown { - const format = peekFormatPrefix(value); - if (format !== ZSTD_FORMAT_PREFIX && format !== GZIP_FORMAT_PREFIX) { + const format = peekSerializationFormat(value); + if ( + format !== SerializationFormat.ZSTD && + format !== SerializationFormat.GZIP + ) { return value; } const bytes = value as Uint8Array; - return decompress(format, bytes.subarray(FORMAT_PREFIX_LENGTH)); + return decompress(format, bytes.subarray(format.length)); } -export function normalizeWorkflowRunData>( - run: T +const PAYLOAD_FIELDS = ['input', 'output', 'error'] as const; + +function normalizeFields>( + value: T, + fields: readonly string[] ): T { return { - ...run, - input: normalizeSerializedData(run.input), - output: normalizeSerializedData(run.output), - error: normalizeSerializedData(run.error), + ...value, + ...Object.fromEntries( + fields.map((field) => [field, normalizeSerializedData(value[field])]) + ), }; } +export function normalizeWorkflowRunData>( + run: T +): T { + return normalizeFields(run, PAYLOAD_FIELDS); +} + export function normalizeStepData>( step: T ): T { // Only the resolved payload fields can carry a compression wrapper. // `*Ref` fields are RefDescriptor objects (lazy mode), never byte // payloads, so they need no normalization. - return { - ...step, - input: normalizeSerializedData(step.input), - output: normalizeSerializedData(step.output), - error: normalizeSerializedData(step.error), - }; + return normalizeFields(step, PAYLOAD_FIELDS); } export function normalizeHookData>( hook: T ): T { - return { - ...hook, - metadata: normalizeSerializedData(hook.metadata), - }; + return normalizeFields(hook, ['metadata']); } diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index a8325cc44c..290c73c535 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -102,6 +102,12 @@ export { LegacySerializedDataSchemaV1, SerializedDataSchema, } from './serialization.js'; +export { + isSerializationFormat, + peekSerializationFormat, + SerializationFormat, + type SerializationFormatType, +} from './serialization-format.js'; export type * from './shared.js'; export type { GetChunksOptions, diff --git a/packages/world/src/serialization-format.ts b/packages/world/src/serialization-format.ts new file mode 100644 index 0000000000..cbcf958c90 --- /dev/null +++ b/packages/world/src/serialization-format.ts @@ -0,0 +1,45 @@ +/** Known four-byte envelope prefixes in the persisted payload protocol. */ +export const SerializationFormat = { + /** devalue stringify/parse with TextEncoder/TextDecoder */ + DEVALUE_V1: 'devl', + /** Symmetrically encrypted payload */ + ENCRYPTED: 'encr', + /** Payload sealed to a run's public key */ + SEALED: 'encp', + /** Gzip-compressed payload */ + GZIP: 'gzip', + /** Zstandard-compressed payload */ + ZSTD: 'zstd', +} as const; + +export type SerializationFormatType = + (typeof SerializationFormat)[keyof typeof SerializationFormat]; + +const serializationFormats = new Set( + Object.values(SerializationFormat) +); + +/** Whether a value is one of the persisted payload protocol's known formats. */ +export function isSerializationFormat( + value: unknown +): value is SerializationFormatType { + return typeof value === 'string' && serializationFormats.has(value); +} + +const FORMAT_PREFIX_LENGTH = 4; +const formatDecoder = new TextDecoder(); + +/** Read a known persisted payload format without consuming its bytes. */ +export function peekSerializationFormat( + value: unknown +): SerializationFormatType | null { + if ( + !(value instanceof Uint8Array) || + value.byteLength < FORMAT_PREFIX_LENGTH + ) { + return null; + } + + const format = formatDecoder.decode(value.subarray(0, FORMAT_PREFIX_LENGTH)); + return isSerializationFormat(format) ? format : null; +} From 324318fb68702d07d5911b5e19632186c5e9f418 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:54:47 -0700 Subject: [PATCH 15/16] refactor(world-vercel): keep payload normalization explicit --- packages/world-vercel/src/serialized-data.ts | 33 ++++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/world-vercel/src/serialized-data.ts b/packages/world-vercel/src/serialized-data.ts index c723578a21..65e9c45f3d 100644 --- a/packages/world-vercel/src/serialized-data.ts +++ b/packages/world-vercel/src/serialized-data.ts @@ -36,24 +36,15 @@ export function normalizeSerializedData(value: unknown): unknown { return decompress(format, bytes.subarray(format.length)); } -const PAYLOAD_FIELDS = ['input', 'output', 'error'] as const; - -function normalizeFields>( - value: T, - fields: readonly string[] -): T { - return { - ...value, - ...Object.fromEntries( - fields.map((field) => [field, normalizeSerializedData(value[field])]) - ), - }; -} - export function normalizeWorkflowRunData>( run: T ): T { - return normalizeFields(run, PAYLOAD_FIELDS); + return { + ...run, + input: normalizeSerializedData(run.input), + output: normalizeSerializedData(run.output), + error: normalizeSerializedData(run.error), + }; } export function normalizeStepData>( @@ -62,11 +53,19 @@ export function normalizeStepData>( // Only the resolved payload fields can carry a compression wrapper. // `*Ref` fields are RefDescriptor objects (lazy mode), never byte // payloads, so they need no normalization. - return normalizeFields(step, PAYLOAD_FIELDS); + return { + ...step, + input: normalizeSerializedData(step.input), + output: normalizeSerializedData(step.output), + error: normalizeSerializedData(step.error), + }; } export function normalizeHookData>( hook: T ): T { - return normalizeFields(hook, ['metadata']); + return { + ...hook, + metadata: normalizeSerializedData(hook.metadata), + }; } From 623458210cd2a590b5ebde020e98f07a30c72984 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:31:56 -0700 Subject: [PATCH 16/16] refactor(serialization): share replay format codecs --- packages/core/src/serialization-format.ts | 8 ++- packages/core/src/serialization.ts | 7 +-- .../core/src/serialization/compression.ts | 16 +----- packages/core/src/serialization/index.ts | 1 + .../src/serialization/serialization.test.ts | 1 + packages/core/src/serialization/types.ts | 31 +++------- packages/web-shared/src/lib/hydration.ts | 26 ++++++--- .../world-vercel/src/serialized-data.test.ts | 15 +++++ packages/world-vercel/src/serialized-data.ts | 28 +++------- packages/world/src/index.ts | 3 + .../world/src/serialization-compression.ts | 56 +++++++++++++++++++ packages/world/src/serialization-format.ts | 43 +++++++++++--- 12 files changed, 153 insertions(+), 82 deletions(-) create mode 100644 packages/world/src/serialization-compression.ts diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index 12f8b9ccc6..41668c6425 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -24,7 +24,10 @@ import { isEncrypted, peekFormatPrefix, } from './serialization/format.js'; -import { SerializationFormat } from './serialization/types.js'; +import { + SerializationFormat, + type SerializationFormatType, +} from './serialization/types.js'; // --------------------------------------------------------------------------- // Key material (browser-safe re-exports) @@ -57,8 +60,7 @@ export { type DecryptionKey, decrypt as decryptEnvelope, isRunPayloadKeys }; export { encodeWithFormatPrefix, SerializationFormat }; -export type SerializationFormatType = - (typeof SerializationFormat)[keyof typeof SerializationFormat]; +export type { SerializationFormatType }; export interface HydrateDataOptions { /** diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index c7df7e3275..9b1a4b53a1 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -89,6 +89,7 @@ import { type FormatPrefix, isFormatPrefix, SerializationFormat, + type SerializationFormatType, } from './serialization/types.js'; import * as workflowModule from './serialization/workflow.js'; import { contextStorage } from './step/context-storage.js'; @@ -117,6 +118,7 @@ import { WorkflowAbortSignal } from './workflow/abort-controller.js'; // so existing consumers of `@workflow/core/serialization` keep working. export { SerializationFormat, + type SerializationFormatType, type FormatPrefix, isFormatPrefix, encodeWithFormatPrefix, @@ -141,11 +143,6 @@ export { export { compress, decompress } from './serialization/compression.js'; -// Re-export the legacy SerializationFormatType for backwards compatibility. -// New code should use FormatPrefix from './serialization/types.js'. -export type SerializationFormatType = - (typeof SerializationFormat)[keyof typeof SerializationFormat]; - /** * Default ULID generator for contexts where VM's seeded `stableUlid` isn't available. * Used as a fallback when serializing streams outside the workflow VM context diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index ac0dd0c87f..f6e2a5aa23 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -30,6 +30,7 @@ * archives, etc.) from wasted CPU and size inflation. */ +import { decompressSerializedDataSync } from '@workflow/world/serialization-compression.js'; import { decodeFormatPrefix, encodeWithFormatPrefix, @@ -368,20 +369,7 @@ export function decompress( * its async hydration path is requested. */ export function decompressSync(data: Uint8Array): Uint8Array | undefined { - const prefix = peekFormatPrefix(data); - if (prefix === SerializationFormat.ZSTD) { - const decompress = nodeZlib?.zstdDecompressSync; - return decompress - ? asUint8Array(decompress(decodeFormatPrefix(data).payload)) - : undefined; - } - if (prefix === SerializationFormat.GZIP) { - const decompress = nodeZlib?.gunzipSync; - return decompress - ? asUint8Array(decompress(decodeFormatPrefix(data).payload)) - : undefined; - } - return data; + return decompressSerializedDataSync(data); } /** diff --git a/packages/core/src/serialization/index.ts b/packages/core/src/serialization/index.ts index d2e0ec0869..a241ca034d 100644 --- a/packages/core/src/serialization/index.ts +++ b/packages/core/src/serialization/index.ts @@ -41,6 +41,7 @@ export type { Reducers, Revivers, SerializableSpecial, + SerializationFormatType, } from './types.js'; export { isFormatPrefix, SerializationFormat } from './types.js'; diff --git a/packages/core/src/serialization/serialization.test.ts b/packages/core/src/serialization/serialization.test.ts index 11073be72d..46f1bc8e46 100644 --- a/packages/core/src/serialization/serialization.test.ts +++ b/packages/core/src/serialization/serialization.test.ts @@ -66,6 +66,7 @@ describe('isFormatPrefix', () => { it('should reject strings that are too long', () => { expect(isFormatPrefix('abcde')).toBe(false); expect(isFormatPrefix('abcdef')).toBe(false); + expect(isFormatPrefix('abcd\n')).toBe(false); }); it('should reject uppercase characters', () => { diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 8d3f042030..5da3211b18 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -4,33 +4,18 @@ import type { RuntimeDecryptionErrorContext } from '@workflow/errors'; import { + type FormatPrefix, + isFormatPrefix, SerializationFormat, type SerializationFormatType, } from '@workflow/world/serialization-format.js'; -export { SerializationFormat }; - -// ---- Format Prefix ---- - -/** - * A format prefix is exactly 4 lowercase alphanumeric characters [a-z0-9]. - * - * This is a branded string type — use `isFormatPrefix()` to validate - * at runtime. The `SerializationFormat` object provides well-known - * constants, but codecs may define additional prefixes. - */ -export type FormatPrefix = - | SerializationFormatType - | (string & { readonly __brand: 'FormatPrefix' }); - -/** - * Runtime type guard for format prefix strings. - * - * Validates that a string is exactly 4 characters of [a-z0-9]. - */ -export function isFormatPrefix(value: string): value is FormatPrefix { - return value.length === 4 && /^[a-z0-9]{4}$/.test(value); -} +export { + type FormatPrefix, + isFormatPrefix, + SerializationFormat, + type SerializationFormatType, +}; // ---- Serializable Types ---- diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index 26f6e10612..f1a10b04bb 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -7,7 +7,9 @@ */ import { + deriveRunPayloadKeys, extractClassName, + hydrateDataWithKey, hydrateResourceIO as hydrateResourceIOGeneric, isEncryptedData, isExpiredStub, @@ -18,6 +20,15 @@ import { } from '@workflow/core/serialization-format'; import { getEventDataRefFields } from '@workflow/world'; +const browserHydrationOptions = { + zstdDecoder: async (payload: Uint8Array): Promise => { + const { decompressZstdInBrowser } = await import( + './zstd-browser-decoder.js' + ); + return decompressZstdInBrowser(payload); + }, +}; + // Re-export types and utilities that consumers need export { CLASS_INSTANCE_REF_TYPE, @@ -487,11 +498,6 @@ export async function hydrateResourceIOAsync( resource: T, key?: Uint8Array ): Promise { - const [serialization, { decompressZstdInBrowser }] = await Promise.all([ - import('@workflow/core/serialization-format'), - import('./zstd-browser-decoder.js'), - ]); - const { hydrateDataWithKey, deriveRunPayloadKeys } = serialization; // Resolve the *full* key capability, not just the symmetric key: a run's // event log can contain sealed ('encp') payloads that another run wrote to // it (a cross-deployment hook resumption, say), and opening those needs the @@ -499,19 +505,23 @@ export async function hydrateResourceIOAsync( // 32 bytes the key-retrieval endpoint returns. const cryptoKey = key ? await deriveRunPayloadKeys(key) : undefined; const revivers = getRevivers(); - const hydrationOptions = { zstdDecoder: decompressZstdInBrowser }; async function hydrateField(value: unknown): Promise { // Already-hydrated: encrypted marker with stored bytes if (isEncryptedMarker(value)) { const raw = (value as any).__encryptedData as Uint8Array; return cryptoKey - ? hydrateDataWithKey(raw, revivers, cryptoKey, hydrationOptions) + ? hydrateDataWithKey(raw, revivers, cryptoKey, browserHydrationOptions) : value; } // Raw Uint8Array: may be encrypted, compressed, or plain devalue. if (value instanceof Uint8Array) { - return hydrateDataWithKey(value, revivers, cryptoKey, hydrationOptions); + return hydrateDataWithKey( + value, + revivers, + cryptoKey, + browserHydrationOptions + ); } // Not serialized — return as-is. return value; diff --git a/packages/world-vercel/src/serialized-data.test.ts b/packages/world-vercel/src/serialized-data.test.ts index a19a357acb..f1731f53dc 100644 --- a/packages/world-vercel/src/serialized-data.test.ts +++ b/packages/world-vercel/src/serialized-data.test.ts @@ -1,3 +1,4 @@ +import { gzipSync, zstdCompressSync } from 'node:zlib'; import { peekSerializationFormat } from '@workflow/world/serialization-format.js'; import { describe, expect, it } from 'vitest'; import { normalizeSerializedData } from './serialized-data.js'; @@ -28,4 +29,18 @@ describe('serialized data normalization', () => { const encrypted = envelope('encr', new Uint8Array([1, 2, 3])); expect(normalizeSerializedData(encrypted)).toBe(encrypted); }); + + it('decompresses gzip envelopes with the shared world codec', () => { + const original = new TextEncoder().encode('persisted workflow payload'); + const compressed = envelope('gzip', gzipSync(original)); + + expect(normalizeSerializedData(compressed)).toEqual(original); + }); + + it('decompresses zstd envelopes with the shared world codec', () => { + const original = new TextEncoder().encode('persisted workflow payload'); + const compressed = envelope('zstd', zstdCompressSync(original)); + + expect(normalizeSerializedData(compressed)).toEqual(original); + }); }); diff --git a/packages/world-vercel/src/serialized-data.ts b/packages/world-vercel/src/serialized-data.ts index 65e9c45f3d..5485aa55ce 100644 --- a/packages/world-vercel/src/serialized-data.ts +++ b/packages/world-vercel/src/serialized-data.ts @@ -1,29 +1,10 @@ -import * as nodeZlib from 'node:zlib'; import { WorkflowWorldError } from '@workflow/errors'; +import { decompressSerializedDataSync } from '@workflow/world/serialization-compression.js'; import { peekSerializationFormat, SerializationFormat, } from '@workflow/world/serialization-format.js'; -function decompress( - format: typeof SerializationFormat.GZIP | typeof SerializationFormat.ZSTD, - payload: Uint8Array -): Uint8Array { - const decompress = - format === SerializationFormat.ZSTD - ? nodeZlib.zstdDecompressSync - : nodeZlib.gunzipSync; - - if (!decompress) { - throw new WorkflowWorldError( - `Received ${format}-compressed workflow data, but this Node.js runtime does not support ${format} decompression. Use a compatible Node.js runtime or request unresolved data.` - ); - } - - const result = decompress(payload); - return new Uint8Array(result.buffer, result.byteOffset, result.byteLength); -} - export function normalizeSerializedData(value: unknown): unknown { const format = peekSerializationFormat(value); if ( @@ -33,7 +14,12 @@ export function normalizeSerializedData(value: unknown): unknown { return value; } const bytes = value as Uint8Array; - return decompress(format, bytes.subarray(format.length)); + const decompressed = decompressSerializedDataSync(bytes); + if (decompressed) return decompressed; + + throw new WorkflowWorldError( + `Received ${format}-compressed workflow data, but this Node.js runtime does not support ${format} decompression. Use a compatible Node.js runtime or request unresolved data.` + ); } export function normalizeWorkflowRunData>( diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 290c73c535..4e395f414e 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -103,8 +103,11 @@ export { SerializedDataSchema, } from './serialization.js'; export { + type FormatPrefix, + isFormatPrefix, isSerializationFormat, peekSerializationFormat, + SERIALIZATION_FORMAT_PREFIX_LENGTH, SerializationFormat, type SerializationFormatType, } from './serialization-format.js'; diff --git a/packages/world/src/serialization-compression.ts b/packages/world/src/serialization-compression.ts new file mode 100644 index 0000000000..9a0d1b413b --- /dev/null +++ b/packages/world/src/serialization-compression.ts @@ -0,0 +1,56 @@ +import { + peekSerializationFormat, + SERIALIZATION_FORMAT_PREFIX_LENGTH, + SerializationFormat, +} from './serialization-format.js'; + +interface NodeZlibDecode { + gunzipSync?: (data: Uint8Array) => Uint8Array; + zstdDecompressSync?: (data: Uint8Array) => Uint8Array; +} + +/** Resolve Node codecs without introducing a static Node import for browsers. */ +const nodeZlib = (() => { + try { + return ( + globalThis as { + process?: { getBuiltinModule?: (id: string) => NodeZlibDecode }; + } + ).process?.getBuiltinModule?.('node:zlib'); + } catch { + return undefined; + } +})(); + +function asUint8Array(value: Uint8Array): Uint8Array { + return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); +} + +/** + * Inflate a persisted gzip/zstd envelope synchronously when Node exposes the + * corresponding codec. Non-compressed bytes pass through unchanged; an + * unavailable codec returns `undefined` so the owning runtime can choose its + * fallback or error contract. + */ +export function decompressSerializedDataSync( + data: Uint8Array +): Uint8Array | undefined { + const format = peekSerializationFormat(data); + const decompress = + format === SerializationFormat.ZSTD + ? nodeZlib?.zstdDecompressSync + : format === SerializationFormat.GZIP + ? nodeZlib?.gunzipSync + : undefined; + + if (!decompress) { + return format === SerializationFormat.ZSTD || + format === SerializationFormat.GZIP + ? undefined + : data; + } + + return asUint8Array( + decompress(data.subarray(SERIALIZATION_FORMAT_PREFIX_LENGTH)) + ); +} diff --git a/packages/world/src/serialization-format.ts b/packages/world/src/serialization-format.ts index cbcf958c90..65dd20a4e5 100644 --- a/packages/world/src/serialization-format.ts +++ b/packages/world/src/serialization-format.ts @@ -1,15 +1,40 @@ +declare const formatPrefixBrand: unique symbol; + +/** A validated four-byte lowercase alphanumeric payload envelope prefix. */ +export type FormatPrefix = string & { + readonly [formatPrefixBrand]: 'FormatPrefix'; +}; + +/** Whether a value is a valid persisted payload envelope prefix. */ +export function isFormatPrefix(value: unknown): value is FormatPrefix { + return ( + typeof value === 'string' && + value.length === 4 && + /^[a-z0-9]{4}$/.test(value) + ); +} + +function defineFormatPrefix( + value: T +): T & FormatPrefix { + if (!isFormatPrefix(value)) { + throw new Error(`Invalid serialization format prefix: ${value}`); + } + return value; +} + /** Known four-byte envelope prefixes in the persisted payload protocol. */ export const SerializationFormat = { /** devalue stringify/parse with TextEncoder/TextDecoder */ - DEVALUE_V1: 'devl', + DEVALUE_V1: defineFormatPrefix('devl'), /** Symmetrically encrypted payload */ - ENCRYPTED: 'encr', + ENCRYPTED: defineFormatPrefix('encr'), /** Payload sealed to a run's public key */ - SEALED: 'encp', + SEALED: defineFormatPrefix('encp'), /** Gzip-compressed payload */ - GZIP: 'gzip', + GZIP: defineFormatPrefix('gzip'), /** Zstandard-compressed payload */ - ZSTD: 'zstd', + ZSTD: defineFormatPrefix('zstd'), } as const; export type SerializationFormatType = @@ -26,7 +51,7 @@ export function isSerializationFormat( return typeof value === 'string' && serializationFormats.has(value); } -const FORMAT_PREFIX_LENGTH = 4; +export const SERIALIZATION_FORMAT_PREFIX_LENGTH = 4; const formatDecoder = new TextDecoder(); /** Read a known persisted payload format without consuming its bytes. */ @@ -35,11 +60,13 @@ export function peekSerializationFormat( ): SerializationFormatType | null { if ( !(value instanceof Uint8Array) || - value.byteLength < FORMAT_PREFIX_LENGTH + value.byteLength < SERIALIZATION_FORMAT_PREFIX_LENGTH ) { return null; } - const format = formatDecoder.decode(value.subarray(0, FORMAT_PREFIX_LENGTH)); + const format = formatDecoder.decode( + value.subarray(0, SERIALIZATION_FORMAT_PREFIX_LENGTH) + ); return isSerializationFormat(format) ? format : null; }