From 3068028d91b6595c9f9f07aa4bf6ecde589f75ad Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 24 Jul 2026 14:00:57 +0200 Subject: [PATCH 1/6] =?UTF-8?q?test(=F0=9F=A7=AA):=20add=20ImageBitmap=20a?= =?UTF-8?q?lpha=20and=20options=20test=20suites?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two snapshot-based suites covering createImageBitmap and copyExternalImageToTexture, with baselines validated against Chrome (yarn test:ref) and the dawn.node client (yarn test:node): - ImageBitmapAlpha.spec.ts: full matrix of premultiplyAlpha x destination premultipliedAlpha x flipY, over both the blob and the ArrayBuffer overload, plus exact-value checks where losslessness is guaranteed. The alpha baselines encode the reference integer conversion math; native platforms pass once straight-alpha support lands (issue #432 / PR #433). - ImageBitmapOptions.spec.ts: crop-rect overload, resizeWidth/Height, imageOrientation, and colorSpaceConversion (Display-P3 fixture generated by Chrome). Native currently ignores these options, so on iOS/Android each case asserts today's identity behavior and flips to the reference snapshot once the option is implemented. setup.ts: the node client polyfills now implement premultiplyAlpha, crop, bilinear resize, flipY orientation, close(), and alpha-aware copyExternalImageToTexture with the same integer math as the native ImageBitmap, so reference results can be generated and cross-checked locally. Co-Authored-By: Claude Fable 5 --- .../src/__tests__/ImageBitmapAlpha.spec.ts | 348 ++++++++++++++++++ .../src/__tests__/ImageBitmapOptions.spec.ts | 183 +++++++++ .../src/__tests__/assets/alpha-gradient.png | Bin 0 -> 204 bytes .../src/__tests__/assets/opaque-gradient.png | Bin 0 -> 189 bytes .../src/__tests__/assets/p3-gradient.png | Bin 0 -> 583 bytes packages/webgpu/src/__tests__/setup.ts | 224 ++++++++++- .../image-bitmap-alpha-premultiplied.png | Bin 0 -> 1365 bytes .../image-bitmap-alpha-roundtrip.png | Bin 0 -> 1119 bytes .../snapshots/image-bitmap-alpha-straight.png | Bin 0 -> 204 bytes .../image-bitmap-options-crop-resize.png | Bin 0 -> 196 bytes .../snapshots/image-bitmap-options-crop.png | Bin 0 -> 155 bytes .../snapshots/image-bitmap-options-flip.png | Bin 0 -> 190 bytes .../image-bitmap-options-p3-default.png | Bin 0 -> 1116 bytes .../image-bitmap-options-resize-down.png | Bin 0 -> 138 bytes .../image-bitmap-options-resize-up.png | Bin 0 -> 264 bytes packages/webgpu/src/index.tsx | 1 + 16 files changed, 741 insertions(+), 15 deletions(-) create mode 100644 packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts create mode 100644 packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts create mode 100644 packages/webgpu/src/__tests__/assets/alpha-gradient.png create mode 100644 packages/webgpu/src/__tests__/assets/opaque-gradient.png create mode 100644 packages/webgpu/src/__tests__/assets/p3-gradient.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-premultiplied.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-roundtrip.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-straight.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-options-crop-resize.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-options-crop.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-options-flip.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-options-p3-default.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-options-resize-down.png create mode 100644 packages/webgpu/src/__tests__/snapshots/image-bitmap-options-resize-up.png diff --git a/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts new file mode 100644 index 0000000000..1767ed1a7c --- /dev/null +++ b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts @@ -0,0 +1,348 @@ +import fs from "fs"; +import path from "path"; + +import { PNG } from "pngjs"; + +import { checkImage, client, encodeImage } from "./setup"; + +// A 32x32 gradient with a vertical alpha ramp (row 0 fully transparent, last +// row fully opaque) and a horizontal color ramp, so alpha-mode mistakes and +// vertical flips both produce large snapshot diffs. The baselines are computed +// with the same integer arithmetic as ImageBitmap::convertAlpha; pixelmatch +// tolerance absorbs the platform decoders' off-by-one rounding differences +// (e.g. Android decodes premultiplied, so "none" is a lossy round trip there). +const assetPath = path.resolve(__dirname, "./assets/alpha-gradient.png"); +const pngBase64 = fs.readFileSync(assetPath).toString("base64"); + +type SourceAlpha = "none" | "premultiply" | "default" | "omitted"; +type DestinationAlpha = boolean | "omitted"; + +interface BitmapResult { + data: number[]; + width: number; + height: number; + format: string; +} + +const flipRows = (bitmap: BitmapResult): BitmapResult => { + const rowSize = bitmap.width * 4; + const data: number[] = []; + for (let y = bitmap.height - 1; y >= 0; y--) { + data.push(...bitmap.data.slice(y * rowSize, (y + 1) * rowSize)); + } + return { ...bitmap, data }; +}; + +const snapshotFor = ( + sourceAlpha: SourceAlpha, + destinationAlpha: DestinationAlpha, +) => { + if (destinationAlpha === true) { + return "snapshots/image-bitmap-alpha-premultiplied.png"; + } + if (sourceAlpha === "none") { + return "snapshots/image-bitmap-alpha-straight.png"; + } + // "premultiply", "default", and omitted options all store premultiplied + // data, so copying to a straight-alpha destination is a lossy round trip. + return "snapshots/image-bitmap-alpha-roundtrip.png"; +}; + +interface BlobCase { + sourceAlpha: SourceAlpha; + destinationAlpha: DestinationAlpha; + flipY: boolean; +} + +const blobCases: BlobCase[] = ( + ["none", "premultiply", "omitted"] as const +).flatMap((sourceAlpha) => + (["omitted", false, true] as const).flatMap((destinationAlpha) => + ([false, true] as const).map((flipY) => ({ + sourceAlpha, + destinationAlpha, + flipY, + })), + ), +); +// premultiplyAlpha: "default" behaves like "premultiply"; one case is enough. +blobCases.push({ + sourceAlpha: "default", + destinationAlpha: false, + flipY: false, +}); + +describe("ImageBitmap alpha representation", () => { + it.each(blobCases)( + "blob source=$sourceAlpha destination=$destinationAlpha flipY=$flipY", + async ({ sourceAlpha, destinationAlpha, flipY }) => { + const result = await client.eval( + ({ + device, + pngBase64: encodedPng, + sourceAlpha: sourceRepresentation, + destinationAlpha: destinationRepresentation, + flipY: shouldFlip, + }) => { + return fetch(`data:image/png;base64,${encodedPng}`) + .then((response) => response.blob()) + .then((blob) => + createImageBitmap( + blob, + sourceRepresentation === "omitted" + ? undefined + : { premultiplyAlpha: sourceRepresentation }, + ), + ) + .then((bitmap) => { + const { width } = bitmap; + const { height } = bitmap; + const texture = device.createTexture({ + size: [width, height], + format: "rgba8unorm", + usage: + GPUTextureUsage.COPY_DST | + GPUTextureUsage.COPY_SRC | + GPUTextureUsage.RENDER_ATTACHMENT, + }); + device.queue.copyExternalImageToTexture( + { source: bitmap, flipY: shouldFlip }, + destinationRepresentation === "omitted" + ? { texture } + : { texture, premultipliedAlpha: destinationRepresentation }, + [width, height], + ); + + const bytesPerRow = 256; + const output = device.createBuffer({ + size: bytesPerRow * height, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [width, height], + ); + device.queue.submit([encoder.finish()]); + + return output.mapAsync(GPUMapMode.READ).then(() => { + const mapped = new Uint8Array(output.getMappedRange()); + const data: number[] = []; + for (let y = 0; y < height; y++) { + for (let i = 0; i < width * 4; i++) { + data.push(mapped[y * bytesPerRow + i]); + } + } + output.unmap(); + output.destroy(); + texture.destroy(); + bitmap.close(); + return { data, width, height, format: "rgba8unorm" }; + }); + }); + }, + { + pngBase64, + sourceAlpha, + destinationAlpha, + flipY, + }, + ); + + const upright = flipY ? flipRows(result) : result; + checkImage( + encodeImage(upright), + snapshotFor(sourceAlpha, destinationAlpha), + ); + }, + ); + + // The React Native ArrayBuffer overload of createImageBitmap goes through + // createImageBitmapFromDataAsync, a separate native path from the blob one. + // It is not part of the standard web API, so the reference client skips it. + const bufferCases: { + sourceAlpha: "none" | "premultiply"; + destinationAlpha: boolean; + }[] = (["none", "premultiply"] as const).flatMap((sourceAlpha) => + ([false, true] as const).map((destinationAlpha) => ({ + sourceAlpha, + destinationAlpha, + })), + ); + it.each(bufferCases)( + "buffer source=$sourceAlpha destination=$destinationAlpha", + async ({ sourceAlpha, destinationAlpha }) => { + if (client.OS === "web") { + return; + } + const pngData = Array.from(fs.readFileSync(assetPath)); + const result = await client.eval( + ({ + device, + pngData: encodedPng, + sourceAlpha: sourceRepresentation, + destinationAlpha: destinationRepresentation, + }) => { + const bytes = new Uint8Array(encodedPng); + return createImageBitmap(bytes.buffer, { + premultiplyAlpha: sourceRepresentation, + }).then((bitmap) => { + const { width } = bitmap; + const { height } = bitmap; + const texture = device.createTexture({ + size: [width, height], + format: "rgba8unorm", + usage: + GPUTextureUsage.COPY_DST | + GPUTextureUsage.COPY_SRC | + GPUTextureUsage.RENDER_ATTACHMENT, + }); + device.queue.copyExternalImageToTexture( + { source: bitmap }, + { texture, premultipliedAlpha: destinationRepresentation }, + [width, height], + ); + + const bytesPerRow = 256; + const output = device.createBuffer({ + size: bytesPerRow * height, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [width, height], + ); + device.queue.submit([encoder.finish()]); + + return output.mapAsync(GPUMapMode.READ).then(() => { + const mapped = new Uint8Array(output.getMappedRange()); + const data: number[] = []; + for (let y = 0; y < height; y++) { + for (let i = 0; i < width * 4; i++) { + data.push(mapped[y * bytesPerRow + i]); + } + } + output.unmap(); + output.destroy(); + texture.destroy(); + bitmap.close(); + return { data, width, height, format: "rgba8unorm" }; + }); + }); + }, + { + pngData, + sourceAlpha, + destinationAlpha, + }, + ); + + checkImage( + encodeImage(result), + snapshotFor(sourceAlpha, destinationAlpha), + ); + }, + ); + + // Exact-value check for the lossless straight-alpha guarantee this feature + // adds on Apple: premultiplyAlpha "none" must preserve the decoded bytes + // bit-for-bit, and conversions must use round-to-nearest. Runs where + // exactness is guaranteed: iOS (Core Image decode) and the node client + // (whose polyfill mirrors the C++ integer math). Android decodes + // premultiplied, so "none" is lossy there; the reference browser's rounding + // may legitimately differ by one. Both are covered by the snapshot matrix. + const straightRows = [ + [128, 128, 128, 128], + [17, 34, 51, 64], + ]; + const premultipliedRows = [ + [64, 64, 64, 128], + [4, 9, 13, 64], + ]; + it.each([ + { sourceAlpha: "none", destinationAlpha: false, expected: straightRows }, + { + sourceAlpha: "none", + destinationAlpha: true, + expected: premultipliedRows, + }, + ] as const)( + "preserves exact bytes for source=$sourceAlpha destination=$destinationAlpha", + async ({ sourceAlpha, destinationAlpha, expected }) => { + if (client.OS !== "ios" && client.OS !== "node") { + return; + } + const png = new PNG({ width: 1, height: 2 }); + png.data = Buffer.from(straightRows.flat()); + const fixtureBase64 = PNG.sync.write(png).toString("base64"); + + const result = await client.eval( + ({ + device, + pngBase64: encodedPng, + sourceAlpha: sourceRepresentation, + destinationAlpha: destinationRepresentation, + }) => { + return fetch(`data:image/png;base64,${encodedPng}`) + .then((response) => response.blob()) + .then((blob) => + createImageBitmap(blob, { + premultiplyAlpha: sourceRepresentation, + }), + ) + .then((bitmap) => { + const texture = device.createTexture({ + size: [1, 2], + format: "rgba8unorm", + usage: + GPUTextureUsage.COPY_DST | + GPUTextureUsage.COPY_SRC | + GPUTextureUsage.RENDER_ATTACHMENT, + }); + device.queue.copyExternalImageToTexture( + { source: bitmap }, + { texture, premultipliedAlpha: destinationRepresentation }, + [1, 2], + ); + + const bytesPerRow = 256; + const output = device.createBuffer({ + size: bytesPerRow * 2, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [1, 2], + ); + device.queue.submit([encoder.finish()]); + + return output.mapAsync(GPUMapMode.READ).then(() => { + const data = new Uint8Array(output.getMappedRange()); + const rows = [ + Array.from(data.slice(0, 4)), + Array.from(data.slice(bytesPerRow, bytesPerRow + 4)), + ]; + output.unmap(); + output.destroy(); + texture.destroy(); + bitmap.close(); + return rows; + }); + }); + }, + { + pngBase64: fixtureBase64, + sourceAlpha, + destinationAlpha, + }, + ); + + expect(result).toEqual(expected); + }, + ); +}); diff --git a/packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts b/packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts new file mode 100644 index 0000000000..16256fab3d --- /dev/null +++ b/packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts @@ -0,0 +1,183 @@ +import fs from "fs"; +import path from "path"; + +import { checkImage, client, encodeImage } from "./setup"; + +// createImageBitmap options beyond premultiplyAlpha (which is covered by +// ImageBitmapAlpha.spec.ts): the crop-rect overload, resizeWidth/resizeHeight, +// imageOrientation, and colorSpaceConversion. +// +// The native implementation currently ignores all of these, so this suite has +// a dual role: +// - On the reference clients (Chrome via test:ref, dawn.node via test:node) +// it asserts the spec behavior. The snapshots were written by Chrome, and +// the node polyfill independently reproduces them (crop and flip exactly, +// resize within pixelmatch tolerance). +// - On iOS and Android it asserts the current behavior: the option is ignored +// and the full, upright, unresized image comes through. When one of these +// options gets implemented natively, the corresponding case fails here and +// should be flipped over to the reference snapshot. +const assetPath = path.resolve(__dirname, "./assets/opaque-gradient.png"); +const pngBase64 = fs.readFileSync(assetPath).toString("base64"); +const p3AssetPath = path.resolve(__dirname, "./assets/p3-gradient.png"); +const p3Base64 = fs.readFileSync(p3AssetPath).toString("base64"); + +const identitySnapshot = "assets/opaque-gradient.png"; + +interface OptionsCase { + name: string; + cropRect?: [number, number, number, number]; + options?: ImageBitmapOptions; + referenceSnapshot: string; +} + +const cases: OptionsCase[] = [ + { + name: "crop", + cropRect: [8, 4, 16, 24], + referenceSnapshot: "snapshots/image-bitmap-options-crop.png", + }, + { + name: "resize down", + options: { resizeWidth: 16, resizeHeight: 16 }, + referenceSnapshot: "snapshots/image-bitmap-options-resize-down.png", + }, + { + name: "resize up", + options: { resizeWidth: 64, resizeHeight: 48 }, + referenceSnapshot: "snapshots/image-bitmap-options-resize-up.png", + }, + { + name: "crop and resize", + cropRect: [8, 4, 16, 24], + options: { resizeWidth: 32, resizeHeight: 32 }, + referenceSnapshot: "snapshots/image-bitmap-options-crop-resize.png", + }, + { + name: "imageOrientation flipY", + options: { imageOrientation: "flipY" }, + referenceSnapshot: "snapshots/image-bitmap-options-flip.png", + }, + { + // No EXIF data in a PNG, so "from-image" must behave like the identity. + name: "imageOrientation from-image", + options: { imageOrientation: "from-image" }, + referenceSnapshot: identitySnapshot, + }, +]; + +const runCase = ( + encodedPng: string, + cropRect: [number, number, number, number] | null, + options: ImageBitmapOptions | null, +) => + client.eval( + ({ device, pngBase64: png, cropRect: rect, options: bitmapOptions }) => { + return fetch(`data:image/png;base64,${png}`) + .then((response) => response.blob()) + .then((blob) => + rect === null + ? createImageBitmap(blob, bitmapOptions ?? undefined) + : createImageBitmap( + blob, + rect[0], + rect[1], + rect[2], + rect[3], + bitmapOptions ?? undefined, + ), + ) + .then((bitmap) => { + const { width, height } = bitmap; + const texture = device.createTexture({ + size: [width, height], + format: "rgba8unorm", + usage: + GPUTextureUsage.COPY_DST | + GPUTextureUsage.COPY_SRC | + GPUTextureUsage.RENDER_ATTACHMENT, + }); + device.queue.copyExternalImageToTexture( + { source: bitmap }, + { texture }, + [width, height], + ); + + const bytesPerRow = Math.ceil((width * 4) / 256) * 256; + const output = device.createBuffer({ + size: bytesPerRow * height, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [width, height], + ); + device.queue.submit([encoder.finish()]); + + return output.mapAsync(GPUMapMode.READ).then(() => { + const mapped = new Uint8Array(output.getMappedRange()); + const data: number[] = []; + for (let y = 0; y < height; y++) { + for (let i = 0; i < width * 4; i++) { + data.push(mapped[y * bytesPerRow + i]); + } + } + output.unmap(); + output.destroy(); + texture.destroy(); + bitmap.close(); + return { data, width, height, format: "rgba8unorm" }; + }); + }); + }, + { + pngBase64: encodedPng, + cropRect, + options, + }, + ); + +describe("createImageBitmap options", () => { + it.each(cases)("$name", async ({ cropRect, options, referenceSnapshot }) => { + const result = await runCase(pngBase64, cropRect ?? null, options ?? null); + const isReference = client.OS === "web" || client.OS === "node"; + checkImage( + encodeImage(result), + isReference ? referenceSnapshot : identitySnapshot, + ); + }); + + // colorSpaceConversion on a Display-P3 tagged PNG (generated by Chrome, so + // the profile chunk is exactly what a browser produces). "none" must return + // the raw encoded values, which is also what pngjs decodes the asset to, so + // the baseline is the asset itself. "default" converts P3 to sRGB and only + // the browser implements that, so it runs against a Chrome-written snapshot + // on the reference client only. Native behavior is currently inconsistent + // between the straight-alpha (Core Image, unmanaged) and premultiplied + // (CoreGraphics, device RGB) decode paths; these cases are skipped there + // until that behavior is measured on-device and pinned down. + it("colorSpaceConversion none returns raw pixel values", async () => { + if (client.OS !== "web" && client.OS !== "node") { + return; + } + const result = await runCase(p3Base64, null, { + colorSpaceConversion: "none", + }); + checkImage(encodeImage(result), "assets/p3-gradient.png"); + }); + + it("colorSpaceConversion default converts to sRGB", async () => { + if (client.OS !== "web") { + return; + } + const result = await runCase(p3Base64, null, { + colorSpaceConversion: "default", + }); + checkImage( + encodeImage(result), + "snapshots/image-bitmap-options-p3-default.png", + ); + }); +}); diff --git a/packages/webgpu/src/__tests__/assets/alpha-gradient.png b/packages/webgpu/src/__tests__/assets/alpha-gradient.png new file mode 100644 index 0000000000000000000000000000000000000000..e247d7112ceeed21a1ce8a27e5f53f564ba3dde1 GIT binary patch literal 204 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=ffJ$(}BbAr*{A4{pq5apYkzlnOAo zUB7Ypo7gsw>;JxJYrp=PlA3Bee+>{kpFDNy*P2b6HqG-73k$pU`Q)inRd$h)%9bYc zo?r0b>wN+uo?Dtsas~>439RCeE1^o!)WgL8GB37KDp$F^`3%r`44$rjF6*2UngDq@ BQX2pO literal 0 HcmV?d00001 diff --git a/packages/webgpu/src/__tests__/assets/opaque-gradient.png b/packages/webgpu/src/__tests__/assets/opaque-gradient.png new file mode 100644 index 0000000000000000000000000000000000000000..2ad649ad2d28b70ea29995aa1d4291db1dec7e86 GIT binary patch literal 189 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=ffJ7Ec$)kP6152RG)jI0`TuNChl- zyZ^@QH?eIe?CbVhmCo~@3ngCJ%(p5maVTvG1{&UMKRp0%P&H{7~gQu&X%Q~loCIG$fL>~YE literal 0 HcmV?d00001 diff --git a/packages/webgpu/src/__tests__/assets/p3-gradient.png b/packages/webgpu/src/__tests__/assets/p3-gradient.png new file mode 100644 index 0000000000000000000000000000000000000000..67a5ab282dd6c7c428c34b6db50a79ef71e4f386 GIT binary patch literal 583 zcmV-N0=WH&P)KZM}yC_-lk z1wn8WKY}>8#wQ7qE**T2_wIMkJ>1>!6a1&w;mIBA_4)1ORfq+wd2&*YHn z=RNbjt?>Vt?bxBE^-I|a^ti5UikF;>dkMEQzDjr)`H}c0;h@>r5#Nf-&KS4H*z$wK z|192e`vWWgHznIUKGM3UTxOSZLVWs|Tw-v@hUU6zV6D5Rm2ny5)2hbN_r zEhgz-_QdrEY>(nNe$2Il5!_c{?jhI8o3K!Vr%SJOqID{)QT+uVlui4zg2_Ap007BJL_t(|0qmC%7Xm>T zMeiBfEDggj4Z|=@!!S(4FigWR4AXFNadmO=w(|?3&bKh|_`UPqJ!j@ipcTDSI%l-b zX { + // Same integer arithmetic as ImageBitmap::convertAlpha (ImageBitmap.h), + // so this client is a bit-exact reference for the native conversion. + const convertAlpha = ( + data: Uint8Array | Uint8ClampedArray, + sourcePremultiplied: boolean, + destinationPremultiplied: boolean, + ) => { + if (sourcePremultiplied === destinationPremultiplied) { + return; + } + for (let i = 0; i + 3 < data.length; i += 4) { + const alpha = data[i + 3]; + for (let channel = 0; channel < 3; channel++) { + const value = data[i + channel]; + if (destinationPremultiplied) { + data[i + channel] = Math.floor((value * alpha + 127) / 255); + } else if (alpha === 0) { + data[i + channel] = 0; + } else { + data[i + channel] = Math.min( + 255, + Math.floor((value * 255 + (alpha >> 1)) / alpha), + ); + } + } + } + }; + interface RawImage { + data: Uint8ClampedArray; + width: number; + height: number; + } + const crop = ( + image: RawImage, + sx: number, + sy: number, + sw: number, + sh: number, + ): RawImage => { + const data = new Uint8ClampedArray(sw * sh * 4); + for (let y = 0; y < sh; y++) { + const from = ((sy + y) * image.width + sx) * 4; + data.set(image.data.subarray(from, from + sw * 4), y * sw * 4); + } + return { data, width: sw, height: sh }; + }; + // Bilinear resampling with edge clamp; browsers use comparable filtering + // for the default resizeQuality, and the specs compare with pixelmatch + // tolerance rather than bit-exactly. + const resize = ( + image: RawImage, + width: number, + height: number, + ): RawImage => { + const data = new Uint8ClampedArray(width * height * 4); + for (let y = 0; y < height; y++) { + const srcY = Math.min( + image.height - 1, + Math.max(0, ((y + 0.5) * image.height) / height - 0.5), + ); + const y0 = Math.floor(srcY); + const y1 = Math.min(image.height - 1, y0 + 1); + const fy = srcY - y0; + for (let x = 0; x < width; x++) { + const srcX = Math.min( + image.width - 1, + Math.max(0, ((x + 0.5) * image.width) / width - 0.5), + ); + const x0 = Math.floor(srcX); + const x1 = Math.min(image.width - 1, x0 + 1); + const fx = srcX - x0; + for (let channel = 0; channel < 4; channel++) { + const top = + image.data[(y0 * image.width + x0) * 4 + channel] * (1 - fx) + + image.data[(y0 * image.width + x1) * 4 + channel] * fx; + const bottom = + image.data[(y1 * image.width + x0) * 4 + channel] * (1 - fx) + + image.data[(y1 * image.width + x1) * 4 + channel] * fx; + data[(y * width + x) * 4 + channel] = Math.round( + top * (1 - fy) + bottom * fy, + ); + } + } + } + return { data, width, height }; + }; + const flipVertically = (image: RawImage): RawImage => { + const rowSize = image.width * 4; + const data = new Uint8ClampedArray(image.data.length); + for (let y = 0; y < image.height; y++) { + data.set( + image.data.subarray(y * rowSize, (y + 1) * rowSize), + (image.height - 1 - y) * rowSize, + ); + } + return { ...image, data }; + }; + interface PolyfillImageBitmapOptions { + premultiplyAlpha?: string; + imageOrientation?: string; + resizeWidth?: number; + resizeHeight?: number; + } + const decodePng = ( + bytes: Uint8Array, + cropRect: number[] | undefined, + options: PolyfillImageBitmapOptions | undefined, + ) => { const png = PNG.sync.read( Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), ); - return { + // Spec processing order: crop, then resize, then orientation, then + // the requested alpha representation. pngjs ignores embedded color + // profiles, so this decoder behaves like colorSpaceConversion "none". + let image: RawImage = { data: new Uint8ClampedArray(png.data), width: png.width, height: png.height, - close() {}, }; + if (cropRect !== undefined) { + image = crop(image, cropRect[0], cropRect[1], cropRect[2], cropRect[3]); + } + if ( + options?.resizeWidth !== undefined || + options?.resizeHeight !== undefined + ) { + image = resize( + image, + options.resizeWidth ?? image.width, + options.resizeHeight ?? image.height, + ); + } + if (options?.imageOrientation === "flipY") { + image = flipVertically(image); + } + // Like the native implementation, "default" premultiplies. + const premultiplied = options?.premultiplyAlpha !== "none"; + convertAlpha(image.data, false, premultiplied); + const bitmap = { + data: image.data, + width: image.width, + height: image.height, + premultiplied, + // Mirror the native ImageBitmap: close() releases the pixels and + // zeroes the dimensions. + close() { + bitmap.data = new Uint8ClampedArray(0); + bitmap.width = 0; + bitmap.height = 0; + }, + }; + return bitmap; }; (globalThis as Record).createImageBitmap = async ( source: unknown, + ...rest: unknown[] ) => { + // Both overloads: (source, options?) and (source, sx, sy, sw, sh, + // options?). + const cropRect = + typeof rest[0] === "number" + ? (rest.slice(0, 4) as number[]) + : undefined; + const options = (cropRect !== undefined ? rest[4] : rest[0]) as + | PolyfillImageBitmapOptions + | undefined; if (source instanceof ArrayBuffer) { - return decodePng(new Uint8Array(source)); + return decodePng(new Uint8Array(source), cropRect, options); } if (ArrayBuffer.isView(source)) { return decodePng( new Uint8Array(source.buffer, source.byteOffset, source.byteLength), + cropRect, + options, ); } if (typeof Blob !== "undefined" && source instanceof Blob) { - return decodePng(new Uint8Array(await source.arrayBuffer())); + return decodePng( + new Uint8Array(await source.arrayBuffer()), + cropRect, + options, + ); } if ( source !== null && @@ -518,30 +676,66 @@ class NodeTestingClient implements TestingClient { "width" in source ) { // Already an ImageData-like object (e.g. one of the test assets). + // Left untagged so the copy shim writes its bytes through unchanged. return source; } throw new Error("createImageBitmap polyfill: unsupported source"); }; // copyExternalImageToTexture expects an ImageBitmap; route the raw RGBA - // bytes of our ImageData-like sources through writeTexture instead. + // bytes of our ImageData-like sources through writeTexture instead, + // honoring flipY and the alpha representations on both sides. Object.defineProperty(device.queue, "copyExternalImageToTexture", { configurable: true, value: ( source: { - source: { data: Uint8ClampedArray; width: number; height: number }; + source: { + data: Uint8ClampedArray; + width: number; + height: number; + premultiplied?: boolean; + }; + flipY?: boolean; }, - destination: GPUTexelCopyTextureInfo, + destination: GPUTexelCopyTextureInfo & { premultipliedAlpha?: boolean }, copySize: GPUExtent3DStrict, ) => { - const { data, width } = source.source; + const { data, width, height, premultiplied } = source.source; + const rowSize = width * 4; + let bytes = new Uint8Array( + data.buffer as ArrayBuffer, + data.byteOffset, + data.byteLength, + ); + const flipY = source.flipY === true; + // Untagged sources (raw test assets) are copied through unchanged. + const needsConversion = + premultiplied !== undefined && + premultiplied !== (destination.premultipliedAlpha === true); + if (flipY || needsConversion) { + const converted = new Uint8Array(bytes.length); + if (flipY) { + for (let row = 0; row < height; row++) { + converted.set( + bytes.subarray(row * rowSize, (row + 1) * rowSize), + (height - 1 - row) * rowSize, + ); + } + } else { + converted.set(bytes); + } + if (needsConversion) { + convertAlpha( + converted, + premultiplied === true, + destination.premultipliedAlpha === true, + ); + } + bytes = converted; + } device.queue.writeTexture( destination, - new Uint8Array( - data.buffer as ArrayBuffer, - data.byteOffset, - data.byteLength, - ), - { bytesPerRow: width * 4 }, + bytes, + { bytesPerRow: rowSize }, copySize, ); }, diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-premultiplied.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-premultiplied.png new file mode 100644 index 0000000000000000000000000000000000000000..935053edaaea4819f2dbb64138ea2acba00ee0fb GIT binary patch literal 1365 zcmY+Ee@Gi=7{?v7U5M+duBK{i?J9v}-h~-LsZ<*3&bVGiX=K%ybZ46^aGOcDWFTs- z1tBzPehB5f{HN&QU)H7PM@AE$I z=l#CV`@HwQz6 zS_hLaZ~f=Mz=;JEI$Z$9tO@Un%6k19vwXb)Oqh_6Sj!rRr^aqBc*|kbohUX{A};nt zL)liguW|VDuNzkq;A#>b;7=EA>dhRc)@Y^bC20vgFm|fSmGA@O^fSV(*OCg-+!_$xSh}d1$w98ACkU4WU!%>DoLQadYhYU3}w$0@(68Rs9 z8SXKG;$3H8G@@q&y&N@Dq%bL%?vB?HbGdpVf05W^=Cjjcp+`Eh)2-KHCQ@4y#*JtTf0k0%L13cZFo7sH`UQy7|RIMV*kVoEfCr zLAKT3L6fXtA`x44hP0_NEaq3p%n0)5*7Nc@VRCe&9al|V=af^)3Y0eSEw(GC%@$rm zN2w_!RbfC7M5ogYJ}-Zit3Q)tc0PdJ(A}u{$qNCdQaE8ayNro1nE76ZvnRZ|qK+75DFbNAEtgEY1IKXc zIF4Zx7f23BQ0;f7R^sv4%x6T{t-Ajpp@=_7dP+0hOCL>?8KoYL)YAf3%8jH7=1_i( zusozhL9rw~@*`^G#@^2Iq!_1BDUci+fc*qEV-RBe5)tnG;y$Rl{Q#p^Vf~>o$qC7c zVU5(z38CJiroAhakIh2}|7fCWm;deX19oSt-|MY9I}xbzXwpjN=q3pIry*8Lh%n8$u%?nP9(bEfxwe2pT5^=i3`4V3$ zjFH3>D#MYbOI_(X!+C*PUxJe;Te%%0gwsC~|H!5d4VrEti&$Uk%v|@}{uqVldG2}c zzQ6DH^Lzfd;c8tOlqOG;NF-4C$5;(`z9Y_8PJ*?wuWwHxdFkD9>_ejW1pg3k&+?ym z`KLbNtKzRueTmw)hma>H(=#sJ!VX5V9z8UBU)$`c-FmM0{23>^QF0XZl-(HE|9xbr zcdzG#Rxe$D)%@>BH_boKhu z(KkOHCS1=CH+he5Y%8*IB^7JljjnlQ`PY{%Ps3Xzde>?Xo4;}78_%YlDL%L%aui)Y+hVV58%q2L_IA%7i2f!Ey7RrUE) zES5amdfVz7W18gFi)?hW-ANI3~$@Y0m<4M5bGbG znNi3_F{PRi=}hXR;2VKeaJ4A@HmCW63V4pX>95b&}Ps?cacN@Sr$lWoc+O{3l14 z7zU8gQWysp0pkD$WX3^NAU3uP#KWKk>%~^NgJg>UT7gzj@MyoSnJ)YTRXX}wG6kD%u@#WC8 zgDQh<)MPZni@i-}YZNeS6WX7RQ0L;K*l|8Q^u1xYZ$3PJNs3p5%0N#!8a*~h%n)w@ zq;R%=hY`Jzcr`vl44mG-z#77d?hk-MBmg2QZey5HMC#f|%66{e+#U{Gaw<#t4t_!i zFB+yIo;JxJYrp=PlA3Bee+>{kpFDNy*P2b6HqG-73k$pU`Q)inRd$h)%9bYc zo?r0b>wN+uo?Dtsas~>439RCeE1^o!)WgL8GB37KDp$F^`3%r`44$rjF6*2UngDq@ BQX2pO literal 0 HcmV?d00001 diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-crop-resize.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-crop-resize.png new file mode 100644 index 0000000000000000000000000000000000000000..6d56b3141a40c92609cafa156f77752c60b341df GIT binary patch literal 196 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=ffJZci7-kP6152N?@x7zLOOR3G0y z9H{kFHs!r1_u)&|@7vq|-}ko*4615eT*1KBZV?2?$p?bL{ko&cTIWLMtgP@oIv(=&Owsdv+R)%k}{}F|H}Vdca@@i`V%dI&SLO%^>bP0l+XkKM`%l{ literal 0 HcmV?d00001 diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-crop.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-crop.png new file mode 100644 index 0000000000000000000000000000000000000000..a6990c19444dc3aab1b619039ce3d448ef8943d6 GIT binary patch literal 155 zcmeAS@N?(olHy`uVBq!ia0vp^0zfRm!3HEhFR0}KQc0dJjv*C{OAk5*vIYn+Z!oy* zJfCOH1L@fMe1%0Pub=<^K7OuM>AZfc(pNU}rLV43+P%86QrNU}))C{*SuVbd)B-10 u0PzaXD=PDVI7`U1GaZP*QXhRl()Ftigsj^3apVJSWAJqKb6Mw<&;$SnTsHs! literal 0 HcmV?d00001 diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-flip.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-flip.png new file mode 100644 index 0000000000000000000000000000000000000000..b68e6bdedfa3e4333280a90775568fbc3314611c GIT binary patch literal 190 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=ffJR!El<`~COdUs-v+_!X20p8x#HN+e>r-T9TlU?MO$F!-a7t1k#1H5N66!t)_l lz|aOLjjRMo5m?C;`?FgdY7WI^W&<6>;OXk;vd$@?2>?<1N>Km+ literal 0 HcmV?d00001 diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-p3-default.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-options-p3-default.png new file mode 100644 index 0000000000000000000000000000000000000000..691340c456c55a59d394b41edf94954af99c428b GIT binary patch literal 1116 zcmV-i1f%XPal6Cz~gmk2W7|{%LdZjW;J}2IDS7 zQ5b~$Px*f(e{`}j5ket^XlPxu z$k$HJEG$e6LQ#aG5C+lEU}0gW8?Ns1Npc8DzIJl<>WU>3gQN&WQ7J0IL^L$9V#Q83 zTyga(VSMf6?A4AP3loD-4Rpm{_rB#|7i0pb{!Yp(sKrnkXhFW;9q>SlBt)kh^?x z!G?vAi9skr5h_IyDurTDD$S_HU@%x1PR`!MZ+MR#3xg?xWFjd-r3gi(s1%Bc7SUiZ z7*5WS?TR9IA;igpL+Kj9IZ z+_1`yg^?*E$&_SDQX`|5yeQ-lLY$oaw{pm@c#oUxSY>3%V9H1`C7DJ>jVyKXNl^|V zPR>5dA%}>cfNxnajHtBj0{ zOc@!YM#eO9Qw)kwymJ>Po4=4le#E!&AUD}%msOUGj4_RoF{Y8-bQQuxk-NM)Ir}V! zyvZRS<+0pkmnBP9Su!#*jWM!JyF4pGm?(!hIlIdthmb?akNAL>xM9aGOIBGj#u(EW zOLo(@J5KqPz+D zLH;K1Lqv(NG-g#1x% z<)eHn55_JxV{a_WGM3$Vv0^Zo7=&W*O`M$F3Ku27~hIWb;`FIfM{G2zilz8IO%e<7&AXH_P5`xmrG2SeO_LCI;g!PBwQ@-h>cB z2w{-F%lGn9uEx!>TXwtMixmqC3loFE#5kOM<1U8~LI@!YLKw^V+4$agFm9Hc-R@?2 ivaql)SQrcj9GAIAEIO?^3x!IB h9;>`kIp%rgqyLjv*C{PY*WcN*MAm7@ApB z{;!Squ$MtE)W?BQw0&SuTx!~I%nGREjAFU=dzLC z>Eft)5b9W_; } From d480511e8a72a5bb3225cd56a86faabf23ac7984 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 24 Jul 2026 15:32:36 +0200 Subject: [PATCH 2/6] =?UTF-8?q?fix(=F0=9F=90=9B):=20honor=20premultiplyAlp?= =?UTF-8?q?ha=20in=20createImageBitmap=20and=20destination=20premultiplied?= =?UTF-8?q?Alpha?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createImageBitmap ignored the premultiplyAlpha option and copyExternalImageToTexture ignored the destination's premultipliedAlpha, so an ImageBitmap always came through premultiplied regardless of what was requested. This diverged from the web API and failed the new ImageBitmapAlpha suite on device. - Track the alpha representation on ImageData (premultiplied flag) and expose it on ImageBitmap. - Add a shared convertAlpha helper that matches the reference client's integer rounding, so native results stay bit-exact with dawn.node. - Parse premultiplyAlpha (from both the (source, options) and crop-rect overloads) and convert the decoded pixels to the requested representation before wrapping them in an ImageBitmap. - Convert to the destination's premultipliedAlpha (default false) in copyExternalImageToTexture, composing with the existing flipY. - Decode straight (non-premultiplied) RGBA on Apple via vImage so premultiplyAlpha "none" preserves the decoded bytes exactly; CGBitmapContext cannot target a non-premultiplied layout. Link Accelerate for vImage. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../android/cpp/AndroidPlatformContext.h | 3 ++ packages/webgpu/apple/ApplePlatformContext.mm | 47 +++++++++++++++---- packages/webgpu/cpp/rnwgpu/PlatformContext.h | 7 +++ packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp | 44 +++++++++++------ packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h | 32 +++++++++++++ packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h | 46 ++++++++++++++++-- packages/webgpu/react-native-webgpu.podspec | 4 +- 7 files changed, 155 insertions(+), 28 deletions(-) diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index b20078669f..d77d00282d 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -179,6 +179,9 @@ class AndroidPlatformContext : public PlatformContext { result.height = static_cast(bitmapInfo.height); result.data.resize(bitmapInfo.height * bitmapInfo.stride); memcpy(result.data.data(), bitmapPixels, result.data.size()); + // BitmapFactory hands back premultiplied ARGB_8888 pixels; createImageBitmap + // converts to the representation requested by premultiplyAlpha. + result.premultiplied = true; AndroidBitmap_unlockPixels(env, bitmap); diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index 594337cfcb..89045d6c5a 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -2,6 +2,7 @@ #include +#import #import #import #import @@ -117,7 +118,6 @@ void checkIfUsingSimulatorWithAPIValidation() { #endif size_t width = CGImageGetWidth(cgImage); size_t height = CGImageGetHeight(cgImage); - size_t bitsPerComponent = 8; size_t bytesPerRow = width * 4; ImageData result; @@ -125,17 +125,48 @@ void checkIfUsingSimulatorWithAPIValidation() { result.height = static_cast(height); result.data.resize(height * bytesPerRow); result.format = wgpu::TextureFormat::RGBA8Unorm; + // Straight (non-premultiplied) alpha: createImageBitmap converts to the + // representation requested by premultiplyAlpha, and premultiplyAlpha "none" + // must return the decoded bytes untouched. CGBitmapContext cannot target a + // non-premultiplied layout, so decode with vImage instead. Keeping the + // source's own color space avoids any color-managed conversion, so the RGBA + // samples match what the PNG stored. + CGColorSpaceRef sourceColorSpace = CGImageGetColorSpace(cgImage); + bool ownColorSpace = false; + if (sourceColorSpace == NULL) { + sourceColorSpace = CGColorSpaceCreateDeviceRGB(); + ownColorSpace = true; + } - CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); - CGContextRef context = CGBitmapContextCreate( - result.data.data(), width, height, bitsPerComponent, bytesPerRow, - colorSpace, kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); + vImage_CGImageFormat format = { + .bitsPerComponent = 8, + .bitsPerPixel = 32, + .colorSpace = sourceColorSpace, + .bitmapInfo = + static_cast(kCGImageAlphaLast | kCGBitmapByteOrder32Big), + .version = 0, + .decode = NULL, + .renderingIntent = kCGRenderingIntentDefault, + }; + vImage_Buffer buffer = { + .data = result.data.data(), + .height = static_cast(height), + .width = static_cast(width), + .rowBytes = bytesPerRow, + }; + vImage_Error err = vImageBuffer_InitWithCGImage( + &buffer, &format, NULL, cgImage, kvImageNoAllocate); - CGContextDrawImage(context, CGRectMake(0, 0, width, height), cgImage); + if (ownColorSpace) { + CGColorSpaceRelease(sourceColorSpace); + } - CGContextRelease(context); - CGColorSpaceRelease(colorSpace); + if (err != kvImageNoError) { + throw std::runtime_error("Couldn't decode image (vImage error " + + std::to_string(err) + ")"); + } + result.premultiplied = false; return result; } diff --git a/packages/webgpu/cpp/rnwgpu/PlatformContext.h b/packages/webgpu/cpp/rnwgpu/PlatformContext.h index fec049256b..3e3981ef44 100644 --- a/packages/webgpu/cpp/rnwgpu/PlatformContext.h +++ b/packages/webgpu/cpp/rnwgpu/PlatformContext.h @@ -16,6 +16,13 @@ struct ImageData { size_t width; size_t height; wgpu::TextureFormat format; + // Whether the RGB channels are premultiplied by alpha. Each platform decoder + // records the representation it produced; createImageBitmap then converts to + // the representation requested through premultiplyAlpha, and + // copyExternalImageToTexture converts again to match the destination's + // premultipliedAlpha. Defaults to true because most native decoders + // (CoreGraphics, Android Bitmap) hand back premultiplied pixels. + bool premultiplied = true; }; // Pixel layout of a VideoFrame. Determines whether the underlying surface is diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp index c0b3ff3a67..ee85d3fe63 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp @@ -128,26 +128,40 @@ void GPUQueue::copyExternalImageToTexture( throw std::runtime_error("Invalid input for GPUQueue::writeTexture()"); } - if (source->flipY.value_or(false)) { - // Calculate the row size and total size + const bool flipY = source->flipY.value_or(false); + // premultipliedAlpha defaults to false per the WebGPU spec: an untagged + // destination expects straight alpha. Convert only when the ImageBitmap's + // representation differs, using the same rounding as the reference client. + const bool sourcePremultiplied = source->source->getPremultiplied(); + const bool destinationPremultiplied = + destination->premultipliedAlpha.value_or(false); + const bool needsAlphaConversion = + bytesPerPixel == 4 && sourcePremultiplied != destinationPremultiplied; + + if (flipY || needsAlphaConversion) { uint32_t rowSize = bytesPerPixel * source->source->getWidth(); uint32_t totalSize = source->source->getSize(); - // Create a new buffer for the flipped data - std::vector flippedData(totalSize); - - // Flip the data vertically - for (uint32_t row = 0; row < source->source->getHeight(); ++row) { - std::memcpy(flippedData.data() + - (source->source->getHeight() - 1 - row) * rowSize, - static_cast(source->source->getData()) + - row * rowSize, - rowSize); + // Stage a mutable copy so flipping and/or alpha conversion never touch the + // ImageBitmap's backing store. + std::vector staged(totalSize); + const uint8_t *src = + static_cast(source->source->getData()); + if (flipY) { + for (uint32_t row = 0; row < source->source->getHeight(); ++row) { + std::memcpy(staged.data() + + (source->source->getHeight() - 1 - row) * rowSize, + src + row * rowSize, rowSize); + } + } else { + std::memcpy(staged.data(), src, totalSize); + } + if (needsAlphaConversion) { + convertAlpha(staged.data(), totalSize, sourcePremultiplied, + destinationPremultiplied); } - // Use the flipped data for writing to texture - _instance.WriteTexture(&dst, flippedData.data(), totalSize, &layout, &sz); + _instance.WriteTexture(&dst, staged.data(), totalSize, &layout, &sz); } else { - _instance.WriteTexture(&dst, source->source->getData(), source->source->getSize(), &layout, &sz); } diff --git a/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h b/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h index 3a76c35e07..5a04f559f8 100644 --- a/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h +++ b/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h @@ -11,6 +11,33 @@ namespace rnwgpu { namespace jsi = facebook::jsi; +// Convert RGBA8 pixel data in place between straight and premultiplied alpha. +// Uses the same integer rounding as the web reference polyfill in the test +// harness (setup.ts, convertAlpha) so the native result is bit-exact with the +// dawn.node client. A no-op when the source and destination representations +// already match. +inline void convertAlpha(uint8_t *data, size_t byteLength, + bool sourcePremultiplied, + bool destinationPremultiplied) { + if (sourcePremultiplied == destinationPremultiplied) { + return; + } + for (size_t i = 0; i + 3 < byteLength; i += 4) { + const uint32_t alpha = data[i + 3]; + for (size_t channel = 0; channel < 3; channel++) { + const uint32_t value = data[i + channel]; + if (destinationPremultiplied) { + data[i + channel] = static_cast((value * alpha + 127) / 255); + } else if (alpha == 0) { + data[i + channel] = 0; + } else { + const uint32_t straight = (value * 255 + (alpha >> 1)) / alpha; + data[i + channel] = static_cast(straight > 255 ? 255 : straight); + } + } + } +} + class ImageBitmap : public NativeObject { public: static constexpr const char *CLASS_NAME = "ImageBitmap"; @@ -26,6 +53,11 @@ class ImageBitmap : public NativeObject { size_t getSize() { return _imageData.data.size(); } + // Whether the stored pixels are premultiplied by alpha. Used by + // copyExternalImageToTexture to decide whether a conversion to the + // destination's premultipliedAlpha representation is needed. + bool getPremultiplied() { return _imageData.premultiplied; } + void close() { _imageData.data.clear(); _imageData.data.shrink_to_fit(); diff --git a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h index e7ecd8a7cc..9670a45092 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h +++ b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h @@ -90,6 +90,41 @@ class RNWebGPU : public NativeObject { auto platformContext = _platformContext; auto callInvoker = _callInvoker; + // Resolve the requested alpha representation from the ImageBitmapOptions. + // The options bag is the second argument for createImageBitmap(source, + // options) and the sixth for the crop-rect overload createImageBitmap( + // source, sx, sy, sw, sh, options). premultiplyAlpha defaults to + // "default", which (like "premultiply") stores premultiplied pixels; only + // "none" keeps straight alpha. The other options (crop rect, resize, + // imageOrientation, colorSpaceConversion) are not yet implemented natively. + bool wantPremultiplied = true; + const jsi::Value *optionsArg = nullptr; + if (count >= 2 && args[1].isObject()) { + optionsArg = &args[1]; + } else if (count >= 6 && args[5].isObject()) { + optionsArg = &args[5]; + } + if (optionsArg != nullptr) { + auto options = optionsArg->getObject(runtime); + if (options.hasProperty(runtime, "premultiplyAlpha")) { + auto value = options.getProperty(runtime, "premultiplyAlpha"); + if (value.isString() && + value.getString(runtime).utf8(runtime) == "none") { + wantPremultiplied = false; + } + } + } + + // Bring the decoded pixels into the representation requested via + // premultiplyAlpha before wrapping them in an ImageBitmap. + auto toRequestedAlpha = [wantPremultiplied](ImageData &imageData) { + if (imageData.premultiplied != wantPremultiplied) { + convertAlpha(imageData.data.data(), imageData.data.size(), + imageData.premultiplied, wantPremultiplied); + imageData.premultiplied = wantPremultiplied; + } + }; + // Check if the argument is an ArrayBuffer or ArrayBufferView // (TypedArray / DataView). Only a real buffer source is run through the // ArrayBuffer converter, which validates byteOffset/byteLength against the @@ -119,12 +154,14 @@ class RNWebGPU : public NativeObject { return Promise::createPromise( runtime, - [platformContext, callInvoker, dataCopy = std::move(dataCopy)]( + [platformContext, callInvoker, toRequestedAlpha, + dataCopy = std::move(dataCopy)]( jsi::Runtime & /*runtime*/, std::shared_ptr promise) mutable { platformContext->createImageBitmapFromDataAsync( dataCopy, - [callInvoker, promise](ImageData imageData) { + [callInvoker, promise, toRequestedAlpha](ImageData imageData) { + toRequestedAlpha(imageData); auto imageBitmap = std::make_shared(imageData); callInvoker->invokeAsync([promise, imageBitmap]() { promise->resolve( @@ -149,11 +186,12 @@ class RNWebGPU : public NativeObject { return Promise::createPromise( runtime, - [platformContext, callInvoker, blobId, offset, + [platformContext, callInvoker, toRequestedAlpha, blobId, offset, size](jsi::Runtime & /*runtime*/, std::shared_ptr promise) { platformContext->createImageBitmapAsync( blobId, offset, size, - [callInvoker, promise](ImageData imageData) { + [callInvoker, promise, toRequestedAlpha](ImageData imageData) { + toRequestedAlpha(imageData); auto imageBitmap = std::make_shared(imageData); callInvoker->invokeAsync([promise, imageBitmap]() { promise->resolve( diff --git a/packages/webgpu/react-native-webgpu.podspec b/packages/webgpu/react-native-webgpu.podspec index a6b12cf442..c7bf8be0a9 100644 --- a/packages/webgpu/react-native-webgpu.podspec +++ b/packages/webgpu/react-native-webgpu.podspec @@ -23,7 +23,9 @@ Pod::Spec.new do |s| # The VideoPlayer API uses AVFoundation / CoreMedia, and shared-texture # surfaces use CoreVideo (CVPixelBuffer). Link them so their symbols resolve. - s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo" + # Accelerate provides vImage, used to decode images to straight (non- + # premultiplied) RGBA for createImageBitmap. + s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo", "Accelerate" s.pod_target_xcconfig = { 'HEADER_SEARCH_PATHS' => '$(PODS_TARGET_SRCROOT)/cpp', From a5d2665def1dec543db713f039410553f93bc4b0 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 24 Jul 2026 16:35:25 +0200 Subject: [PATCH 3/6] :wrench: --- apps/example/ios/Podfile.lock | 2 +- packages/webgpu/apple/ApplePlatformContext.mm | 231 +++++++++++++----- packages/webgpu/react-native-webgpu.podspec | 10 +- 3 files changed, 178 insertions(+), 65 deletions(-) diff --git a/apps/example/ios/Podfile.lock b/apps/example/ios/Podfile.lock index 23802bd713..9b93bfb63f 100644 --- a/apps/example/ios/Podfile.lock +++ b/apps/example/ios/Podfile.lock @@ -3074,7 +3074,7 @@ SPEC CHECKSUMS: React-microtasksnativemodule: 75b6604b667d297292345302cc5bfb6b6aeccc1b react-native-safe-area-context: c00143b4823773bba23f2f19f85663ae89ceb460 react-native-skia: fc73e9bdc46ebb420a98c9c2be29fee80f565e79 - react-native-webgpu: 3eb051bebe030f328725fc318a147599c4628e4e + react-native-webgpu: cc416064f9c8a68c6fde764a6c36305676f5fd65 React-NativeModulesApple: 879fbdc5dcff7136abceb7880fe8a2022a1bd7c3 React-oscompat: 93b5535ea7f7dff46aaee4f78309a70979bdde9d React-perflogger: 5536d2df3d18fe0920263466f7b46a56351c0510 diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index 89045d6c5a..f0fa7fe52e 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -2,9 +2,14 @@ #include -#import +#include +#include + +#include + #import #import +#import #import #import #import @@ -87,35 +92,167 @@ void checkIfUsingSimulatorWithAPIValidation() { std::move(onError)); } +// Decode an 8-bit, non-interlaced, truecolor (with or without alpha) PNG into +// straight (non-premultiplied) RGBA. Returns false for any other PNG variant +// or a non-PNG input, leaving `out` untouched so the caller can fall back to +// ImageIO. +// +// Apple's imaging stack (UIImage, CGImageSource, CGBitmapContext, Core Image) +// always premultiplies alpha at decode and quantizes to 8 bits, which destroys +// the original straight samples for low-alpha pixels. createImageBitmap's +// premultiplyAlpha "none" must return those samples intact, so PNGs (the only +// format that carries alpha here) are decoded directly instead. +static bool decodeStraightPng(const uint8_t *data, size_t size, + ImageData &out) { + static const uint8_t kSignature[8] = {137, 80, 78, 71, 13, 10, 26, 10}; + if (size < 8 || memcmp(data, kSignature, 8) != 0) { + return false; + } + + auto readBE32 = [](const uint8_t *p) -> uint32_t { + return (static_cast(p[0]) << 24) | + (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | static_cast(p[3]); + }; + + uint32_t width = 0; + uint32_t height = 0; + uint8_t bitDepth = 0; + uint8_t colorType = 0; + uint8_t interlace = 0; + bool haveHeader = false; + std::vector idat; + + size_t pos = 8; + while (pos + 12 <= size) { + uint32_t chunkLen = readBE32(data + pos); + const uint8_t *type = data + pos + 4; + const uint8_t *chunk = data + pos + 8; + // Guard against a truncated / malformed chunk running past the buffer. + if (chunkLen > size - pos - 12) { + return false; + } + if (memcmp(type, "IHDR", 4) == 0 && chunkLen >= 13) { + width = readBE32(chunk); + height = readBE32(chunk + 4); + bitDepth = chunk[8]; + colorType = chunk[9]; + interlace = chunk[12]; + haveHeader = true; + } else if (memcmp(type, "IDAT", 4) == 0) { + idat.insert(idat.end(), chunk, chunk + chunkLen); + } else if (memcmp(type, "IEND", 4) == 0) { + break; + } + pos += 12 + chunkLen; + } + + int channels = 0; + if (colorType == 2) { + channels = 3; // truecolor RGB + } else if (colorType == 6) { + channels = 4; // truecolor RGBA + } + if (!haveHeader || bitDepth != 8 || interlace != 0 || channels == 0 || + width == 0 || height == 0 || idat.empty()) { + return false; + } + + const size_t rowBytes = static_cast(width) * channels; + const uLongf inflatedSize = (rowBytes + 1) * height; // +1 filter byte per row + std::vector inflated(inflatedSize); + uLongf actualSize = inflatedSize; + if (uncompress(inflated.data(), &actualSize, idat.data(), + static_cast(idat.size())) != Z_OK || + actualSize != inflatedSize) { + return false; + } + + // Reverse the per-scanline PNG filters in place. + std::vector image(rowBytes * height); + for (uint32_t y = 0; y < height; y++) { + const uint8_t filter = inflated[y * (rowBytes + 1)]; + const uint8_t *src = &inflated[y * (rowBytes + 1) + 1]; + uint8_t *row = &image[y * rowBytes]; + const uint8_t *prev = y > 0 ? &image[(y - 1) * rowBytes] : nullptr; + for (size_t i = 0; i < rowBytes; i++) { + const int a = i >= static_cast(channels) ? row[i - channels] : 0; + const int b = prev ? prev[i] : 0; + const int c = + (prev && i >= static_cast(channels)) ? prev[i - channels] : 0; + int value = src[i]; + switch (filter) { + case 0: // None + break; + case 1: // Sub + value += a; + break; + case 2: // Up + value += b; + break; + case 3: // Average + value += (a + b) / 2; + break; + case 4: { // Paeth + const int p = a + b - c; + const int pa = std::abs(p - a); + const int pb = std::abs(p - b); + const int pc = std::abs(p - c); + value += (pa <= pb && pa <= pc) ? a : (pb <= pc ? b : c); + break; + } + default: + return false; + } + row[i] = static_cast(value & 0xFF); + } + } + + out.width = width; + out.height = height; + out.format = wgpu::TextureFormat::RGBA8Unorm; + out.premultiplied = false; + out.data.resize(static_cast(width) * height * 4); + const size_t pixelCount = static_cast(width) * height; + for (size_t p = 0; p < pixelCount; p++) { + const uint8_t *s = &image[p * channels]; + uint8_t *d = &out.data[p * 4]; + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = channels == 4 ? s[3] : 255; + } + return true; +} + ImageData ApplePlatformContext::createImageBitmapFromData(std::span data) { - // This avoids a copy by assuming the UIImage/NSImage constructors - // decode `nsData` eagerly before the memory for the wrapped `data` - // is freed. - // - // Since we get the `CGImageRef` from `image` and then throw - // it away, that's a fairly safe assumption. + // PNGs carry alpha, and premultiplyAlpha "none" must preserve their straight + // samples exactly, so decode them directly (Apple's imaging APIs premultiply + // at decode). Everything else goes through ImageIO below. + ImageData pngResult; + if (decodeStraightPng(data.data(), data.size(), pngResult)) { + return pngResult; + } + NSData *nsData = [NSData dataWithBytesNoCopy:const_cast(data.data()) length:data.size() freeWhenDone:NO]; -#if !TARGET_OS_OSX - UIImage *image = [UIImage imageWithData:nsData]; -#else - NSImage *image = [[NSImage alloc] initWithData:nsData]; -#endif - if (!image) { + CGImageSourceRef imageSource = + CGImageSourceCreateWithData((__bridge CFDataRef)nsData, NULL); + if (imageSource == NULL) { + throw std::runtime_error("Couldn't create image source"); + } + NSDictionary *decodeOptions = @{(id)kCGImageSourceShouldCache : @NO}; + CGImageRef cgImage = CGImageSourceCreateImageAtIndex( + imageSource, 0, (__bridge CFDictionaryRef)decodeOptions); + CFRelease(imageSource); + if (cgImage == NULL) { throw std::runtime_error("Couldn't decode image"); } -#if !TARGET_OS_OSX - CGImageRef cgImage = image.CGImage; -#else - CGImageRef cgImage = [image CGImageForProposedRect:NULL - context:NULL - hints:NULL]; -#endif size_t width = CGImageGetWidth(cgImage); size_t height = CGImageGetHeight(cgImage); size_t bytesPerRow = width * 4; @@ -125,48 +262,20 @@ void checkIfUsingSimulatorWithAPIValidation() { result.height = static_cast(height); result.data.resize(height * bytesPerRow); result.format = wgpu::TextureFormat::RGBA8Unorm; - // Straight (non-premultiplied) alpha: createImageBitmap converts to the - // representation requested by premultiplyAlpha, and premultiplyAlpha "none" - // must return the decoded bytes untouched. CGBitmapContext cannot target a - // non-premultiplied layout, so decode with vImage instead. Keeping the - // source's own color space avoids any color-managed conversion, so the RGBA - // samples match what the PNG stored. - CGColorSpaceRef sourceColorSpace = CGImageGetColorSpace(cgImage); - bool ownColorSpace = false; - if (sourceColorSpace == NULL) { - sourceColorSpace = CGColorSpaceCreateDeviceRGB(); - ownColorSpace = true; - } - - vImage_CGImageFormat format = { - .bitsPerComponent = 8, - .bitsPerPixel = 32, - .colorSpace = sourceColorSpace, - .bitmapInfo = - static_cast(kCGImageAlphaLast | kCGBitmapByteOrder32Big), - .version = 0, - .decode = NULL, - .renderingIntent = kCGRenderingIntentDefault, - }; - vImage_Buffer buffer = { - .data = result.data.data(), - .height = static_cast(height), - .width = static_cast(width), - .rowBytes = bytesPerRow, - }; - vImage_Error err = vImageBuffer_InitWithCGImage( - &buffer, &format, NULL, cgImage, kvImageNoAllocate); - - if (ownColorSpace) { - CGColorSpaceRelease(sourceColorSpace); - } - - if (err != kvImageNoError) { - throw std::runtime_error("Couldn't decode image (vImage error " + - std::to_string(err) + ")"); - } - result.premultiplied = false; + // Non-PNG sources (JPEG, ...) have no alpha channel, so the premultiplied + // draw is exact for them; flag the result premultiplied so createImageBitmap + // and copyExternalImageToTexture convert consistently. + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate( + result.data.data(), width, height, 8, bytesPerRow, colorSpace, + kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); + CGContextDrawImage(context, CGRectMake(0, 0, width, height), cgImage); + CGContextRelease(context); + CGColorSpaceRelease(colorSpace); + CGImageRelease(cgImage); + + result.premultiplied = true; return result; } diff --git a/packages/webgpu/react-native-webgpu.podspec b/packages/webgpu/react-native-webgpu.podspec index c7bf8be0a9..5902faaee6 100644 --- a/packages/webgpu/react-native-webgpu.podspec +++ b/packages/webgpu/react-native-webgpu.podspec @@ -23,9 +23,13 @@ Pod::Spec.new do |s| # The VideoPlayer API uses AVFoundation / CoreMedia, and shared-texture # surfaces use CoreVideo (CVPixelBuffer). Link them so their symbols resolve. - # Accelerate provides vImage, used to decode images to straight (non- - # premultiplied) RGBA for createImageBitmap. - s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo", "Accelerate" + # ImageIO provides CGImageSource, the createImageBitmap fallback decoder for + # non-PNG formats. + s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo", "ImageIO" + + # zlib inflates PNG IDAT streams in the straight-alpha PNG decoder used by + # createImageBitmap. + s.libraries = "z" s.pod_target_xcconfig = { 'HEADER_SEARCH_PATHS' => '$(PODS_TARGET_SRCROOT)/cpp', From dbd2583631ffd8f793b97fb069cd616162ed04bf Mon Sep 17 00:00:00 2001 From: William Candillon Date: Fri, 24 Jul 2026 16:49:56 +0200 Subject: [PATCH 4/6] :wrench: --- packages/webgpu/apple/ApplePlatformContext.mm | 157 +----------------- packages/webgpu/react-native-webgpu.podspec | 7 +- .../src/__tests__/ImageBitmapAlpha.spec.ts | 18 +- 3 files changed, 18 insertions(+), 164 deletions(-) diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index f0fa7fe52e..6f953bbfc1 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -2,11 +2,6 @@ #include -#include -#include - -#include - #import #import #import @@ -92,149 +87,14 @@ void checkIfUsingSimulatorWithAPIValidation() { std::move(onError)); } -// Decode an 8-bit, non-interlaced, truecolor (with or without alpha) PNG into -// straight (non-premultiplied) RGBA. Returns false for any other PNG variant -// or a non-PNG input, leaving `out` untouched so the caller can fall back to -// ImageIO. -// -// Apple's imaging stack (UIImage, CGImageSource, CGBitmapContext, Core Image) -// always premultiplies alpha at decode and quantizes to 8 bits, which destroys -// the original straight samples for low-alpha pixels. createImageBitmap's -// premultiplyAlpha "none" must return those samples intact, so PNGs (the only -// format that carries alpha here) are decoded directly instead. -static bool decodeStraightPng(const uint8_t *data, size_t size, - ImageData &out) { - static const uint8_t kSignature[8] = {137, 80, 78, 71, 13, 10, 26, 10}; - if (size < 8 || memcmp(data, kSignature, 8) != 0) { - return false; - } - - auto readBE32 = [](const uint8_t *p) -> uint32_t { - return (static_cast(p[0]) << 24) | - (static_cast(p[1]) << 16) | - (static_cast(p[2]) << 8) | static_cast(p[3]); - }; - - uint32_t width = 0; - uint32_t height = 0; - uint8_t bitDepth = 0; - uint8_t colorType = 0; - uint8_t interlace = 0; - bool haveHeader = false; - std::vector idat; - - size_t pos = 8; - while (pos + 12 <= size) { - uint32_t chunkLen = readBE32(data + pos); - const uint8_t *type = data + pos + 4; - const uint8_t *chunk = data + pos + 8; - // Guard against a truncated / malformed chunk running past the buffer. - if (chunkLen > size - pos - 12) { - return false; - } - if (memcmp(type, "IHDR", 4) == 0 && chunkLen >= 13) { - width = readBE32(chunk); - height = readBE32(chunk + 4); - bitDepth = chunk[8]; - colorType = chunk[9]; - interlace = chunk[12]; - haveHeader = true; - } else if (memcmp(type, "IDAT", 4) == 0) { - idat.insert(idat.end(), chunk, chunk + chunkLen); - } else if (memcmp(type, "IEND", 4) == 0) { - break; - } - pos += 12 + chunkLen; - } - - int channels = 0; - if (colorType == 2) { - channels = 3; // truecolor RGB - } else if (colorType == 6) { - channels = 4; // truecolor RGBA - } - if (!haveHeader || bitDepth != 8 || interlace != 0 || channels == 0 || - width == 0 || height == 0 || idat.empty()) { - return false; - } - - const size_t rowBytes = static_cast(width) * channels; - const uLongf inflatedSize = (rowBytes + 1) * height; // +1 filter byte per row - std::vector inflated(inflatedSize); - uLongf actualSize = inflatedSize; - if (uncompress(inflated.data(), &actualSize, idat.data(), - static_cast(idat.size())) != Z_OK || - actualSize != inflatedSize) { - return false; - } - - // Reverse the per-scanline PNG filters in place. - std::vector image(rowBytes * height); - for (uint32_t y = 0; y < height; y++) { - const uint8_t filter = inflated[y * (rowBytes + 1)]; - const uint8_t *src = &inflated[y * (rowBytes + 1) + 1]; - uint8_t *row = &image[y * rowBytes]; - const uint8_t *prev = y > 0 ? &image[(y - 1) * rowBytes] : nullptr; - for (size_t i = 0; i < rowBytes; i++) { - const int a = i >= static_cast(channels) ? row[i - channels] : 0; - const int b = prev ? prev[i] : 0; - const int c = - (prev && i >= static_cast(channels)) ? prev[i - channels] : 0; - int value = src[i]; - switch (filter) { - case 0: // None - break; - case 1: // Sub - value += a; - break; - case 2: // Up - value += b; - break; - case 3: // Average - value += (a + b) / 2; - break; - case 4: { // Paeth - const int p = a + b - c; - const int pa = std::abs(p - a); - const int pb = std::abs(p - b); - const int pc = std::abs(p - c); - value += (pa <= pb && pa <= pc) ? a : (pb <= pc ? b : c); - break; - } - default: - return false; - } - row[i] = static_cast(value & 0xFF); - } - } - - out.width = width; - out.height = height; - out.format = wgpu::TextureFormat::RGBA8Unorm; - out.premultiplied = false; - out.data.resize(static_cast(width) * height * 4); - const size_t pixelCount = static_cast(width) * height; - for (size_t p = 0; p < pixelCount; p++) { - const uint8_t *s = &image[p * channels]; - uint8_t *d = &out.data[p * 4]; - d[0] = s[0]; - d[1] = s[1]; - d[2] = s[2]; - d[3] = channels == 4 ? s[3] : 255; - } - return true; -} - ImageData ApplePlatformContext::createImageBitmapFromData(std::span data) { - // PNGs carry alpha, and premultiplyAlpha "none" must preserve their straight - // samples exactly, so decode them directly (Apple's imaging APIs premultiply - // at decode). Everything else goes through ImageIO below. - ImageData pngResult; - if (decodeStraightPng(data.data(), data.size(), pngResult)) { - return pngResult; - } - + // All formats are decoded through ImageIO. Apple's imaging stack always + // premultiplies alpha at decode, so the result is flagged premultiplied and + // createImageBitmap / copyExternalImageToTexture convert from there. This + // means premultiplyAlpha "none" is a lossy round trip for low-alpha pixels + // (as it is on Android); the snapshot suites compare with pixelmatch + // tolerance to absorb it. NSData *nsData = [NSData dataWithBytesNoCopy:const_cast(data.data()) length:data.size() @@ -263,9 +123,8 @@ static bool decodeStraightPng(const uint8_t *data, size_t size, result.data.resize(height * bytesPerRow); result.format = wgpu::TextureFormat::RGBA8Unorm; - // Non-PNG sources (JPEG, ...) have no alpha channel, so the premultiplied - // draw is exact for them; flag the result premultiplied so createImageBitmap - // and copyExternalImageToTexture convert consistently. + // The draw premultiplies alpha; flag the result premultiplied so + // createImageBitmap and copyExternalImageToTexture convert consistently. CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); CGContextRef context = CGBitmapContextCreate( result.data.data(), width, height, 8, bytesPerRow, colorSpace, diff --git a/packages/webgpu/react-native-webgpu.podspec b/packages/webgpu/react-native-webgpu.podspec index 5902faaee6..69e169e825 100644 --- a/packages/webgpu/react-native-webgpu.podspec +++ b/packages/webgpu/react-native-webgpu.podspec @@ -23,14 +23,9 @@ Pod::Spec.new do |s| # The VideoPlayer API uses AVFoundation / CoreMedia, and shared-texture # surfaces use CoreVideo (CVPixelBuffer). Link them so their symbols resolve. - # ImageIO provides CGImageSource, the createImageBitmap fallback decoder for - # non-PNG formats. + # ImageIO provides CGImageSource, the image decoder behind createImageBitmap. s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo", "ImageIO" - # zlib inflates PNG IDAT streams in the straight-alpha PNG decoder used by - # createImageBitmap. - s.libraries = "z" - s.pod_target_xcconfig = { 'HEADER_SEARCH_PATHS' => '$(PODS_TARGET_SRCROOT)/cpp', } diff --git a/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts index 1767ed1a7c..8d7acc7d71 100644 --- a/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts +++ b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts @@ -10,7 +10,8 @@ import { checkImage, client, encodeImage } from "./setup"; // vertical flips both produce large snapshot diffs. The baselines are computed // with the same integer arithmetic as ImageBitmap::convertAlpha; pixelmatch // tolerance absorbs the platform decoders' off-by-one rounding differences -// (e.g. Android decodes premultiplied, so "none" is a lossy round trip there). +// (iOS and Android both decode premultiplied, so "none" is a lossy round trip +// there). const assetPath = path.resolve(__dirname, "./assets/alpha-gradient.png"); const pngBase64 = fs.readFileSync(assetPath).toString("base64"); @@ -247,13 +248,12 @@ describe("ImageBitmap alpha representation", () => { }, ); - // Exact-value check for the lossless straight-alpha guarantee this feature - // adds on Apple: premultiplyAlpha "none" must preserve the decoded bytes - // bit-for-bit, and conversions must use round-to-nearest. Runs where - // exactness is guaranteed: iOS (Core Image decode) and the node client - // (whose polyfill mirrors the C++ integer math). Android decodes - // premultiplied, so "none" is lossy there; the reference browser's rounding - // may legitimately differ by one. Both are covered by the snapshot matrix. + // Exact-value check that conversions use round-to-nearest integer math. + // Runs only on the node client, whose polyfill mirrors the C++ convertAlpha + // arithmetic bit-for-bit. iOS and Android both decode premultiplied through + // the platform imaging stack, so premultiplyAlpha "none" is a lossy round + // trip on device; the reference browser's rounding may also differ by one. + // Those platforms are covered by the tolerance-based snapshot matrix above. const straightRows = [ [128, 128, 128, 128], [17, 34, 51, 64], @@ -272,7 +272,7 @@ describe("ImageBitmap alpha representation", () => { ] as const)( "preserves exact bytes for source=$sourceAlpha destination=$destinationAlpha", async ({ sourceAlpha, destinationAlpha, expected }) => { - if (client.OS !== "ios" && client.OS !== "node") { + if (client.OS !== "node") { return; } const png = new PNG({ width: 1, height: 2 }); From 6fdb2d2114ebdc94c4b4ffaa4023f355cfbd6180 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Sat, 25 Jul 2026 08:30:13 +0200 Subject: [PATCH 5/6] :wrench: --- .../src/__tests__/ImageBitmapAlpha.spec.ts | 11 ++-- .../src/__tests__/ImageBitmapOptions.spec.ts | 28 ++++---- packages/webgpu/src/__tests__/config.ts | 3 + packages/webgpu/src/__tests__/globalSetup.ts | 64 +++++++++++++++++-- .../webgpu/src/__tests__/globalTeardown.ts | 3 + packages/webgpu/src/__tests__/setup.ts | 23 ++++++- 6 files changed, 105 insertions(+), 27 deletions(-) diff --git a/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts index 8d7acc7d71..4164db6c72 100644 --- a/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts +++ b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts @@ -3,7 +3,7 @@ import path from "path"; import { PNG } from "pngjs"; -import { checkImage, client, encodeImage } from "./setup"; +import { checkImage, client, encodeImage, fixtureUrl } from "./setup"; // A 32x32 gradient with a vertical alpha ramp (row 0 fully transparent, last // row fully opaque) and a horizontal color ramp, so alpha-mode mistakes and @@ -13,7 +13,6 @@ import { checkImage, client, encodeImage } from "./setup"; // (iOS and Android both decode premultiplied, so "none" is a lossy round trip // there). const assetPath = path.resolve(__dirname, "./assets/alpha-gradient.png"); -const pngBase64 = fs.readFileSync(assetPath).toString("base64"); type SourceAlpha = "none" | "premultiply" | "default" | "omitted"; type DestinationAlpha = boolean | "omitted"; @@ -80,12 +79,12 @@ describe("ImageBitmap alpha representation", () => { const result = await client.eval( ({ device, - pngBase64: encodedPng, + url, sourceAlpha: sourceRepresentation, destinationAlpha: destinationRepresentation, flipY: shouldFlip, }) => { - return fetch(`data:image/png;base64,${encodedPng}`) + return fetch(url) .then((response) => response.blob()) .then((blob) => createImageBitmap( @@ -144,7 +143,7 @@ describe("ImageBitmap alpha representation", () => { }); }, { - pngBase64, + url: fixtureUrl("alpha-gradient.png"), sourceAlpha, destinationAlpha, flipY, @@ -254,6 +253,8 @@ describe("ImageBitmap alpha representation", () => { // the platform imaging stack, so premultiplyAlpha "none" is a lossy round // trip on device; the reference browser's rounding may also differ by one. // Those platforms are covered by the tolerance-based snapshot matrix above. + // Being node-only, this case can build its fixture at runtime and pass it as + // a data: URI rather than going through fixtureUrl. const straightRows = [ [128, 128, 128, 128], [17, 34, 51, 64], diff --git a/packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts b/packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts index 16256fab3d..e522ee5955 100644 --- a/packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts +++ b/packages/webgpu/src/__tests__/ImageBitmapOptions.spec.ts @@ -1,7 +1,4 @@ -import fs from "fs"; -import path from "path"; - -import { checkImage, client, encodeImage } from "./setup"; +import { checkImage, client, encodeImage, fixtureUrl } from "./setup"; // createImageBitmap options beyond premultiplyAlpha (which is covered by // ImageBitmapAlpha.spec.ts): the crop-rect overload, resizeWidth/resizeHeight, @@ -17,11 +14,6 @@ import { checkImage, client, encodeImage } from "./setup"; // and the full, upright, unresized image comes through. When one of these // options gets implemented natively, the corresponding case fails here and // should be flipped over to the reference snapshot. -const assetPath = path.resolve(__dirname, "./assets/opaque-gradient.png"); -const pngBase64 = fs.readFileSync(assetPath).toString("base64"); -const p3AssetPath = path.resolve(__dirname, "./assets/p3-gradient.png"); -const p3Base64 = fs.readFileSync(p3AssetPath).toString("base64"); - const identitySnapshot = "assets/opaque-gradient.png"; interface OptionsCase { @@ -67,13 +59,13 @@ const cases: OptionsCase[] = [ ]; const runCase = ( - encodedPng: string, + fileName: string, cropRect: [number, number, number, number] | null, options: ImageBitmapOptions | null, ) => client.eval( - ({ device, pngBase64: png, cropRect: rect, options: bitmapOptions }) => { - return fetch(`data:image/png;base64,${png}`) + ({ device, url, cropRect: rect, options: bitmapOptions }) => { + return fetch(url) .then((response) => response.blob()) .then((blob) => rect === null @@ -133,7 +125,7 @@ const runCase = ( }); }, { - pngBase64: encodedPng, + url: fixtureUrl(fileName), cropRect, options, }, @@ -141,7 +133,11 @@ const runCase = ( describe("createImageBitmap options", () => { it.each(cases)("$name", async ({ cropRect, options, referenceSnapshot }) => { - const result = await runCase(pngBase64, cropRect ?? null, options ?? null); + const result = await runCase( + "opaque-gradient.png", + cropRect ?? null, + options ?? null, + ); const isReference = client.OS === "web" || client.OS === "node"; checkImage( encodeImage(result), @@ -162,7 +158,7 @@ describe("createImageBitmap options", () => { if (client.OS !== "web" && client.OS !== "node") { return; } - const result = await runCase(p3Base64, null, { + const result = await runCase("p3-gradient.png", null, { colorSpaceConversion: "none", }); checkImage(encodeImage(result), "assets/p3-gradient.png"); @@ -172,7 +168,7 @@ describe("createImageBitmap options", () => { if (client.OS !== "web") { return; } - const result = await runCase(p3Base64, null, { + const result = await runCase("p3-gradient.png", null, { colorSpaceConversion: "default", }); checkImage( diff --git a/packages/webgpu/src/__tests__/config.ts b/packages/webgpu/src/__tests__/config.ts index 88be309f61..262e7f9696 100644 --- a/packages/webgpu/src/__tests__/config.ts +++ b/packages/webgpu/src/__tests__/config.ts @@ -1,3 +1,6 @@ export const DEBUG = process.env.DEBUG === "true"; export const REFERENCE = process.env.REFERENCE === "true"; export const NODE_WEBGPU = process.env.NODE_WEBGPU === "true"; +// Serves both the WebSocket endpoint the example app connects to and the static +// fixtures under ./assets. Kept in sync with PORT in apps/example/src/useClient. +export const TEST_SERVER_PORT = 4242; diff --git a/packages/webgpu/src/__tests__/globalSetup.ts b/packages/webgpu/src/__tests__/globalSetup.ts index 81bcb8cff6..a9bdc9c342 100644 --- a/packages/webgpu/src/__tests__/globalSetup.ts +++ b/packages/webgpu/src/__tests__/globalSetup.ts @@ -1,6 +1,11 @@ +import fs from "fs"; +import http from "http"; +import path from "path"; +import { execFileSync } from "child_process"; + import { WebSocketServer } from "ws"; -import { NODE_WEBGPU, REFERENCE } from "./config"; +import { NODE_WEBGPU, REFERENCE, TEST_SERVER_PORT } from "./config"; const isOS = (os: string): os is "android" | "ios" | "web" => { return ["ios", "android", "web"].indexOf(os) !== -1; @@ -10,6 +15,47 @@ const isArch = (arc: string): arc is "paper" | "fabric" => { return ["paper", "fabric"].indexOf(arc) !== -1; }; +// Static file server for the fixtures in ./assets, sharing the port with the +// WebSocket endpoint. Device tests that need a Blob load their PNGs from here: +// React Native's Android networking stack (OkHttp) cannot fetch data: URIs, so +// `fetch("data:image/png;base64,...")` rejects with "Network request failed" +// there even though it works on iOS (NSURLSession handles data: natively). +const assetsDir = path.resolve(__dirname, "assets"); +const createFixtureServer = () => + http.createServer((req, res) => { + const name = path.basename( + decodeURIComponent((req.url ?? "").split("?")[0]), + ); + const file = path.join(assetsDir, name); + if (!file.startsWith(assetsDir + path.sep) || !fs.existsSync(file)) { + res.writeHead(404).end(); + return; + } + res.writeHead(200, { + "Content-Type": "image/png", + "Cache-Control": "no-store", + "Access-Control-Allow-Origin": "*", + }); + res.end(fs.readFileSync(file)); + }); + +// Map the device's own localhost:4242 onto this machine, the same way the React +// Native CLI does for Metro on 8081. With it, an Android emulator and a physical +// device both reach the test server (WebSocket and fixtures alike) at +// "localhost", so useClient needs no per-platform host and no LAN address. +// Best effort: no adb, no device, or several devices attached just leaves the +// connection to whatever host the app is configured with. +const reversePort = (port: number) => { + try { + execFileSync("adb", ["reverse", `tcp:${port}`, `tcp:${port}`], { + stdio: "ignore", + }); + } catch { + // Not an Android run (or adb is unavailable); iOS and the emulator default + // work without it. + } +}; + const globalSetup = () => { return new Promise((resolve) => { // The reference (Chrome) and node (dawn.node) clients run in-process, so @@ -18,8 +64,13 @@ const globalSetup = () => { resolve(); return; } - const port = 4242; - global.testServer = new WebSocketServer({ port }); + const port = TEST_SERVER_PORT; + reversePort(port); + global.testFixtureServer = createFixtureServer(); + global.testServer = new WebSocketServer({ + server: global.testFixtureServer, + }); + global.testFixtureServer.listen(port); console.log( `\n\nTest server listening on port ${port} (waiting for the example app to open on E2E tests screen)`, ); @@ -27,7 +78,7 @@ const globalSetup = () => { global.testClient = client; client.once("message", (msg) => { const obj = JSON.parse(msg.toString("utf8")); - const { OS, arch } = obj; + const { OS, arch, host } = obj; if (!isOS(OS)) { throw new Error("Unknown testing platform: " + OS); } @@ -36,7 +87,10 @@ const globalSetup = () => { } global.testOS = OS; global.testArch = arch; - console.log(`${OS} device connected (${arch})`); + // The host the device reached us on (localhost for the simulator, a LAN + // address for a physical device); fixture URLs are built from it. + global.testHost = typeof host === "string" ? host : "localhost"; + console.log(`${OS} device connected (${arch}) from ${global.testHost}`); resolve(); }); }); diff --git a/packages/webgpu/src/__tests__/globalTeardown.ts b/packages/webgpu/src/__tests__/globalTeardown.ts index ec3ee6a1e4..d02a3f9f4c 100644 --- a/packages/webgpu/src/__tests__/globalTeardown.ts +++ b/packages/webgpu/src/__tests__/globalTeardown.ts @@ -5,6 +5,9 @@ const globalTeardown = () => { if (global.testServer) { global.testServer.close(); } + if (global.testFixtureServer) { + global.testFixtureServer.close(); + } }; // eslint-disable-next-line import/no-default-export diff --git a/packages/webgpu/src/__tests__/setup.ts b/packages/webgpu/src/__tests__/setup.ts index 3501ec7cf8..34abf396a9 100644 --- a/packages/webgpu/src/__tests__/setup.ts +++ b/packages/webgpu/src/__tests__/setup.ts @@ -2,6 +2,7 @@ import fs from "fs"; import path from "path"; +import type { Server as HTTPServer } from "http"; import puppeteer from "puppeteer"; import { PNG } from "pngjs"; @@ -14,7 +15,7 @@ import type { GPUOffscreenCanvas } from "../Offscreen"; import { cubeVertexArray } from "./components/cube"; import { redFragWGSL, triangleVertWGSL } from "./components/triangle"; -import { DEBUG, NODE_WEBGPU, REFERENCE } from "./config"; +import { DEBUG, NODE_WEBGPU, REFERENCE, TEST_SERVER_PORT } from "./config"; jest.setTimeout(180 * 1000); @@ -22,9 +23,11 @@ type TestOS = "ios" | "android" | "web" | "node"; declare global { var testServer: Server; + var testFixtureServer: HTTPServer; var testClient: WebSocket; var testOS: TestOS; var testArch: "paper" | "fabric"; + var testHost: string; } interface GPUTestingContext { @@ -857,6 +860,24 @@ export const itSkipsOnWeb = (name: string, fn: () => Promise) => { }); }; +// URL a test can `fetch(...).then(r => r.blob())` on the device to get one of +// the fixtures in ./assets. +// +// React Native's Android networking stack (OkHttp) has no handler for the data: +// scheme, so an inline `data:image/png;base64,...` URL rejects there with +// "Network request failed" (iOS works, since NSURLSession decodes data: URLs +// itself). Device clients therefore load fixtures over HTTP from the test +// server started in globalSetup. Chrome and node both support data: URLs and +// keep the inline form, which leaves the reference clients independent of that +// server. +export const fixtureUrl = (fileName: string): string => { + if (client.OS === "ios" || client.OS === "android") { + return `http://${global.testHost}:${TEST_SERVER_PORT}/${fileName}`; + } + const p = path.resolve(__dirname, "assets", fileName); + return `data:image/png;base64,${fs.readFileSync(p).toString("base64")}`; +}; + export const decodeImage = (relPath: string): BitmapData => { const p = path.resolve(__dirname, relPath); const data = fs.readFileSync(p); From 72ecc5aaa42e08da85d8d80dcfe6023d438ffbe3 Mon Sep 17 00:00:00 2001 From: William Candillon Date: Sat, 25 Jul 2026 08:38:56 +0200 Subject: [PATCH 6/6] :wrench: --- apps/example/src/useClient.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/example/src/useClient.ts b/apps/example/src/useClient.ts index f03b872768..edddff6ec9 100644 --- a/apps/example/src/useClient.ts +++ b/apps/example/src/useClient.ts @@ -1,9 +1,12 @@ import { useEffect, useState } from "react"; import { Platform } from "react-native"; -const ANDROID_WS_HOST = "10.0.2.2"; -const IOS_WS_HOST = "localhost"; -const HOST = Platform.OS === "android" ? ANDROID_WS_HOST : IOS_WS_HOST; +// Both platforms reach the test server on localhost: the iOS Simulator shares +// the host network, and on Android the jest globalSetup runs +// "adb reverse tcp:4242 tcp:4242" (as the RN CLI does for Metro on 8081), which +// covers the emulator and a physical device alike. Only a device that cannot be +// reached over adb needs this machine's LAN IP here instead. +const HOST = "localhost"; const PORT = 4242; type UseClient = [client: WebSocket | null, hostname: string]; @@ -27,7 +30,9 @@ export const useClient = (): UseClient => { }; ws.onopen = () => { setClient(ws); - ws.send(JSON.stringify({ OS: Platform.OS, arch: "paper" })); + // The host is reported back so the test server can hand out fixture URLs + // this device can actually reach (see fixtureUrl in setup.ts). + ws.send(JSON.stringify({ OS: Platform.OS, arch: "paper", host: HOST })); }; // Reconnect on every close, not only on error: the test server closes the // socket cleanly at the end of each jest run, and without a retry here the