diff --git a/packages/webgpu/android/cpp/AndroidPlatformContext.h b/packages/webgpu/android/cpp/AndroidPlatformContext.h index b20078669f..80fe77f859 100644 --- a/packages/webgpu/android/cpp/AndroidPlatformContext.h +++ b/packages/webgpu/android/cpp/AndroidPlatformContext.h @@ -82,8 +82,8 @@ class AndroidPlatformContext : public PlatformContext { return instance.CreateSurface(&surfaceDescriptor); } - ImageData createImageBitmap(std::string blobId, double offset, - double size) override { + ImageData createImageBitmap(std::string blobId, double offset, double size, + bool premultiplyAlpha) override { jni::Environment::ensureCurrentThreadIsAttached(); JNIEnv *env = facebook::jni::Environment::current(); @@ -92,15 +92,16 @@ class AndroidPlatformContext : public PlatformContext { } auto data = resolveBlob(env, blobId, offset, size); - return createImageBitmapFromData(data); + return createImageBitmapFromData(data, premultiplyAlpha); } void createImageBitmapAsync(std::string blobId, double offset, double size, + bool premultiplyAlpha, std::function onSuccess, std::function onError) override { std::thread([this, blobId = std::move(blobId), offset, size, - onSuccess = std::move(onSuccess), + premultiplyAlpha, onSuccess = std::move(onSuccess), onError = std::move(onError)]() { jni::Environment::ensureCurrentThreadIsAttached(); try { @@ -109,7 +110,7 @@ class AndroidPlatformContext : public PlatformContext { throw std::runtime_error("Couldn't get JNI environment"); } auto data = resolveBlob(env, blobId, offset, size); - auto result = createImageBitmapFromData(data); + auto result = createImageBitmapFromData(data, premultiplyAlpha); onSuccess(std::move(result)); } catch (const std::exception &e) { onError(e.what()); @@ -117,7 +118,8 @@ class AndroidPlatformContext : public PlatformContext { }).detach(); } - ImageData createImageBitmapFromData(std::span data) override { + ImageData createImageBitmapFromData(std::span data, + bool /*premultiplyAlpha*/) override { jni::Environment::ensureCurrentThreadIsAttached(); JNIEnv *env = facebook::jni::Environment::current(); @@ -179,6 +181,9 @@ class AndroidPlatformContext : public PlatformContext { result.height = static_cast(bitmapInfo.height); result.data.resize(bitmapInfo.height * bitmapInfo.stride); memcpy(result.data.data(), bitmapPixels, result.data.size()); + result.premultipliedAlpha = + (bitmapInfo.flags & ANDROID_BITMAP_FLAGS_ALPHA_MASK) == + ANDROID_BITMAP_FLAGS_ALPHA_PREMUL; AndroidBitmap_unlockPixels(env, bitmap); @@ -189,15 +194,16 @@ class AndroidPlatformContext : public PlatformContext { } void createImageBitmapFromDataAsync( - std::span data, std::function onSuccess, + std::span data, bool premultiplyAlpha, + std::function onSuccess, std::function onError) override { std::thread([this, ownedData = std::vector(data.begin(), data.end()), - onSuccess = std::move(onSuccess), + premultiplyAlpha, onSuccess = std::move(onSuccess), onError = std::move(onError)]() mutable { jni::Environment::ensureCurrentThreadIsAttached(); try { - auto result = createImageBitmapFromData(ownedData); + auto result = createImageBitmapFromData(ownedData, premultiplyAlpha); onSuccess(std::move(result)); } catch (const std::exception &e) { onError(e.what()); diff --git a/packages/webgpu/apple/ApplePlatformContext.h b/packages/webgpu/apple/ApplePlatformContext.h index 6536663c40..7f3ea3847b 100644 --- a/packages/webgpu/apple/ApplePlatformContext.h +++ b/packages/webgpu/apple/ApplePlatformContext.h @@ -13,18 +13,21 @@ class ApplePlatformContext : public PlatformContext { wgpu::Surface makeSurface(wgpu::Instance instance, void *surface, int width, int height) override; - ImageData createImageBitmap(std::string blobId, double offset, - double size) override; + ImageData createImageBitmap(std::string blobId, double offset, double size, + bool premultiplyAlpha) override; void createImageBitmapAsync(std::string blobId, double offset, double size, + bool premultiplyAlpha, std::function onSuccess, std::function onError) override; - ImageData createImageBitmapFromData(std::span data) override; + ImageData createImageBitmapFromData(std::span data, + bool premultiplyAlpha) override; void createImageBitmapFromDataAsync( - std::span data, std::function onSuccess, + std::span data, bool premultiplyAlpha, + std::function onSuccess, std::function onError) override; VideoFrameHandle loadVideoFrame(const std::string &path) override; diff --git a/packages/webgpu/apple/ApplePlatformContext.mm b/packages/webgpu/apple/ApplePlatformContext.mm index 594337cfcb..d80f5dc77b 100644 --- a/packages/webgpu/apple/ApplePlatformContext.mm +++ b/packages/webgpu/apple/ApplePlatformContext.mm @@ -3,6 +3,7 @@ #include #import +#import #import #import #import @@ -48,7 +49,8 @@ void checkIfUsingSimulatorWithAPIValidation() { } ImageData ApplePlatformContext::createImageBitmap(std::string blobId, - double offset, double size) { + double offset, double size, + bool premultiplyAlpha) { RCTBlobManager *blobManager = [[RCTBridge currentBridge] moduleForClass:RCTBlobManager.class]; NSData *blobData = @@ -60,11 +62,11 @@ void checkIfUsingSimulatorWithAPIValidation() { throw std::runtime_error("Couldn't retrieve blob data"); } - return createImageBitmapFromData(nsDataToSpan(blobData)); + return createImageBitmapFromData(nsDataToSpan(blobData), premultiplyAlpha); } void ApplePlatformContext::createImageBitmapAsync( - std::string blobId, double offset, double size, + std::string blobId, double offset, double size, bool premultiplyAlpha, std::function onSuccess, std::function onError) { // Resolve blob on current thread (requires RCTBridge access) @@ -82,23 +84,48 @@ void checkIfUsingSimulatorWithAPIValidation() { // blobData is alive during this synchronous call; // createImageBitmapFromDataAsync copies the span before dispatching - createImageBitmapFromDataAsync(nsDataToSpan(blobData), std::move(onSuccess), - std::move(onError)); + createImageBitmapFromDataAsync(nsDataToSpan(blobData), premultiplyAlpha, + std::move(onSuccess), std::move(onError)); } ImageData -ApplePlatformContext::createImageBitmapFromData(std::span data) { - // This avoids a copy by assuming the UIImage/NSImage constructors - // decode `nsData` eagerly before the memory for the wrapped `data` - // is freed. - // - // Since we get the `CGImageRef` from `image` and then throw - // it away, that's a fairly safe assumption. +ApplePlatformContext::createImageBitmapFromData(std::span data, + bool premultiplyAlpha) { NSData *nsData = [NSData dataWithBytesNoCopy:const_cast(data.data()) length:data.size() freeWhenDone:NO]; + if (!premultiplyAlpha) { + CIImage *ciImage = + [CIImage imageWithData:nsData + options:@{kCIImageColorSpace : [NSNull null]}]; + if (ciImage != nil) { + // Core Image retains higher precision until it writes straight RGBA8. + size_t width = static_cast(CGRectGetWidth(ciImage.extent)); + size_t height = static_cast(CGRectGetHeight(ciImage.extent)); + size_t bytesPerRow = width * 4; + ImageData result; + result.width = static_cast(width); + result.height = static_cast(height); + result.data.resize(height * bytesPerRow); + result.format = wgpu::TextureFormat::RGBA8Unorm; + result.premultipliedAlpha = false; + + static CIContext *ciContext = [CIContext contextWithOptions:@{ + kCIContextWorkingColorSpace : [NSNull null], + kCIContextOutputPremultiplied : @NO, + }]; + [ciContext render:ciImage + toBitmap:result.data.data() + rowBytes:bytesPerRow + bounds:ciImage.extent + format:kCIFormatRGBA8 + colorSpace:nil]; + return result; + } + } + #if !TARGET_OS_OSX UIImage *image = [UIImage imageWithData:nsData]; #else @@ -125,12 +152,18 @@ void checkIfUsingSimulatorWithAPIValidation() { result.height = static_cast(height); result.data.resize(height * bytesPerRow); result.format = wgpu::TextureFormat::RGBA8Unorm; + result.premultipliedAlpha = true; CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); CGContextRef context = CGBitmapContextCreate( result.data.data(), width, height, bitsPerComponent, bytesPerRow, colorSpace, kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); + if (context == nullptr) { + CGColorSpaceRelease(colorSpace); + throw std::runtime_error("Couldn't create image bitmap context"); + } + CGContextDrawImage(context, CGRectMake(0, 0, width, height), cgImage); CGContextRelease(context); @@ -140,7 +173,8 @@ void checkIfUsingSimulatorWithAPIValidation() { } void ApplePlatformContext::createImageBitmapFromDataAsync( - std::span data, std::function onSuccess, + std::span data, bool premultiplyAlpha, + std::function onSuccess, std::function onError) { // Copy span data into shared_ptr so the dispatch_async block owns the memory auto ownedData = @@ -149,7 +183,7 @@ void checkIfUsingSimulatorWithAPIValidation() { dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ @autoreleasepool { try { - auto result = createImageBitmapFromData(*ownedData); + auto result = createImageBitmapFromData(*ownedData, premultiplyAlpha); onSuccess(std::move(result)); } catch (const std::exception &e) { onError(e.what()); diff --git a/packages/webgpu/cpp/rnwgpu/PlatformContext.h b/packages/webgpu/cpp/rnwgpu/PlatformContext.h index fec049256b..0f0e4b168b 100644 --- a/packages/webgpu/cpp/rnwgpu/PlatformContext.h +++ b/packages/webgpu/cpp/rnwgpu/PlatformContext.h @@ -16,6 +16,7 @@ struct ImageData { size_t width; size_t height; wgpu::TextureFormat format; + bool premultipliedAlpha = true; }; // Pixel layout of a VideoFrame. Determines whether the underlying surface is @@ -73,20 +74,22 @@ class PlatformContext { virtual wgpu::Surface makeSurface(wgpu::Instance instance, void *surface, int width, int height) = 0; virtual ImageData createImageBitmap(std::string blobId, double offset, - double size) = 0; + double size, bool premultiplyAlpha) = 0; // Async version that performs image decoding on a background thread virtual void createImageBitmapAsync(std::string blobId, double offset, double size, + bool premultiplyAlpha, std::function onSuccess, std::function onError) = 0; // Create ImageBitmap from raw encoded image bytes (PNG/JPEG/etc.) - virtual ImageData - createImageBitmapFromData(std::span data) = 0; + virtual ImageData createImageBitmapFromData(std::span data, + bool premultiplyAlpha) = 0; virtual void createImageBitmapFromDataAsync(std::span data, + bool premultiplyAlpha, std::function onSuccess, std::function onError) = 0; diff --git a/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp b/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp index c0b3ff3a67..cde2a2eda0 100644 --- a/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp +++ b/packages/webgpu/cpp/rnwgpu/api/GPUQueue.cpp @@ -1,5 +1,6 @@ #include "GPUQueue.h" +#include #include #include #include @@ -128,26 +129,32 @@ void GPUQueue::copyExternalImageToTexture( throw std::runtime_error("Invalid input for GPUQueue::writeTexture()"); } - if (source->flipY.value_or(false)) { - // Calculate the row size and total size - uint32_t rowSize = bytesPerPixel * source->source->getWidth(); - uint32_t totalSize = source->source->getSize(); - - // Create a new buffer for the flipped data - std::vector flippedData(totalSize); - - // Flip the data vertically - for (uint32_t row = 0; row < source->source->getHeight(); ++row) { - std::memcpy(flippedData.data() + - (source->source->getHeight() - 1 - row) * rowSize, - static_cast(source->source->getData()) + - row * rowSize, - rowSize); + bool flipY = source->flipY.value_or(false); + bool sourcePremultipliedAlpha = source->source->isPremultiplied(); + bool destinationPremultipliedAlpha = + destination->premultipliedAlpha.value_or(false); + bool convertAlpha = sourcePremultipliedAlpha != destinationPremultipliedAlpha; + + if (flipY || convertAlpha) { + size_t rowSize = bytesPerPixel * source->source->getWidth(); + size_t totalSize = source->source->getSize(); + auto sourceData = static_cast(source->source->getData()); + std::vector uploadData(totalSize); + + if (flipY) { + for (size_t row = 0; row < source->source->getHeight(); ++row) { + std::memcpy(uploadData.data() + + (source->source->getHeight() - 1 - row) * rowSize, + sourceData + row * rowSize, rowSize); + } + } else { + std::memcpy(uploadData.data(), sourceData, totalSize); } - // Use the flipped data for writing to texture - _instance.WriteTexture(&dst, flippedData.data(), totalSize, &layout, &sz); - } else { + ImageBitmap::convertAlpha(uploadData, sourcePremultipliedAlpha, + destinationPremultipliedAlpha); + _instance.WriteTexture(&dst, uploadData.data(), totalSize, &layout, &sz); + } else { _instance.WriteTexture(&dst, source->source->getData(), source->source->getSize(), &layout, &sz); } diff --git a/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h b/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h index 3a76c35e07..0e7feca572 100644 --- a/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h +++ b/packages/webgpu/cpp/rnwgpu/api/ImageBitmap.h @@ -15,8 +15,12 @@ class ImageBitmap : public NativeObject { public: static constexpr const char *CLASS_NAME = "ImageBitmap"; - explicit ImageBitmap(ImageData &imageData) - : NativeObject(CLASS_NAME), _imageData(imageData) {} + ImageBitmap(ImageData &imageData, bool premultipliedAlpha) + : NativeObject(CLASS_NAME), _imageData(imageData) { + convertAlpha(_imageData.data, _imageData.premultipliedAlpha, + premultipliedAlpha); + _imageData.premultipliedAlpha = premultipliedAlpha; + } size_t getWidth() { return _imageData.width; } @@ -26,6 +30,32 @@ class ImageBitmap : public NativeObject { size_t getSize() { return _imageData.data.size(); } + bool isPremultiplied() { return _imageData.premultipliedAlpha; } + + static void convertAlpha(std::vector &data, + bool sourcePremultipliedAlpha, + bool destinationPremultipliedAlpha) { + if (sourcePremultipliedAlpha == destinationPremultipliedAlpha) { + return; + } + + for (size_t i = 0; i + 3 < data.size(); i += 4) { + uint32_t alpha = data[i + 3]; + for (size_t channel = 0; channel < 3; ++channel) { + uint32_t value = data[i + channel]; + if (destinationPremultipliedAlpha) { + value = (value * alpha + 127) / 255; + } else if (alpha == 0) { + value = 0; + } else { + value = (value * 255 + alpha / 2) / alpha; + value = value > 255 ? 255 : value; + } + data[i + channel] = static_cast(value); + } + } + } + void close() { _imageData.data.clear(); _imageData.data.shrink_to_fit(); diff --git a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h index e7ecd8a7cc..edcefc90b6 100644 --- a/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h +++ b/packages/webgpu/cpp/rnwgpu/api/RNWebGPU.h @@ -89,6 +89,18 @@ class RNWebGPU : public NativeObject { auto platformContext = _platformContext; auto callInvoker = _callInvoker; + bool premultipliedAlpha = true; + if (count > 1 && args[1].isObject()) { + auto options = args[1].getObject(runtime); + if (options.hasProperty(runtime, "premultiplyAlpha")) { + auto value = options.getProperty(runtime, "premultiplyAlpha"); + auto option = JSIConverter>::fromJSI( + runtime, value, false); + if (option.has_value()) { + premultipliedAlpha = option.value() != PremultiplyAlpha::None; + } + } + } // Check if the argument is an ArrayBuffer or ArrayBufferView // (TypedArray / DataView). Only a real buffer source is run through the @@ -118,14 +130,16 @@ class RNWebGPU : public NativeObject { std::vector dataCopy(data.begin(), data.end()); return Promise::createPromise( - runtime, - [platformContext, callInvoker, dataCopy = std::move(dataCopy)]( - jsi::Runtime & /*runtime*/, - std::shared_ptr promise) mutable { + runtime, [platformContext, callInvoker, premultipliedAlpha, + dataCopy = std::move(dataCopy)]( + jsi::Runtime & /*runtime*/, + std::shared_ptr promise) mutable { platformContext->createImageBitmapFromDataAsync( - dataCopy, - [callInvoker, promise](ImageData imageData) { - auto imageBitmap = std::make_shared(imageData); + dataCopy, premultipliedAlpha, + [callInvoker, promise, + premultipliedAlpha](ImageData imageData) { + auto imageBitmap = std::make_shared( + imageData, premultipliedAlpha); callInvoker->invokeAsync([promise, imageBitmap]() { promise->resolve( JSIConverter>::toJSI( @@ -148,13 +162,14 @@ class RNWebGPU : public NativeObject { double size = blob->size; return Promise::createPromise( - runtime, - [platformContext, callInvoker, blobId, offset, - size](jsi::Runtime & /*runtime*/, std::shared_ptr promise) { + runtime, [platformContext, callInvoker, blobId, offset, size, + premultipliedAlpha](jsi::Runtime & /*runtime*/, + std::shared_ptr promise) { platformContext->createImageBitmapAsync( - blobId, offset, size, - [callInvoker, promise](ImageData imageData) { - auto imageBitmap = std::make_shared(imageData); + blobId, offset, size, premultipliedAlpha, + [callInvoker, promise, premultipliedAlpha](ImageData imageData) { + auto imageBitmap = std::make_shared( + imageData, premultipliedAlpha); callInvoker->invokeAsync([promise, imageBitmap]() { promise->resolve( JSIConverter>::toJSI( diff --git a/packages/webgpu/react-native-webgpu.podspec b/packages/webgpu/react-native-webgpu.podspec index a6b12cf442..8cd56a1466 100644 --- a/packages/webgpu/react-native-webgpu.podspec +++ b/packages/webgpu/react-native-webgpu.podspec @@ -21,9 +21,9 @@ Pod::Spec.new do |s| s.vendored_frameworks = 'libs/apple/libwebgpu_dawn.xcframework' - # The VideoPlayer API uses AVFoundation / CoreMedia, and shared-texture - # surfaces use CoreVideo (CVPixelBuffer). Link them so their symbols resolve. - s.frameworks = "AVFoundation", "CoreMedia", "CoreVideo" + # ImageBitmap uses CoreImage, VideoPlayer uses AVFoundation / CoreMedia, and + # shared-texture surfaces use CoreVideo (CVPixelBuffer). + s.frameworks = "AVFoundation", "CoreImage", "CoreMedia", "CoreVideo" s.pod_target_xcconfig = { 'HEADER_SEARCH_PATHS' => '$(PODS_TARGET_SRCROOT)/cpp', diff --git a/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts new file mode 100644 index 0000000000..1767ed1a7c --- /dev/null +++ b/packages/webgpu/src/__tests__/ImageBitmapAlpha.spec.ts @@ -0,0 +1,348 @@ +import fs from "fs"; +import path from "path"; + +import { PNG } from "pngjs"; + +import { checkImage, client, encodeImage } from "./setup"; + +// A 32x32 gradient with a vertical alpha ramp (row 0 fully transparent, last +// row fully opaque) and a horizontal color ramp, so alpha-mode mistakes and +// vertical flips both produce large snapshot diffs. The baselines are computed +// with the same integer arithmetic as ImageBitmap::convertAlpha; pixelmatch +// tolerance absorbs the platform decoders' off-by-one rounding differences +// (e.g. Android decodes premultiplied, so "none" is a lossy round trip there). +const assetPath = path.resolve(__dirname, "./assets/alpha-gradient.png"); +const pngBase64 = fs.readFileSync(assetPath).toString("base64"); + +type SourceAlpha = "none" | "premultiply" | "default" | "omitted"; +type DestinationAlpha = boolean | "omitted"; + +interface BitmapResult { + data: number[]; + width: number; + height: number; + format: string; +} + +const flipRows = (bitmap: BitmapResult): BitmapResult => { + const rowSize = bitmap.width * 4; + const data: number[] = []; + for (let y = bitmap.height - 1; y >= 0; y--) { + data.push(...bitmap.data.slice(y * rowSize, (y + 1) * rowSize)); + } + return { ...bitmap, data }; +}; + +const snapshotFor = ( + sourceAlpha: SourceAlpha, + destinationAlpha: DestinationAlpha, +) => { + if (destinationAlpha === true) { + return "snapshots/image-bitmap-alpha-premultiplied.png"; + } + if (sourceAlpha === "none") { + return "snapshots/image-bitmap-alpha-straight.png"; + } + // "premultiply", "default", and omitted options all store premultiplied + // data, so copying to a straight-alpha destination is a lossy round trip. + return "snapshots/image-bitmap-alpha-roundtrip.png"; +}; + +interface BlobCase { + sourceAlpha: SourceAlpha; + destinationAlpha: DestinationAlpha; + flipY: boolean; +} + +const blobCases: BlobCase[] = ( + ["none", "premultiply", "omitted"] as const +).flatMap((sourceAlpha) => + (["omitted", false, true] as const).flatMap((destinationAlpha) => + ([false, true] as const).map((flipY) => ({ + sourceAlpha, + destinationAlpha, + flipY, + })), + ), +); +// premultiplyAlpha: "default" behaves like "premultiply"; one case is enough. +blobCases.push({ + sourceAlpha: "default", + destinationAlpha: false, + flipY: false, +}); + +describe("ImageBitmap alpha representation", () => { + it.each(blobCases)( + "blob source=$sourceAlpha destination=$destinationAlpha flipY=$flipY", + async ({ sourceAlpha, destinationAlpha, flipY }) => { + const result = await client.eval( + ({ + device, + pngBase64: encodedPng, + sourceAlpha: sourceRepresentation, + destinationAlpha: destinationRepresentation, + flipY: shouldFlip, + }) => { + return fetch(`data:image/png;base64,${encodedPng}`) + .then((response) => response.blob()) + .then((blob) => + createImageBitmap( + blob, + sourceRepresentation === "omitted" + ? undefined + : { premultiplyAlpha: sourceRepresentation }, + ), + ) + .then((bitmap) => { + const { width } = bitmap; + const { height } = bitmap; + const texture = device.createTexture({ + size: [width, height], + format: "rgba8unorm", + usage: + GPUTextureUsage.COPY_DST | + GPUTextureUsage.COPY_SRC | + GPUTextureUsage.RENDER_ATTACHMENT, + }); + device.queue.copyExternalImageToTexture( + { source: bitmap, flipY: shouldFlip }, + destinationRepresentation === "omitted" + ? { texture } + : { texture, premultipliedAlpha: destinationRepresentation }, + [width, height], + ); + + const bytesPerRow = 256; + const output = device.createBuffer({ + size: bytesPerRow * height, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [width, height], + ); + device.queue.submit([encoder.finish()]); + + return output.mapAsync(GPUMapMode.READ).then(() => { + const mapped = new Uint8Array(output.getMappedRange()); + const data: number[] = []; + for (let y = 0; y < height; y++) { + for (let i = 0; i < width * 4; i++) { + data.push(mapped[y * bytesPerRow + i]); + } + } + output.unmap(); + output.destroy(); + texture.destroy(); + bitmap.close(); + return { data, width, height, format: "rgba8unorm" }; + }); + }); + }, + { + pngBase64, + sourceAlpha, + destinationAlpha, + flipY, + }, + ); + + const upright = flipY ? flipRows(result) : result; + checkImage( + encodeImage(upright), + snapshotFor(sourceAlpha, destinationAlpha), + ); + }, + ); + + // The React Native ArrayBuffer overload of createImageBitmap goes through + // createImageBitmapFromDataAsync, a separate native path from the blob one. + // It is not part of the standard web API, so the reference client skips it. + const bufferCases: { + sourceAlpha: "none" | "premultiply"; + destinationAlpha: boolean; + }[] = (["none", "premultiply"] as const).flatMap((sourceAlpha) => + ([false, true] as const).map((destinationAlpha) => ({ + sourceAlpha, + destinationAlpha, + })), + ); + it.each(bufferCases)( + "buffer source=$sourceAlpha destination=$destinationAlpha", + async ({ sourceAlpha, destinationAlpha }) => { + if (client.OS === "web") { + return; + } + const pngData = Array.from(fs.readFileSync(assetPath)); + const result = await client.eval( + ({ + device, + pngData: encodedPng, + sourceAlpha: sourceRepresentation, + destinationAlpha: destinationRepresentation, + }) => { + const bytes = new Uint8Array(encodedPng); + return createImageBitmap(bytes.buffer, { + premultiplyAlpha: sourceRepresentation, + }).then((bitmap) => { + const { width } = bitmap; + const { height } = bitmap; + const texture = device.createTexture({ + size: [width, height], + format: "rgba8unorm", + usage: + GPUTextureUsage.COPY_DST | + GPUTextureUsage.COPY_SRC | + GPUTextureUsage.RENDER_ATTACHMENT, + }); + device.queue.copyExternalImageToTexture( + { source: bitmap }, + { texture, premultipliedAlpha: destinationRepresentation }, + [width, height], + ); + + const bytesPerRow = 256; + const output = device.createBuffer({ + size: bytesPerRow * height, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [width, height], + ); + device.queue.submit([encoder.finish()]); + + return output.mapAsync(GPUMapMode.READ).then(() => { + const mapped = new Uint8Array(output.getMappedRange()); + const data: number[] = []; + for (let y = 0; y < height; y++) { + for (let i = 0; i < width * 4; i++) { + data.push(mapped[y * bytesPerRow + i]); + } + } + output.unmap(); + output.destroy(); + texture.destroy(); + bitmap.close(); + return { data, width, height, format: "rgba8unorm" }; + }); + }); + }, + { + pngData, + sourceAlpha, + destinationAlpha, + }, + ); + + checkImage( + encodeImage(result), + snapshotFor(sourceAlpha, destinationAlpha), + ); + }, + ); + + // Exact-value check for the lossless straight-alpha guarantee this feature + // adds on Apple: premultiplyAlpha "none" must preserve the decoded bytes + // bit-for-bit, and conversions must use round-to-nearest. Runs where + // exactness is guaranteed: iOS (Core Image decode) and the node client + // (whose polyfill mirrors the C++ integer math). Android decodes + // premultiplied, so "none" is lossy there; the reference browser's rounding + // may legitimately differ by one. Both are covered by the snapshot matrix. + const straightRows = [ + [128, 128, 128, 128], + [17, 34, 51, 64], + ]; + const premultipliedRows = [ + [64, 64, 64, 128], + [4, 9, 13, 64], + ]; + it.each([ + { sourceAlpha: "none", destinationAlpha: false, expected: straightRows }, + { + sourceAlpha: "none", + destinationAlpha: true, + expected: premultipliedRows, + }, + ] as const)( + "preserves exact bytes for source=$sourceAlpha destination=$destinationAlpha", + async ({ sourceAlpha, destinationAlpha, expected }) => { + if (client.OS !== "ios" && client.OS !== "node") { + return; + } + const png = new PNG({ width: 1, height: 2 }); + png.data = Buffer.from(straightRows.flat()); + const fixtureBase64 = PNG.sync.write(png).toString("base64"); + + const result = await client.eval( + ({ + device, + pngBase64: encodedPng, + sourceAlpha: sourceRepresentation, + destinationAlpha: destinationRepresentation, + }) => { + return fetch(`data:image/png;base64,${encodedPng}`) + .then((response) => response.blob()) + .then((blob) => + createImageBitmap(blob, { + premultiplyAlpha: sourceRepresentation, + }), + ) + .then((bitmap) => { + const texture = device.createTexture({ + size: [1, 2], + format: "rgba8unorm", + usage: + GPUTextureUsage.COPY_DST | + GPUTextureUsage.COPY_SRC | + GPUTextureUsage.RENDER_ATTACHMENT, + }); + device.queue.copyExternalImageToTexture( + { source: bitmap }, + { texture, premultipliedAlpha: destinationRepresentation }, + [1, 2], + ); + + const bytesPerRow = 256; + const output = device.createBuffer({ + size: bytesPerRow * 2, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer( + { texture }, + { buffer: output, bytesPerRow }, + [1, 2], + ); + device.queue.submit([encoder.finish()]); + + return output.mapAsync(GPUMapMode.READ).then(() => { + const data = new Uint8Array(output.getMappedRange()); + const rows = [ + Array.from(data.slice(0, 4)), + Array.from(data.slice(bytesPerRow, bytesPerRow + 4)), + ]; + output.unmap(); + output.destroy(); + texture.destroy(); + bitmap.close(); + return rows; + }); + }); + }, + { + pngBase64: fixtureBase64, + sourceAlpha, + destinationAlpha, + }, + ); + + expect(result).toEqual(expected); + }, + ); +}); diff --git a/packages/webgpu/src/__tests__/assets/alpha-gradient.png b/packages/webgpu/src/__tests__/assets/alpha-gradient.png new file mode 100644 index 0000000000..e247d7112c Binary files /dev/null and b/packages/webgpu/src/__tests__/assets/alpha-gradient.png differ diff --git a/packages/webgpu/src/__tests__/setup.ts b/packages/webgpu/src/__tests__/setup.ts index dbe182cddf..ca38712057 100644 --- a/packages/webgpu/src/__tests__/setup.ts +++ b/packages/webgpu/src/__tests__/setup.ts @@ -486,30 +486,74 @@ class NodeTestingClient implements TestingClient { // dawn.node implements the core WebGPU API but none of the web-platform // image machinery, so provide the minimal pieces the specs rely on. private installWebPolyfills(device: GPUDevice) { - const decodePng = (bytes: Uint8Array) => { + // Same integer arithmetic as ImageBitmap::convertAlpha (ImageBitmap.h), + // so this client is a bit-exact reference for the native conversion. + const convertAlpha = ( + data: Uint8Array | Uint8ClampedArray, + sourcePremultiplied: boolean, + destinationPremultiplied: boolean, + ) => { + if (sourcePremultiplied === destinationPremultiplied) { + return; + } + for (let i = 0; i + 3 < data.length; i += 4) { + const alpha = data[i + 3]; + for (let channel = 0; channel < 3; channel++) { + const value = data[i + channel]; + if (destinationPremultiplied) { + data[i + channel] = Math.floor((value * alpha + 127) / 255); + } else if (alpha === 0) { + data[i + channel] = 0; + } else { + data[i + channel] = Math.min( + 255, + Math.floor((value * 255 + (alpha >> 1)) / alpha), + ); + } + } + } + }; + const decodePng = (bytes: Uint8Array, premultiplied: boolean) => { const png = PNG.sync.read( Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), ); - return { - data: new Uint8ClampedArray(png.data), + const data = new Uint8ClampedArray(png.data); + convertAlpha(data, false, premultiplied); + const bitmap = { + data, width: png.width, height: png.height, - close() {}, + premultiplied, + // Mirror the native ImageBitmap: close() releases the pixels and + // zeroes the dimensions. + close() { + bitmap.data = new Uint8ClampedArray(0); + bitmap.width = 0; + bitmap.height = 0; + }, }; + return bitmap; }; (globalThis as Record).createImageBitmap = async ( source: unknown, + options?: { premultiplyAlpha?: string }, ) => { + // Like the native implementation, "default" premultiplies. + const premultiplied = options?.premultiplyAlpha !== "none"; if (source instanceof ArrayBuffer) { - return decodePng(new Uint8Array(source)); + return decodePng(new Uint8Array(source), premultiplied); } if (ArrayBuffer.isView(source)) { return decodePng( new Uint8Array(source.buffer, source.byteOffset, source.byteLength), + premultiplied, ); } if (typeof Blob !== "undefined" && source instanceof Blob) { - return decodePng(new Uint8Array(await source.arrayBuffer())); + return decodePng( + new Uint8Array(await source.arrayBuffer()), + premultiplied, + ); } if ( source !== null && @@ -518,30 +562,66 @@ class NodeTestingClient implements TestingClient { "width" in source ) { // Already an ImageData-like object (e.g. one of the test assets). + // Left untagged so the copy shim writes its bytes through unchanged. return source; } throw new Error("createImageBitmap polyfill: unsupported source"); }; // copyExternalImageToTexture expects an ImageBitmap; route the raw RGBA - // bytes of our ImageData-like sources through writeTexture instead. + // bytes of our ImageData-like sources through writeTexture instead, + // honoring flipY and the alpha representations on both sides. Object.defineProperty(device.queue, "copyExternalImageToTexture", { configurable: true, value: ( source: { - source: { data: Uint8ClampedArray; width: number; height: number }; + source: { + data: Uint8ClampedArray; + width: number; + height: number; + premultiplied?: boolean; + }; + flipY?: boolean; }, - destination: GPUTexelCopyTextureInfo, + destination: GPUTexelCopyTextureInfo & { premultipliedAlpha?: boolean }, copySize: GPUExtent3DStrict, ) => { - const { data, width } = source.source; + const { data, width, height, premultiplied } = source.source; + const rowSize = width * 4; + let bytes = new Uint8Array( + data.buffer as ArrayBuffer, + data.byteOffset, + data.byteLength, + ); + const flipY = source.flipY === true; + // Untagged sources (raw test assets) are copied through unchanged. + const needsConversion = + premultiplied !== undefined && + premultiplied !== (destination.premultipliedAlpha === true); + if (flipY || needsConversion) { + const converted = new Uint8Array(bytes.length); + if (flipY) { + for (let row = 0; row < height; row++) { + converted.set( + bytes.subarray(row * rowSize, (row + 1) * rowSize), + (height - 1 - row) * rowSize, + ); + } + } else { + converted.set(bytes); + } + if (needsConversion) { + convertAlpha( + converted, + premultiplied === true, + destination.premultipliedAlpha === true, + ); + } + bytes = converted; + } device.queue.writeTexture( destination, - new Uint8Array( - data.buffer as ArrayBuffer, - data.byteOffset, - data.byteLength, - ), - { bytesPerRow: width * 4 }, + bytes, + { bytesPerRow: rowSize }, copySize, ); }, diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-premultiplied.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-premultiplied.png new file mode 100644 index 0000000000..935053edaa Binary files /dev/null and b/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-premultiplied.png differ diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-roundtrip.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-roundtrip.png new file mode 100644 index 0000000000..5cec866965 Binary files /dev/null and b/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-roundtrip.png differ diff --git a/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-straight.png b/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-straight.png new file mode 100644 index 0000000000..e247d7112c Binary files /dev/null and b/packages/webgpu/src/__tests__/snapshots/image-bitmap-alpha-straight.png differ diff --git a/packages/webgpu/src/index.tsx b/packages/webgpu/src/index.tsx index 2f89eae6ef..7d7ab35360 100644 --- a/packages/webgpu/src/index.tsx +++ b/packages/webgpu/src/index.tsx @@ -110,5 +110,6 @@ declare global { // Extend createImageBitmap to accept ArrayBuffer/TypedArray (encoded image bytes) function createImageBitmap( image: ArrayBuffer | ArrayBufferView, + options?: ImageBitmapOptions, ): Promise; }