From 5bae82d71d4c349bfd439f2335db3816a68f23df Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 4 Jun 2026 20:58:52 +0000 Subject: [PATCH] Stabilize GIF frames with grid detection and content alignment Detect imprecise frame grid boundaries via edge profiles instead of naive equal splits. Crop each cell to its content, scale all frames with a shared factor, and center them on the output canvas to stop jiggle. Co-authored-by: Anshu Chimala --- src/utils/image-processing.ts | 427 +++++++++++++++++++++++++++++----- 1 file changed, 371 insertions(+), 56 deletions(-) diff --git a/src/utils/image-processing.ts b/src/utils/image-processing.ts index 3750588..de7d1e6 100644 --- a/src/utils/image-processing.ts +++ b/src/utils/image-processing.ts @@ -134,6 +134,286 @@ export function buildGifPrompt( return parts.join(" "); } +interface ContentBounds { + minX: number; + minY: number; + maxX: number; + maxY: number; +} + +interface ProcessedGridFrame { + pixels: Uint8Array; + width: number; + height: number; + contentBounds: ContentBounds; +} + +function isForegroundPixel( + red: number, + green: number, + blue: number, + alpha: number, +): boolean { + if (alpha < 128) { + return false; + } + return calculateMagentaScore(red, green, blue) < 0.5; +} + +function processMagentaKeyInPlace(pixels: Uint8Array): void { + for (let index = 0; index < pixels.length; index += 4) { + const red = pixels[index] ?? 0; + const green = pixels[index + 1] ?? 0; + const blue = pixels[index + 2] ?? 0; + + const magentaScore = calculateMagentaScore(red, green, blue); + + if (magentaScore > 0.5) { + pixels[index] = 1; + pixels[index + 1] = 1; + pixels[index + 2] = 1; + pixels[index + 3] = 0; + } else if (magentaScore > 0.1) { + const despilled = despillMagenta(red, green, blue, magentaScore); + pixels[index] = despilled.red; + pixels[index + 1] = despilled.green; + pixels[index + 2] = despilled.blue; + pixels[index + 3] = 255; + } else { + pixels[index + 3] = 255; + } + } +} + +function findContentBounds( + pixels: Uint8Array, + width: number, + height: number, +): ContentBounds | null { + let minX = width; + let minY = height; + let maxX = -1; + let maxY = -1; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const index = (y * width + x) * 4; + const red = pixels[index] ?? 0; + const green = pixels[index + 1] ?? 0; + const blue = pixels[index + 2] ?? 0; + const alpha = pixels[index + 3] ?? 0; + + if (!isForegroundPixel(red, green, blue, alpha)) { + continue; + } + + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + + if (maxX < minX || maxY < minY) { + return null; + } + + return { minX, minY, maxX, maxY }; +} + +function computeVerticalEdgeProfile( + pixels: Uint8Array, + width: number, + height: number, +): Float32Array { + const profile = new Float32Array(width); + + for (let x = 1; x < width - 1; x++) { + let edgeStrength = 0; + + for (let y = 0; y < height; y++) { + for (let channel = 0; channel < 3; channel++) { + const leftIndex = (y * width + (x - 1)) * 4 + channel; + const rightIndex = (y * width + (x + 1)) * 4 + channel; + edgeStrength += Math.abs( + (pixels[rightIndex] ?? 0) - (pixels[leftIndex] ?? 0), + ); + } + } + + profile[x] = edgeStrength; + } + + return profile; +} + +function computeHorizontalEdgeProfile( + pixels: Uint8Array, + width: number, + height: number, +): Float32Array { + const profile = new Float32Array(height); + + for (let y = 1; y < height - 1; y++) { + let edgeStrength = 0; + + for (let x = 0; x < width; x++) { + for (let channel = 0; channel < 3; channel++) { + const topIndex = ((y - 1) * width + x) * 4 + channel; + const bottomIndex = ((y + 1) * width + x) * 4 + channel; + edgeStrength += Math.abs( + (pixels[bottomIndex] ?? 0) - (pixels[topIndex] ?? 0), + ); + } + } + + profile[y] = edgeStrength; + } + + return profile; +} + +function refineGridBoundaries( + dimensionSize: number, + gridSize: number, + edgeProfile: Float32Array, +): number[] { + const boundaries = [0]; + const averageCellSize = dimensionSize / gridSize; + const searchRadius = Math.max(2, Math.floor(averageCellSize / 6)); + let previousBoundary = 0; + + for (let boundaryIndex = 1; boundaryIndex < gridSize; boundaryIndex++) { + const expectedPosition = Math.round(boundaryIndex * averageCellSize); + const remainingCells = gridSize - boundaryIndex; + const searchStart = Math.max(previousBoundary + 1, expectedPosition - searchRadius); + const searchEnd = Math.min( + dimensionSize - remainingCells, + expectedPosition + searchRadius, + ); + + let bestPosition = expectedPosition; + let bestScore = -1; + + for (let position = searchStart; position <= searchEnd; position++) { + const score = edgeProfile[position] ?? 0; + if (score > bestScore) { + bestScore = score; + bestPosition = position; + } + } + + boundaries.push(bestPosition); + previousBoundary = bestPosition; + } + + boundaries.push(dimensionSize); + return boundaries; +} + +function extractRawRegion( + pixels: Uint8Array, + imageWidth: number, + left: number, + top: number, + width: number, + height: number, +): Uint8Array { + const region = new Uint8Array(width * height * 4); + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const sourceIndex = ((top + y) * imageWidth + (left + x)) * 4; + const targetIndex = (y * width + x) * 4; + region[targetIndex] = pixels[sourceIndex] ?? 0; + region[targetIndex + 1] = pixels[sourceIndex + 1] ?? 0; + region[targetIndex + 2] = pixels[sourceIndex + 2] ?? 0; + region[targetIndex + 3] = pixels[sourceIndex + 3] ?? 0; + } + } + + return region; +} + +function extractCroppedRegion( + pixels: Uint8Array, + width: number, + height: number, + bounds: ContentBounds, +): { pixels: Uint8Array; width: number; height: number } { + const cropWidth = bounds.maxX - bounds.minX + 1; + const cropHeight = bounds.maxY - bounds.minY + 1; + const cropped = new Uint8Array(cropWidth * cropHeight * 4); + + for (let y = 0; y < cropHeight; y++) { + for (let x = 0; x < cropWidth; x++) { + const sourceIndex = ((bounds.minY + y) * width + (bounds.minX + x)) * 4; + const targetIndex = (y * cropWidth + x) * 4; + cropped[targetIndex] = pixels[sourceIndex] ?? 0; + cropped[targetIndex + 1] = pixels[sourceIndex + 1] ?? 0; + cropped[targetIndex + 2] = pixels[sourceIndex + 2] ?? 0; + cropped[targetIndex + 3] = pixels[sourceIndex + 3] ?? 0; + } + } + + return { pixels: cropped, width: cropWidth, height: cropHeight }; +} + +async function renderStabilizedFrame( + frame: ProcessedGridFrame, + scale: number, + targetSize: number, +): Promise { + const { pixels: croppedPixels, width: cropWidth, height: cropHeight } = + extractCroppedRegion( + frame.pixels, + frame.width, + frame.height, + frame.contentBounds, + ); + + const scaledWidth = Math.max(1, Math.round(cropWidth * scale)); + const scaledHeight = Math.max(1, Math.round(cropHeight * scale)); + + const { data: scaledData } = await sharp(Buffer.from(croppedPixels), { + raw: { width: cropWidth, height: cropHeight, channels: 4 }, + }) + .resize(scaledWidth, scaledHeight, { kernel: "lanczos3" }) + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + + const scaledPixels = new Uint8Array(scaledData); + const canvas = new Uint8Array(targetSize * targetSize * 4); + const offsetX = Math.round((targetSize - scaledWidth) / 2); + const offsetY = Math.round((targetSize - scaledHeight) / 2); + + for (let y = 0; y < scaledHeight; y++) { + for (let x = 0; x < scaledWidth; x++) { + const sourceIndex = (y * scaledWidth + x) * 4; + const targetX = offsetX + x; + const targetY = offsetY + y; + + if ( + targetX < 0 || + targetY < 0 || + targetX >= targetSize || + targetY >= targetSize + ) { + continue; + } + + const targetIndex = (targetY * targetSize + targetX) * 4; + canvas[targetIndex] = scaledPixels[sourceIndex] ?? 0; + canvas[targetIndex + 1] = scaledPixels[sourceIndex + 1] ?? 0; + canvas[targetIndex + 2] = scaledPixels[sourceIndex + 2] ?? 0; + canvas[targetIndex + 3] = scaledPixels[sourceIndex + 3] ?? 0; + } + } + + return canvas; +} + export async function processGifEmojiGrid( inputBuffer: Buffer, options: GifOptions = { frames: 9, fps: 5, loopDelay: 0 }, @@ -146,77 +426,112 @@ export async function processGifEmojiGrid( } const gridSize = Math.sqrt(options.frames); - const frameWidth = Math.floor(metadata.width / gridSize); - const frameHeight = Math.floor(metadata.height / gridSize); const targetSize = 128; const frameDelay = Math.round(1000 / options.fps); + const contentPaddingRatio = 0.92; - const encoder = new GIFEncoder(targetSize, targetSize, "neuquant", true); - encoder.start(); - encoder.setRepeat(0); - encoder.setQuality(1); - encoder.setDispose(2); - encoder.setTransparent(0x010101); + const { data: fullImageData } = await image + .clone() + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); - const processedFrames: Uint8Array[] = []; + const fullPixels = new Uint8Array(fullImageData); + const columnBoundaries = refineGridBoundaries( + metadata.width, + gridSize, + computeVerticalEdgeProfile(fullPixels, metadata.width, metadata.height), + ); + const rowBoundaries = refineGridBoundaries( + metadata.height, + gridSize, + computeHorizontalEdgeProfile(fullPixels, metadata.width, metadata.height), + ); + + const processedFrames: ProcessedGridFrame[] = []; for (let row = 0; row < gridSize; row++) { for (let col = 0; col < gridSize; col++) { - const left = col * frameWidth; - const top = row * frameHeight; - - const { data } = await image - .clone() - .extract({ - left, - top, - width: frameWidth, - height: frameHeight, - }) - .resize(targetSize, targetSize, { - fit: "contain", - background: { r: 255, g: 0, b: 255, alpha: 1 }, - }) - .ensureAlpha() - .raw() - .toBuffer({ resolveWithObject: true }); - - const pixels = new Uint8Array(data); - - for (let i = 0; i < pixels.length; i += 4) { - const red = pixels[i] ?? 0; - const green = pixels[i + 1] ?? 0; - const blue = pixels[i + 2] ?? 0; - - const magentaScore = calculateMagentaScore(red, green, blue); - - if (magentaScore > 0.5) { - pixels[i] = 1; - pixels[i + 1] = 1; - pixels[i + 2] = 1; - pixels[i + 3] = 0; - } else if (magentaScore > 0.1) { - const despilled = despillMagenta(red, green, blue, magentaScore); - pixels[i] = despilled.red; - pixels[i + 1] = despilled.green; - pixels[i + 2] = despilled.blue; - pixels[i + 3] = 255; - } else { - pixels[i + 3] = 255; - } + const left = columnBoundaries[col] ?? 0; + const top = rowBoundaries[row] ?? 0; + const right = columnBoundaries[col + 1] ?? metadata.width; + const bottom = rowBoundaries[row + 1] ?? metadata.height; + const frameWidth = right - left; + const frameHeight = bottom - top; + + if (frameWidth <= 0 || frameHeight <= 0) { + throw new Error("Invalid grid cell dimensions after boundary detection"); + } + + const framePixels = extractRawRegion( + fullPixels, + metadata.width, + left, + top, + frameWidth, + frameHeight, + ); + + processMagentaKeyInPlace(framePixels); + + const contentBounds = findContentBounds( + framePixels, + frameWidth, + frameHeight, + ); + + if (!contentBounds) { + throw new Error("Grid frame has no visible content after processing"); } - processedFrames.push(pixels); + processedFrames.push({ + pixels: framePixels, + width: frameWidth, + height: frameHeight, + contentBounds, + }); } } - for (let i = 0; i < processedFrames.length; i++) { - const isLastFrame = i === processedFrames.length - 1; + let maxContentWidth = 0; + let maxContentHeight = 0; + + for (const frame of processedFrames) { + const contentWidth = + frame.contentBounds.maxX - frame.contentBounds.minX + 1; + const contentHeight = + frame.contentBounds.maxY - frame.contentBounds.minY + 1; + maxContentWidth = Math.max(maxContentWidth, contentWidth); + maxContentHeight = Math.max(maxContentHeight, contentHeight); + } + + const unifiedScale = Math.min( + (targetSize * contentPaddingRatio) / maxContentWidth, + (targetSize * contentPaddingRatio) / maxContentHeight, + ); + + const stabilizedFrames: Uint8Array[] = []; + + for (const frame of processedFrames) { + stabilizedFrames.push( + await renderStabilizedFrame(frame, unifiedScale, targetSize), + ); + } + + const encoder = new GIFEncoder(targetSize, targetSize, "neuquant", true); + encoder.start(); + encoder.setRepeat(0); + encoder.setQuality(1); + encoder.setDispose(2); + encoder.setTransparent(0x010101); + + for (let frameIndex = 0; frameIndex < stabilizedFrames.length; frameIndex++) { + const isLastFrame = frameIndex === stabilizedFrames.length - 1; const delay = isLastFrame && options.loopDelay > 0 - ? frameDelay + (frameDelay * options.loopDelay) + ? frameDelay + frameDelay * options.loopDelay : frameDelay; encoder.setDelay(delay); - encoder.addFrame(processedFrames[i]!); + encoder.addFrame(stabilizedFrames[frameIndex]!); } encoder.finish();