diff --git a/src/utils/image-processing.ts b/src/utils/image-processing.ts index 3750588..de7d1e6 100644 --- a/src/utils/image-processing.ts +++ b/src/utils/image-processing.ts @@ -134,6 +134,286 @@ export function buildGifPrompt( return parts.join(" "); } +interface ContentBounds { + minX: number; + minY: number; + maxX: number; + maxY: number; +} + +interface ProcessedGridFrame { + pixels: Uint8Array; + width: number; + height: number; + contentBounds: ContentBounds; +} + +function isForegroundPixel( + red: number, + green: number, + blue: number, + alpha: number, +): boolean { + if (alpha < 128) { + return false; + } + return calculateMagentaScore(red, green, blue) < 0.5; +} + +function processMagentaKeyInPlace(pixels: Uint8Array): void { + for (let index = 0; index < pixels.length; index += 4) { + const red = pixels[index] ?? 0; + const green = pixels[index + 1] ?? 0; + const blue = pixels[index + 2] ?? 0; + + const magentaScore = calculateMagentaScore(red, green, blue); + + if (magentaScore > 0.5) { + pixels[index] = 1; + pixels[index + 1] = 1; + pixels[index + 2] = 1; + pixels[index + 3] = 0; + } else if (magentaScore > 0.1) { + const despilled = despillMagenta(red, green, blue, magentaScore); + pixels[index] = despilled.red; + pixels[index + 1] = despilled.green; + pixels[index + 2] = despilled.blue; + pixels[index + 3] = 255; + } else { + pixels[index + 3] = 255; + } + } +} + +function findContentBounds( + pixels: Uint8Array, + width: number, + height: number, +): ContentBounds | null { + let minX = width; + let minY = height; + let maxX = -1; + let maxY = -1; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const index = (y * width + x) * 4; + const red = pixels[index] ?? 0; + const green = pixels[index + 1] ?? 0; + const blue = pixels[index + 2] ?? 0; + const alpha = pixels[index + 3] ?? 0; + + if (!isForegroundPixel(red, green, blue, alpha)) { + continue; + } + + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + + if (maxX < minX || maxY < minY) { + return null; + } + + return { minX, minY, maxX, maxY }; +} + +function computeVerticalEdgeProfile( + pixels: Uint8Array, + width: number, + height: number, +): Float32Array { + const profile = new Float32Array(width); + + for (let x = 1; x < width - 1; x++) { + let edgeStrength = 0; + + for (let y = 0; y < height; y++) { + for (let channel = 0; channel < 3; channel++) { + const leftIndex = (y * width + (x - 1)) * 4 + channel; + const rightIndex = (y * width + (x + 1)) * 4 + channel; + edgeStrength += Math.abs( + (pixels[rightIndex] ?? 0) - (pixels[leftIndex] ?? 0), + ); + } + } + + profile[x] = edgeStrength; + } + + return profile; +} + +function computeHorizontalEdgeProfile( + pixels: Uint8Array, + width: number, + height: number, +): Float32Array { + const profile = new Float32Array(height); + + for (let y = 1; y < height - 1; y++) { + let edgeStrength = 0; + + for (let x = 0; x < width; x++) { + for (let channel = 0; channel < 3; channel++) { + const topIndex = ((y - 1) * width + x) * 4 + channel; + const bottomIndex = ((y + 1) * width + x) * 4 + channel; + edgeStrength += Math.abs( + (pixels[bottomIndex] ?? 0) - (pixels[topIndex] ?? 0), + ); + } + } + + profile[y] = edgeStrength; + } + + return profile; +} + +function refineGridBoundaries( + dimensionSize: number, + gridSize: number, + edgeProfile: Float32Array, +): number[] { + const boundaries = [0]; + const averageCellSize = dimensionSize / gridSize; + const searchRadius = Math.max(2, Math.floor(averageCellSize / 6)); + let previousBoundary = 0; + + for (let boundaryIndex = 1; boundaryIndex < gridSize; boundaryIndex++) { + const expectedPosition = Math.round(boundaryIndex * averageCellSize); + const remainingCells = gridSize - boundaryIndex; + const searchStart = Math.max(previousBoundary + 1, expectedPosition - searchRadius); + const searchEnd = Math.min( + dimensionSize - remainingCells, + expectedPosition + searchRadius, + ); + + let bestPosition = expectedPosition; + let bestScore = -1; + + for (let position = searchStart; position <= searchEnd; position++) { + const score = edgeProfile[position] ?? 0; + if (score > bestScore) { + bestScore = score; + bestPosition = position; + } + } + + boundaries.push(bestPosition); + previousBoundary = bestPosition; + } + + boundaries.push(dimensionSize); + return boundaries; +} + +function extractRawRegion( + pixels: Uint8Array, + imageWidth: number, + left: number, + top: number, + width: number, + height: number, +): Uint8Array { + const region = new Uint8Array(width * height * 4); + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const sourceIndex = ((top + y) * imageWidth + (left + x)) * 4; + const targetIndex = (y * width + x) * 4; + region[targetIndex] = pixels[sourceIndex] ?? 0; + region[targetIndex + 1] = pixels[sourceIndex + 1] ?? 0; + region[targetIndex + 2] = pixels[sourceIndex + 2] ?? 0; + region[targetIndex + 3] = pixels[sourceIndex + 3] ?? 0; + } + } + + return region; +} + +function extractCroppedRegion( + pixels: Uint8Array, + width: number, + height: number, + bounds: ContentBounds, +): { pixels: Uint8Array; width: number; height: number } { + const cropWidth = bounds.maxX - bounds.minX + 1; + const cropHeight = bounds.maxY - bounds.minY + 1; + const cropped = new Uint8Array(cropWidth * cropHeight * 4); + + for (let y = 0; y < cropHeight; y++) { + for (let x = 0; x < cropWidth; x++) { + const sourceIndex = ((bounds.minY + y) * width + (bounds.minX + x)) * 4; + const targetIndex = (y * cropWidth + x) * 4; + cropped[targetIndex] = pixels[sourceIndex] ?? 0; + cropped[targetIndex + 1] = pixels[sourceIndex + 1] ?? 0; + cropped[targetIndex + 2] = pixels[sourceIndex + 2] ?? 0; + cropped[targetIndex + 3] = pixels[sourceIndex + 3] ?? 0; + } + } + + return { pixels: cropped, width: cropWidth, height: cropHeight }; +} + +async function renderStabilizedFrame( + frame: ProcessedGridFrame, + scale: number, + targetSize: number, +): Promise { + const { pixels: croppedPixels, width: cropWidth, height: cropHeight } = + extractCroppedRegion( + frame.pixels, + frame.width, + frame.height, + frame.contentBounds, + ); + + const scaledWidth = Math.max(1, Math.round(cropWidth * scale)); + const scaledHeight = Math.max(1, Math.round(cropHeight * scale)); + + const { data: scaledData } = await sharp(Buffer.from(croppedPixels), { + raw: { width: cropWidth, height: cropHeight, channels: 4 }, + }) + .resize(scaledWidth, scaledHeight, { kernel: "lanczos3" }) + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + + const scaledPixels = new Uint8Array(scaledData); + const canvas = new Uint8Array(targetSize * targetSize * 4); + const offsetX = Math.round((targetSize - scaledWidth) / 2); + const offsetY = Math.round((targetSize - scaledHeight) / 2); + + for (let y = 0; y < scaledHeight; y++) { + for (let x = 0; x < scaledWidth; x++) { + const sourceIndex = (y * scaledWidth + x) * 4; + const targetX = offsetX + x; + const targetY = offsetY + y; + + if ( + targetX < 0 || + targetY < 0 || + targetX >= targetSize || + targetY >= targetSize + ) { + continue; + } + + const targetIndex = (targetY * targetSize + targetX) * 4; + canvas[targetIndex] = scaledPixels[sourceIndex] ?? 0; + canvas[targetIndex + 1] = scaledPixels[sourceIndex + 1] ?? 0; + canvas[targetIndex + 2] = scaledPixels[sourceIndex + 2] ?? 0; + canvas[targetIndex + 3] = scaledPixels[sourceIndex + 3] ?? 0; + } + } + + return canvas; +} + export async function processGifEmojiGrid( inputBuffer: Buffer, options: GifOptions = { frames: 9, fps: 5, loopDelay: 0 }, @@ -146,77 +426,112 @@ export async function processGifEmojiGrid( } const gridSize = Math.sqrt(options.frames); - const frameWidth = Math.floor(metadata.width / gridSize); - const frameHeight = Math.floor(metadata.height / gridSize); const targetSize = 128; const frameDelay = Math.round(1000 / options.fps); + const contentPaddingRatio = 0.92; - const encoder = new GIFEncoder(targetSize, targetSize, "neuquant", true); - encoder.start(); - encoder.setRepeat(0); - encoder.setQuality(1); - encoder.setDispose(2); - encoder.setTransparent(0x010101); + const { data: fullImageData } = await image + .clone() + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); - const processedFrames: Uint8Array[] = []; + const fullPixels = new Uint8Array(fullImageData); + const columnBoundaries = refineGridBoundaries( + metadata.width, + gridSize, + computeVerticalEdgeProfile(fullPixels, metadata.width, metadata.height), + ); + const rowBoundaries = refineGridBoundaries( + metadata.height, + gridSize, + computeHorizontalEdgeProfile(fullPixels, metadata.width, metadata.height), + ); + + const processedFrames: ProcessedGridFrame[] = []; for (let row = 0; row < gridSize; row++) { for (let col = 0; col < gridSize; col++) { - const left = col * frameWidth; - const top = row * frameHeight; - - const { data } = await image - .clone() - .extract({ - left, - top, - width: frameWidth, - height: frameHeight, - }) - .resize(targetSize, targetSize, { - fit: "contain", - background: { r: 255, g: 0, b: 255, alpha: 1 }, - }) - .ensureAlpha() - .raw() - .toBuffer({ resolveWithObject: true }); - - const pixels = new Uint8Array(data); - - for (let i = 0; i < pixels.length; i += 4) { - const red = pixels[i] ?? 0; - const green = pixels[i + 1] ?? 0; - const blue = pixels[i + 2] ?? 0; - - const magentaScore = calculateMagentaScore(red, green, blue); - - if (magentaScore > 0.5) { - pixels[i] = 1; - pixels[i + 1] = 1; - pixels[i + 2] = 1; - pixels[i + 3] = 0; - } else if (magentaScore > 0.1) { - const despilled = despillMagenta(red, green, blue, magentaScore); - pixels[i] = despilled.red; - pixels[i + 1] = despilled.green; - pixels[i + 2] = despilled.blue; - pixels[i + 3] = 255; - } else { - pixels[i + 3] = 255; - } + const left = columnBoundaries[col] ?? 0; + const top = rowBoundaries[row] ?? 0; + const right = columnBoundaries[col + 1] ?? metadata.width; + const bottom = rowBoundaries[row + 1] ?? metadata.height; + const frameWidth = right - left; + const frameHeight = bottom - top; + + if (frameWidth <= 0 || frameHeight <= 0) { + throw new Error("Invalid grid cell dimensions after boundary detection"); + } + + const framePixels = extractRawRegion( + fullPixels, + metadata.width, + left, + top, + frameWidth, + frameHeight, + ); + + processMagentaKeyInPlace(framePixels); + + const contentBounds = findContentBounds( + framePixels, + frameWidth, + frameHeight, + ); + + if (!contentBounds) { + throw new Error("Grid frame has no visible content after processing"); } - processedFrames.push(pixels); + processedFrames.push({ + pixels: framePixels, + width: frameWidth, + height: frameHeight, + contentBounds, + }); } } - for (let i = 0; i < processedFrames.length; i++) { - const isLastFrame = i === processedFrames.length - 1; + let maxContentWidth = 0; + let maxContentHeight = 0; + + for (const frame of processedFrames) { + const contentWidth = + frame.contentBounds.maxX - frame.contentBounds.minX + 1; + const contentHeight = + frame.contentBounds.maxY - frame.contentBounds.minY + 1; + maxContentWidth = Math.max(maxContentWidth, contentWidth); + maxContentHeight = Math.max(maxContentHeight, contentHeight); + } + + const unifiedScale = Math.min( + (targetSize * contentPaddingRatio) / maxContentWidth, + (targetSize * contentPaddingRatio) / maxContentHeight, + ); + + const stabilizedFrames: Uint8Array[] = []; + + for (const frame of processedFrames) { + stabilizedFrames.push( + await renderStabilizedFrame(frame, unifiedScale, targetSize), + ); + } + + const encoder = new GIFEncoder(targetSize, targetSize, "neuquant", true); + encoder.start(); + encoder.setRepeat(0); + encoder.setQuality(1); + encoder.setDispose(2); + encoder.setTransparent(0x010101); + + for (let frameIndex = 0; frameIndex < stabilizedFrames.length; frameIndex++) { + const isLastFrame = frameIndex === stabilizedFrames.length - 1; const delay = isLastFrame && options.loopDelay > 0 - ? frameDelay + (frameDelay * options.loopDelay) + ? frameDelay + frameDelay * options.loopDelay : frameDelay; encoder.setDelay(delay); - encoder.addFrame(processedFrames[i]!); + encoder.addFrame(stabilizedFrames[frameIndex]!); } encoder.finish();