From 188f6d1305e276707ee153088c23f93ea27e6e0e Mon Sep 17 00:00:00 2001 From: Alexander Poon Date: Sun, 16 Aug 2026 17:08:06 -0400 Subject: [PATCH] Capture the 3D pane by rendering and reading back in one step --- .../src/components/viewer/MeshViewer.tsx | 10 +- PanTS-Demo/src/helpers/viewer/meshCapture.ts | 42 ++++++++ PanTS-Demo/src/routes/VisualizationPage.tsx | 102 ++++++++++-------- 3 files changed, 108 insertions(+), 46 deletions(-) create mode 100644 PanTS-Demo/src/helpers/viewer/meshCapture.ts diff --git a/PanTS-Demo/src/components/viewer/MeshViewer.tsx b/PanTS-Demo/src/components/viewer/MeshViewer.tsx index a5bbc234..e883b1c3 100644 --- a/PanTS-Demo/src/components/viewer/MeshViewer.tsx +++ b/PanTS-Demo/src/components/viewer/MeshViewer.tsx @@ -1,5 +1,6 @@ import { Bounds, OrbitControls } from "@react-three/drei"; import { Canvas } from "@react-three/fiber"; +import { registerMeshRoot } from "../../helpers/viewer/meshCapture"; import { Suspense, useEffect, useMemo, useState } from "react"; import { APP_CONSTANTS } from "../../helpers/constants"; import { cornerstoneLpsMmToThree, type Vec3 } from "../../helpers/utils"; @@ -45,6 +46,10 @@ export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, c return unsubscribe; }, []); + // Drop the renderer handle when this pane goes away, so a capture can never + // reach into a disposed WebGL context. + useEffect(() => () => registerMeshRoot(null), []); + // Segment indices touched since the case loaded — includes edits to the STATIC // 32-organ catalog, not just brand-new custom classes. const editedSegments = useMemo(() => getEditedSegments(), [editVersion]); @@ -88,8 +93,9 @@ export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, c camera={{ position: [0, 250, 650], fov: 45, near: 0.1, far: 5000 }} gl={{ preserveDrawingBuffer: true, antialias: true }} frameloop="always" - onCreated={({ gl }) => { - gl.domElement.setAttribute("data-bodymaps-3d", "1"); + onCreated={(state) => { + registerMeshRoot(state); + state.gl.domElement.setAttribute("data-bodymaps-3d", "1"); }} > diff --git a/PanTS-Demo/src/helpers/viewer/meshCapture.ts b/PanTS-Demo/src/helpers/viewer/meshCapture.ts new file mode 100644 index 00000000..12deb467 --- /dev/null +++ b/PanTS-Demo/src/helpers/viewer/meshCapture.ts @@ -0,0 +1,42 @@ +/** + * Snapshot access to the 3D mesh pane's WebGL renderer. + * + * Reading a WebGL canvas from the outside — query the element, then call + * toDataURL — is a race against the browser's compositor. `preserveDrawingBuffer` + * narrows the window but does not close it: on some GPU/driver combinations the + * buffer the read sees is the cleared one, which is what produced a solid black + * "3D view" among the assistant's attached screenshots. + * + * Holding the renderer lets the capture draw a frame and read the pixels back in + * the same synchronous block, where nothing can clear the buffer in between. + * + * This lives outside MeshViewer.tsx so that file exports only components (the + * react-refresh lint rule), and so the capture path has no reason to import a + * React component just to reach the renderer. + */ + +import type { RootState } from "@react-three/fiber"; + +let meshRoot: RootState | null = null; + +/** Called by the mesh viewer on mount, and with null when the pane unmounts. */ +export function registerMeshRoot(state: RootState | null): void { + meshRoot = state; +} + +/** + * A PNG data URL of the current 3D view, or null when no mesh pane is mounted + * (single-view layouts, a case whose meshes have not loaded) or the readback + * fails. Callers should fall back to their own canvas lookup on null. + */ +export function captureMeshCanvas(): string | null { + if (!meshRoot) return null; + try { + const { gl, scene, camera } = meshRoot; + gl.render(scene, camera); + return gl.domElement.toDataURL("image/png"); + } catch (error) { + console.warn("[BodyMaps AI] 3D readback failed", error); + return null; + } +} diff --git a/PanTS-Demo/src/routes/VisualizationPage.tsx b/PanTS-Demo/src/routes/VisualizationPage.tsx index a5bdb967..9fef53c8 100644 --- a/PanTS-Demo/src/routes/VisualizationPage.tsx +++ b/PanTS-Demo/src/routes/VisualizationPage.tsx @@ -45,6 +45,7 @@ import { track } from "../helpers/analytics"; import { buildViewerActions } from "../components/AIAssistant/assistantActions"; import MeasurementPanel from "../components/MeasurementPanel/MeasurementPanel"; import { SegmentationMeshViewer } from "../components/viewer/MeshViewer"; +import { captureMeshCanvas } from "../helpers/viewer/meshCapture"; import OrganCheckbox from "../components/OrganCheckbox"; import PercentileBar from "../components/PercentileBar"; import SessionHUD from "../components/ReadingSession/SessionHUD"; @@ -1573,57 +1574,70 @@ function VisualizationPage() { ); }); - // A WebGL canvas read back after its drawing buffer was cleared comes out as - // one flat color — the "black 3D screenshot". Sample a tiny copy so a dead - // capture is detected here instead of being sent to the vision model, which - // would then confidently describe an empty image. - const captureLooksBlank = (source: HTMLCanvasElement): boolean => { - try { - const probe = document.createElement("canvas"); - probe.width = 32; - probe.height = 32; - const ctx = probe.getContext("2d", { willReadFrequently: true }); - if (!ctx) return false; - ctx.drawImage(source, 0, 0, probe.width, probe.height); - const { data } = ctx.getImageData(0, 0, probe.width, probe.height); - let min = 255; - let max = 0; - for (let i = 0; i < data.length; i += 4) { - const luma = (data[i] * 299 + data[i + 1] * 587 + data[i + 2] * 114) / 1000; - if (luma < min) min = luma; - if (luma > max) max = luma; - } - return max - min < 4; - } catch { - return false; // unreadable canvas — assume the shot is usable - } - }; + // A WebGL readback that lost its drawing buffer comes out as one flat color — + // the "black 3D screenshot". Sample the CAPTURED IMAGE (not the canvas, which + // may have been redrawn since) so a dead capture is caught here instead of + // being sent to the vision model, which would then confidently describe it. + const imageLooksBlank = (dataUrl: string): Promise => + new Promise((resolve) => { + const img = new Image(); + img.onload = () => { + try { + const probe = document.createElement("canvas"); + probe.width = 32; + probe.height = 32; + const ctx = probe.getContext("2d", { willReadFrequently: true }); + if (!ctx) return resolve(false); + ctx.drawImage(img, 0, 0, probe.width, probe.height); + const { data } = ctx.getImageData(0, 0, probe.width, probe.height); + let min = 255; + let max = 0; + for (let i = 0; i < data.length; i += 4) { + const luma = (data[i] * 299 + data[i + 1] * 587 + data[i + 2] * 114) / 1000; + if (luma < min) min = luma; + if (luma > max) max = luma; + } + resolve(max - min < 4); + } catch { + resolve(false); // unreadable — assume the shot is usable + } + }; + img.onerror = () => resolve(true); + img.src = dataUrl; + }); const captureAllViews = useCallback(async () => { const shots: { name: string; dataUrl: string }[] = await captureViewportImages(); try { const pane = document.querySelector(".render"); - // Prefer the canvas the mesh viewer tags on creation; the positional - // lookup is only a fallback for an older render tree. - const canvas = - document.querySelector("canvas[data-bodymaps-3d]") ?? - pane?.querySelector("canvas") ?? - null; const paneVisible = !pane || pane.offsetParent !== null; - if (canvas && canvas.width && paneVisible) { - await nextPresentedFrame(); - let url = canvas.toDataURL("image/png"); - if (captureLooksBlank(canvas)) { - // One more frame: the pane may have only just become visible. - await nextPresentedFrame(); - url = canvas.toDataURL("image/png"); + if (paneVisible) { + // Preferred path: have the mesh viewer draw a frame and hand back the + // pixels in one synchronous step. Querying the canvas and reading it + // afterwards races the compositor, which is what left the 3D pane + // solid black even with preserveDrawingBuffer set. + let url = captureMeshCanvas(); + + if (!url) { + // Fallback for a render tree that never registered a handle. + const canvas = + document.querySelector("canvas[data-bodymaps-3d]") ?? + pane?.querySelector("canvas") ?? + null; + if (canvas && canvas.width) { + await nextPresentedFrame(); + url = canvas.toDataURL("image/png"); + } } - if (captureLooksBlank(canvas)) { - console.warn( - "[BodyMaps AI] 3D pane captured blank — omitting it rather than sending a black image" - ); - } else if (url && url.length > 128) { - shots.push({ name: "3d", dataUrl: url }); + + if (url && url.length > 128) { + if (await imageLooksBlank(url)) { + console.warn( + "[BodyMaps AI] 3D pane read back blank — omitting it rather than sending a black image" + ); + } else { + shots.push({ name: "3d", dataUrl: url }); + } } } } catch (error) {