diff --git a/PanTS-Demo/src/components/viewer/MeshViewer.tsx b/PanTS-Demo/src/components/viewer/MeshViewer.tsx
index a5bbc234..e883b1c3 100644
--- a/PanTS-Demo/src/components/viewer/MeshViewer.tsx
+++ b/PanTS-Demo/src/components/viewer/MeshViewer.tsx
@@ -1,5 +1,6 @@
import { Bounds, OrbitControls } from "@react-three/drei";
import { Canvas } from "@react-three/fiber";
+import { registerMeshRoot } from "../../helpers/viewer/meshCapture";
import { Suspense, useEffect, useMemo, useState } from "react";
import { APP_CONSTANTS } from "../../helpers/constants";
import { cornerstoneLpsMmToThree, type Vec3 } from "../../helpers/utils";
@@ -45,6 +46,10 @@ export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, c
return unsubscribe;
}, []);
+ // Drop the renderer handle when this pane goes away, so a capture can never
+ // reach into a disposed WebGL context.
+ useEffect(() => () => registerMeshRoot(null), []);
+
// Segment indices touched since the case loaded — includes edits to the STATIC
// 32-organ catalog, not just brand-new custom classes.
const editedSegments = useMemo(() => getEditedSegments(), [editVersion]);
@@ -88,8 +93,9 @@ export function SegmentationMeshViewer({ caseId, checkState, loading, opacity, c
camera={{ position: [0, 250, 650], fov: 45, near: 0.1, far: 5000 }}
gl={{ preserveDrawingBuffer: true, antialias: true }}
frameloop="always"
- onCreated={({ gl }) => {
- gl.domElement.setAttribute("data-bodymaps-3d", "1");
+ onCreated={(state) => {
+ registerMeshRoot(state);
+ state.gl.domElement.setAttribute("data-bodymaps-3d", "1");
}}
>
diff --git a/PanTS-Demo/src/helpers/viewer/meshCapture.ts b/PanTS-Demo/src/helpers/viewer/meshCapture.ts
new file mode 100644
index 00000000..12deb467
--- /dev/null
+++ b/PanTS-Demo/src/helpers/viewer/meshCapture.ts
@@ -0,0 +1,42 @@
+/**
+ * Snapshot access to the 3D mesh pane's WebGL renderer.
+ *
+ * Reading a WebGL canvas from the outside — query the element, then call
+ * toDataURL — is a race against the browser's compositor. `preserveDrawingBuffer`
+ * narrows the window but does not close it: on some GPU/driver combinations the
+ * buffer the read sees is the cleared one, which is what produced a solid black
+ * "3D view" among the assistant's attached screenshots.
+ *
+ * Holding the renderer lets the capture draw a frame and read the pixels back in
+ * the same synchronous block, where nothing can clear the buffer in between.
+ *
+ * This lives outside MeshViewer.tsx so that file exports only components (the
+ * react-refresh lint rule), and so the capture path has no reason to import a
+ * React component just to reach the renderer.
+ */
+
+import type { RootState } from "@react-three/fiber";
+
+let meshRoot: RootState | null = null;
+
+/** Called by the mesh viewer on mount, and with null when the pane unmounts. */
+export function registerMeshRoot(state: RootState | null): void {
+ meshRoot = state;
+}
+
+/**
+ * A PNG data URL of the current 3D view, or null when no mesh pane is mounted
+ * (single-view layouts, a case whose meshes have not loaded) or the readback
+ * fails. Callers should fall back to their own canvas lookup on null.
+ */
+export function captureMeshCanvas(): string | null {
+ if (!meshRoot) return null;
+ try {
+ const { gl, scene, camera } = meshRoot;
+ gl.render(scene, camera);
+ return gl.domElement.toDataURL("image/png");
+ } catch (error) {
+ console.warn("[BodyMaps AI] 3D readback failed", error);
+ return null;
+ }
+}
diff --git a/PanTS-Demo/src/routes/VisualizationPage.tsx b/PanTS-Demo/src/routes/VisualizationPage.tsx
index a5bdb967..9fef53c8 100644
--- a/PanTS-Demo/src/routes/VisualizationPage.tsx
+++ b/PanTS-Demo/src/routes/VisualizationPage.tsx
@@ -45,6 +45,7 @@ import { track } from "../helpers/analytics";
import { buildViewerActions } from "../components/AIAssistant/assistantActions";
import MeasurementPanel from "../components/MeasurementPanel/MeasurementPanel";
import { SegmentationMeshViewer } from "../components/viewer/MeshViewer";
+import { captureMeshCanvas } from "../helpers/viewer/meshCapture";
import OrganCheckbox from "../components/OrganCheckbox";
import PercentileBar from "../components/PercentileBar";
import SessionHUD from "../components/ReadingSession/SessionHUD";
@@ -1573,57 +1574,70 @@ function VisualizationPage() {
);
});
- // A WebGL canvas read back after its drawing buffer was cleared comes out as
- // one flat color — the "black 3D screenshot". Sample a tiny copy so a dead
- // capture is detected here instead of being sent to the vision model, which
- // would then confidently describe an empty image.
- const captureLooksBlank = (source: HTMLCanvasElement): boolean => {
- try {
- const probe = document.createElement("canvas");
- probe.width = 32;
- probe.height = 32;
- const ctx = probe.getContext("2d", { willReadFrequently: true });
- if (!ctx) return false;
- ctx.drawImage(source, 0, 0, probe.width, probe.height);
- const { data } = ctx.getImageData(0, 0, probe.width, probe.height);
- let min = 255;
- let max = 0;
- for (let i = 0; i < data.length; i += 4) {
- const luma = (data[i] * 299 + data[i + 1] * 587 + data[i + 2] * 114) / 1000;
- if (luma < min) min = luma;
- if (luma > max) max = luma;
- }
- return max - min < 4;
- } catch {
- return false; // unreadable canvas — assume the shot is usable
- }
- };
+ // A WebGL readback that lost its drawing buffer comes out as one flat color —
+ // the "black 3D screenshot". Sample the CAPTURED IMAGE (not the canvas, which
+ // may have been redrawn since) so a dead capture is caught here instead of
+ // being sent to the vision model, which would then confidently describe it.
+ const imageLooksBlank = (dataUrl: string): Promise =>
+ new Promise((resolve) => {
+ const img = new Image();
+ img.onload = () => {
+ try {
+ const probe = document.createElement("canvas");
+ probe.width = 32;
+ probe.height = 32;
+ const ctx = probe.getContext("2d", { willReadFrequently: true });
+ if (!ctx) return resolve(false);
+ ctx.drawImage(img, 0, 0, probe.width, probe.height);
+ const { data } = ctx.getImageData(0, 0, probe.width, probe.height);
+ let min = 255;
+ let max = 0;
+ for (let i = 0; i < data.length; i += 4) {
+ const luma = (data[i] * 299 + data[i + 1] * 587 + data[i + 2] * 114) / 1000;
+ if (luma < min) min = luma;
+ if (luma > max) max = luma;
+ }
+ resolve(max - min < 4);
+ } catch {
+ resolve(false); // unreadable — assume the shot is usable
+ }
+ };
+ img.onerror = () => resolve(true);
+ img.src = dataUrl;
+ });
const captureAllViews = useCallback(async () => {
const shots: { name: string; dataUrl: string }[] = await captureViewportImages();
try {
const pane = document.querySelector(".render");
- // Prefer the canvas the mesh viewer tags on creation; the positional
- // lookup is only a fallback for an older render tree.
- const canvas =
- document.querySelector("canvas[data-bodymaps-3d]") ??
- pane?.querySelector("canvas") ??
- null;
const paneVisible = !pane || pane.offsetParent !== null;
- if (canvas && canvas.width && paneVisible) {
- await nextPresentedFrame();
- let url = canvas.toDataURL("image/png");
- if (captureLooksBlank(canvas)) {
- // One more frame: the pane may have only just become visible.
- await nextPresentedFrame();
- url = canvas.toDataURL("image/png");
+ if (paneVisible) {
+ // Preferred path: have the mesh viewer draw a frame and hand back the
+ // pixels in one synchronous step. Querying the canvas and reading it
+ // afterwards races the compositor, which is what left the 3D pane
+ // solid black even with preserveDrawingBuffer set.
+ let url = captureMeshCanvas();
+
+ if (!url) {
+ // Fallback for a render tree that never registered a handle.
+ const canvas =
+ document.querySelector("canvas[data-bodymaps-3d]") ??
+ pane?.querySelector("canvas") ??
+ null;
+ if (canvas && canvas.width) {
+ await nextPresentedFrame();
+ url = canvas.toDataURL("image/png");
+ }
}
- if (captureLooksBlank(canvas)) {
- console.warn(
- "[BodyMaps AI] 3D pane captured blank — omitting it rather than sending a black image"
- );
- } else if (url && url.length > 128) {
- shots.push({ name: "3d", dataUrl: url });
+
+ if (url && url.length > 128) {
+ if (await imageLooksBlank(url)) {
+ console.warn(
+ "[BodyMaps AI] 3D pane read back blank — omitting it rather than sending a black image"
+ );
+ } else {
+ shots.push({ name: "3d", dataUrl: url });
+ }
}
}
} catch (error) {