feat: prototype editable cursor choreography for AI Edition

electron/ai-edition/document-service.test.ts

@@ -112,7 +112,10 @@ describe("DocumentService", () => {
 			expect(asset?.kind).toBe("video");
 			expect(asset?.originalPath).toBe("/tmp/screen.mp4");
 			expect(asset?.label).toBe("Screen");
+			expect(asset?.autoZoomState).toBe("pending");
 			expect(updated.project.primaryAssetId).toBe(asset?.id);
+			const persisted = await service.getProject(doc.project.id);
+			expect(persisted.assets[0]?.autoZoomState).toBe("pending");
 		});
 
 		it("resolves relative paths against the cwd", async () => {

electron/ai-edition/document-service.ts

@@ -252,6 +252,7 @@ export class DocumentService {
 			originalPath: absolutePath,
 			sizeBytes,
 			cameraTrack: null,
+			autoZoomState: "pending",
 		};
 		const next: AxcutDocument = {
 			...doc,

src/components/ai-edition/CursorPreviewLayer.module.css

@@ -32,3 +32,50 @@
 	-webkit-user-drag: none;
 	display: none;
 }
+
+.motionEditor {
+	position: absolute;
+	inset: 0;
+	z-index: 4;
+	width: 100%;
+	height: 100%;
+	overflow: visible;
+	pointer-events: none;
+}
+
+.motionRecordedPath,
+.motionEditedPath {
+	fill: none;
+	stroke-linecap: round;
+	stroke-linejoin: round;
+}
+
+.motionRecordedPath {
+	stroke: rgba(34, 211, 238, 0.9);
+	stroke-width: 2;
+	stroke-dasharray: 5 5;
+}
+
+.motionEditedPath {
+	stroke: #8b5cf6;
+	stroke-width: 3;
+}
+
+.motionAnchor {
+	fill: #fff;
+	stroke: #8b5cf6;
+	stroke-width: 2;
+}
+
+.motionControl {
+	fill: #8b5cf6;
+	stroke: #fff;
+	stroke-width: 2;
+	pointer-events: auto;
+	cursor: grab;
+	touch-action: none;
+}
+
+.motionControl:active {
+	cursor: grabbing;
+}

src/components/ai-edition/CursorPreviewLayer.test.tsx

@@ -1,5 +1,6 @@
 import "@testing-library/jest-dom";
-import { act, cleanup, render } from "@testing-library/react";
+import { act, cleanup, fireEvent, render } from "@testing-library/react";
+import type { ComponentProps } from "react";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { useProjectStore } from "@/lib/ai-edition/store/projectStore";
 import { CursorPreviewLayer } from "./CursorPreviewLayer";
@@ -113,7 +114,10 @@ const SAMPLE_DOC = {
 	history: { revisions: [] },
 };
 
-function renderWithVideo(): { host: HTMLDivElement; video: HTMLVideoElement } {
+function renderWithVideo(props: Partial<ComponentProps<typeof CursorPreviewLayer>> = {}): {
+	host: HTMLDivElement;
+	video: HTMLVideoElement;
+} {
 	const stage = document.createElement("div");
 	const video = document.createElement("video");
 	stage.appendChild(video);
@@ -134,7 +138,12 @@ function renderWithVideo(): { host: HTMLDivElement; video: HTMLVideoElement } {
 	stage.appendChild(host);
 	act(() => {
 		render(
-			<CursorPreviewLayer videoPath="/tmp/recording.mp4" currentTimeSec={0.5} isPlaying={false} />,
+			<CursorPreviewLayer
+				videoPath="/tmp/recording.mp4"
+				currentTimeSec={0.5}
+				isPlaying={false}
+				{...props}
+			/>,
 			{ container: host },
 		);
 	});
@@ -182,22 +191,21 @@ describe("CursorPreviewLayer (shared)", () => {
 		expect(host.querySelector("img")).toBeNull();
 	});
 
-	it("queries the native bridge for cursor data when videoPath is set", async () => {
-		bridgeMocks.getRecordingData.mockImplementation(async () => null);
-		bridgeMocks.getTelemetry.mockImplementation(async () => []);
-		renderWithVideo();
+	it("uses cursor data supplied by the parent without querying the native bridge", async () => {
+		renderWithVideo({
+			cursorRecordingData: null,
+			cursorTelemetry: [{ timeMs: 500, cx: 0.4, cy: 0.6 }],
+		});
 
 		await act(async () => {
 			await new Promise((resolve) => setTimeout(resolve, 0));
 		});
 
-		expect(bridgeMocks.getRecordingData).toHaveBeenCalledWith("/tmp/recording.mp4");
-		expect(bridgeMocks.getTelemetry).toHaveBeenCalledWith("/tmp/recording.mp4");
+		expect(bridgeMocks.getRecordingData).not.toHaveBeenCalled();
+		expect(bridgeMocks.getTelemetry).not.toHaveBeenCalled();
 	});
 
 	it("hides the native cursor <img> when settings.cursorShow is false", async () => {
-		bridgeMocks.getRecordingData.mockImplementation(async () => null);
-		bridgeMocks.getTelemetry.mockImplementation(async () => []);
 		const document = useProjectStore.getState().document;
 		useProjectStore.setState({
 			document: { ...document!, legacyEditor: { cursorShow: false } },
@@ -213,4 +221,187 @@ describe("CursorPreviewLayer (shared)", () => {
 		expect(img).toBeTruthy();
 		expect((img as HTMLImageElement).style.display).toBe("none");
 	});
+
+	it("shows recorded and edited paths with a draggable control for the selected region", async () => {
+		const onControlPointChange = vi.fn();
+		const onControlPointCommit = vi.fn();
+		const cursorRecordingData = {
+			version: 2,
+			provider: "native" as const,
+			assets: [
+				{
+					id: "cursor-arrow",
+					platform: "win32",
+					imageDataUrl: "data:image/png;base64,AA==",
+					width: 32,
+					height: 32,
+					hotspotX: 0,
+					hotspotY: 0,
+				},
+			],
+			samples: [
+				{ timeMs: 0, cx: 0.2, cy: 0.5, visible: true, assetId: "cursor-arrow" },
+				{ timeMs: 1000, cx: 0.8, cy: 0.5, visible: true, assetId: "cursor-arrow" },
+			],
+		};
+		const region = {
+			id: "motion_1",
+			clipId: "clip_1",
+			assetId: "asset_1",
+			startMs: 0,
+			endMs: 1000,
+			sourceStartMs: 0,
+			sourceEndMs: 1000,
+			startPoint: { cx: 0.2, cy: 0.5 },
+			endPoint: { cx: 0.8, cy: 0.5 },
+			controlPoints: [{ cx: 0.5, cy: 0.2 }],
+			startAnchor: "manual" as const,
+			endAnchor: "click" as const,
+			segmentKind: "move" as const,
+			preset: "arc" as const,
+			speed: 1,
+			cycles: 1,
+			easing: "linear" as const,
+		};
+		const { host } = renderWithVideo({
+			cursorRecordingData,
+			cursorTelemetry: [],
+			assetId: "asset_1",
+			clipId: "clip_1",
+			cursorMotionRegions: [region],
+			selectedCursorMotionRegionId: region.id,
+			onControlPointChange,
+			onControlPointCommit,
+		});
+
+		await act(async () => {
+			await new Promise((resolve) => setTimeout(resolve, 0));
+		});
+
+		const overlay = host.querySelector('svg[aria-label="Selected cursor path"]');
+		expect(overlay).toBeTruthy();
+		expect(overlay?.querySelectorAll("polyline")).toHaveLength(2);
+		expect(overlay?.querySelectorAll("circle")).toHaveLength(3);
+		expect(overlay?.querySelectorAll("polyline")[0].getAttribute("points")).not.toBe(
+			overlay?.querySelectorAll("polyline")[1].getAttribute("points"),
+		);
+
+		const control = overlay?.querySelectorAll("circle")[2] as SVGCircleElement;
+		vi.spyOn(overlay as SVGSVGElement, "getBoundingClientRect").mockReturnValue({
+			x: 0,
+			y: 0,
+			width: 100,
+			height: 100,
+			top: 0,
+			right: 100,
+			bottom: 100,
+			left: 0,
+			toJSON: () => "",
+		});
+		Object.defineProperties(control, {
+			setPointerCapture: { value: vi.fn() },
+			hasPointerCapture: { value: vi.fn(() => true) },
+			releasePointerCapture: { value: vi.fn() },
+		});
+
+		fireEvent.pointerDown(control, { pointerId: 1, clientX: 55, clientY: 35 });
+		fireEvent.pointerMove(control, { pointerId: 1, clientX: 75, clientY: 45 });
+		fireEvent.pointerUp(control, { pointerId: 1, clientX: 75, clientY: 45 });
+
+		expect(onControlPointChange).toHaveBeenLastCalledWith(region.id, 0, {
+			cx: 0.75,
+			cy: 0.45,
+		});
+		expect(onControlPointCommit).toHaveBeenCalledTimes(1);
+	});
+
+	it("projects a cropped motion overlay and saves dragged controls in source coordinates", async () => {
+		const onControlPointChange = vi.fn();
+		const cursorRecordingData = {
+			version: 2,
+			provider: "native" as const,
+			assets: [
+				{
+					id: "cursor-arrow",
+					platform: "win32",
+					imageDataUrl: "data:image/png;base64,AA==",
+					width: 32,
+					height: 32,
+					hotspotX: 0,
+					hotspotY: 0,
+				},
+			],
+			samples: [
+				{ timeMs: 0, cx: 0.25, cy: 0.25, visible: true, assetId: "cursor-arrow" },
+				{ timeMs: 1000, cx: 0.75, cy: 0.75, visible: true, assetId: "cursor-arrow" },
+			],
+		};
+		const region = {
+			id: "motion_crop",
+			clipId: "clip_1",
+			assetId: "asset_1",
+			startMs: 0,
+			endMs: 1000,
+			sourceStartMs: 0,
+			sourceEndMs: 1000,
+			startPoint: { cx: 0.25, cy: 0.25 },
+			endPoint: { cx: 0.75, cy: 0.75 },
+			controlPoints: [{ cx: 0.5, cy: 0.5 }],
+			startAnchor: "manual" as const,
+			endAnchor: "click" as const,
+			segmentKind: "move" as const,
+			preset: "arc" as const,
+			speed: 1,
+			cycles: 1,
+			easing: "linear" as const,
+		};
+		const { host } = renderWithVideo({
+			cursorRecordingData,
+			cursorTelemetry: [],
+			cropRegion: { x: 0.25, y: 0.25, width: 0.5, height: 0.5 },
+			assetId: "asset_1",
+			clipId: "clip_1",
+			cursorMotionRegions: [region],
+			selectedCursorMotionRegionId: region.id,
+			onControlPointChange,
+		});
+
+		await act(async () => {
+			await new Promise((resolve) => setTimeout(resolve, 0));
+		});
+
+		const overlay = host.querySelector('svg[aria-label="Selected cursor path"]');
+		const circles = overlay?.querySelectorAll("circle");
+		expect(circles).toHaveLength(3);
+		expect(circles?.[0]).toHaveAttribute("cx", "0");
+		expect(circles?.[0]).toHaveAttribute("cy", "0");
+		expect(circles?.[1]).toHaveAttribute("cx", "1");
+		expect(circles?.[1]).toHaveAttribute("cy", "1");
+		expect(circles?.[2]).toHaveAttribute("cx", "0.5");
+		expect(circles?.[2]).toHaveAttribute("cy", "0.5");
+
+		vi.spyOn(overlay as SVGSVGElement, "getBoundingClientRect").mockReturnValue({
+			x: 0,
+			y: 0,
+			width: 100,
+			height: 100,
+			top: 0,
+			right: 100,
+			bottom: 100,
+			left: 0,
+			toJSON: () => "",
+		});
+		const control = circles?.[2] as SVGCircleElement;
+		Object.defineProperties(control, {
+			setPointerCapture: { value: vi.fn() },
+			hasPointerCapture: { value: vi.fn(() => true) },
+			releasePointerCapture: { value: vi.fn() },
+		});
+		fireEvent.pointerDown(control, { pointerId: 2, clientX: 80, clientY: 20 });
+
+		expect(onControlPointChange).toHaveBeenLastCalledWith(region.id, 0, {
+			cx: 0.65,
+			cy: 0.35,
+		});
+	});
 });