From 707fccdf25f529334721be85d2d5c6184d932e7e Mon Sep 17 00:00:00 2001 From: YoneRai12 <183966348+YoneRai12@users.noreply.github.com> Date: Sun, 19 Jul 2026 06:34:46 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20AI=20Edition=E3=81=AB=E7=B7=A8=E9=9B=86?= =?UTF-8?q?=E5=8F=AF=E8=83=BD=E3=81=AA=E3=82=AB=E3=83=BC=E3=82=BD=E3=83=AB?= =?UTF-8?q?=E8=BB=8C=E9=81=93=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- electron/ai-edition/document-service.test.ts | 3 + electron/ai-edition/document-service.ts | 1 + .../ai-edition/CursorPreviewLayer.module.css | 47 ++ .../ai-edition/CursorPreviewLayer.test.tsx | 213 ++++++- .../ai-edition/CursorPreviewLayer.tsx | 343 ++++++++++- src/components/ai-edition/ExportDialog.tsx | 57 +- src/components/ai-edition/NewEditorShell.tsx | 7 + src/components/ai-edition/Preview.tsx | 17 + src/components/ai-edition/PreviewCanvas.tsx | 9 + .../ai-edition/VirtualPreview.test.tsx | 182 ++++++ src/components/ai-edition/VirtualPreview.tsx | 101 +++- .../ai-edition/v4/EditorShellV4.module.css | 19 + .../ai-edition/v4/FloatingInspector.test.tsx | 54 ++ .../ai-edition/v4/FloatingInspector.tsx | 145 +++++ .../ai-edition/v4/V4Timeline.test.tsx | 251 ++++++++ src/components/ai-edition/v4/V4Timeline.tsx | 231 +++++++- .../videoPlayback/zoomRegionUtils.test.ts | 62 ++ .../videoPlayback/zoomRegionUtils.ts | 41 +- src/i18n/locales/ar/timeline.json | 13 + src/i18n/locales/en/timeline.json | 13 + src/i18n/locales/es/timeline.json | 13 + src/i18n/locales/fr/timeline.json | 13 + src/i18n/locales/it/timeline.json | 13 + src/i18n/locales/ja-JP/timeline.json | 13 + src/i18n/locales/ko-KR/timeline.json | 13 + src/i18n/locales/pt-BR/timeline.json | 13 + src/i18n/locales/ru/timeline.json | 13 + src/i18n/locales/tr/timeline.json | 13 + src/i18n/locales/vi/timeline.json | 13 + src/i18n/locales/zh-CN/timeline.json | 13 + src/i18n/locales/zh-TW/timeline.json | 13 + src/lib/ai-edition/document/migrate.test.ts | 3 +- src/lib/ai-edition/document/migrate.ts | 2 + src/lib/ai-edition/document/timeline.test.ts | 42 ++ src/lib/ai-edition/document/timeline.ts | 48 +- .../exporter/documentExporter.test.ts | 26 +- .../ai-edition/exporter/documentExporter.ts | 6 +- .../ai-edition/exporter/renderPlan.test.ts | 194 ++++-- src/lib/ai-edition/exporter/renderPlan.ts | 133 +++-- src/lib/ai-edition/schema/index.test.ts | 147 ++++- src/lib/ai-edition/schema/index.ts | 40 ++ .../ai-edition/store/editorSettings.test.ts | 16 + src/lib/ai-edition/store/editorSettings.ts | 6 + src/lib/ai-edition/store/projectStore.test.ts | 13 + src/lib/ai-edition/store/projectStore.ts | 5 + src/lib/ai-edition/store/useTimeline.test.ts | 478 +++++++++++++++ src/lib/ai-edition/store/useTimeline.ts | 385 +++++++++--- .../ai-edition/timeline/zoom-preview.test.ts | 40 ++ src/lib/ai-edition/timeline/zoom-preview.ts | 2 + .../timeline/zoom-suggestions.test.ts | 133 ++++- .../ai-edition/timeline/zoom-suggestions.ts | 57 ++ src/lib/cursor/cursorMotion.test.ts | 330 +++++++++++ src/lib/cursor/cursorMotion.ts | 561 ++++++++++++++++++ src/lib/exporter/frameRenderer.test.ts | 50 ++ src/lib/exporter/frameRenderer.ts | 95 ++- src/lib/exporter/videoExporter.ts | 19 +- 56 files changed, 4426 insertions(+), 357 deletions(-) create mode 100644 src/components/ai-edition/VirtualPreview.test.tsx create mode 100644 src/components/ai-edition/v4/FloatingInspector.test.tsx create mode 100644 src/components/ai-edition/v4/V4Timeline.test.tsx create mode 100644 src/components/video-editor/videoPlayback/zoomRegionUtils.test.ts create mode 100644 src/lib/ai-edition/timeline/zoom-preview.test.ts create mode 100644 src/lib/cursor/cursorMotion.test.ts create mode 100644 src/lib/cursor/cursorMotion.ts diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 6baf50838..ab45963cb 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -112,7 +112,10 @@ describe("DocumentService", () => { expect(asset?.kind).toBe("video"); expect(asset?.originalPath).toBe("/tmp/screen.mp4"); expect(asset?.label).toBe("Screen"); + expect(asset?.autoZoomState).toBe("pending"); expect(updated.project.primaryAssetId).toBe(asset?.id); + const persisted = await service.getProject(doc.project.id); + expect(persisted.assets[0]?.autoZoomState).toBe("pending"); }); it("resolves relative paths against the cwd", async () => { diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index fc40d9b9a..f70adbcf4 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -252,6 +252,7 @@ export class DocumentService { originalPath: absolutePath, sizeBytes, cameraTrack: null, + autoZoomState: "pending", }; const next: AxcutDocument = { ...doc, diff --git a/src/components/ai-edition/CursorPreviewLayer.module.css b/src/components/ai-edition/CursorPreviewLayer.module.css index 8237d3aef..5379c4092 100644 --- a/src/components/ai-edition/CursorPreviewLayer.module.css +++ b/src/components/ai-edition/CursorPreviewLayer.module.css @@ -32,3 +32,50 @@ -webkit-user-drag: none; display: none; } + +.motionEditor { + position: absolute; + inset: 0; + z-index: 4; + width: 100%; + height: 100%; + overflow: visible; + pointer-events: none; +} + +.motionRecordedPath, +.motionEditedPath { + fill: none; + stroke-linecap: round; + stroke-linejoin: round; +} + +.motionRecordedPath { + stroke: rgba(34, 211, 238, 0.9); + stroke-width: 2; + stroke-dasharray: 5 5; +} + +.motionEditedPath { + stroke: #8b5cf6; + stroke-width: 3; +} + +.motionAnchor { + fill: #fff; + stroke: #8b5cf6; + stroke-width: 2; +} + +.motionControl { + fill: #8b5cf6; + stroke: #fff; + stroke-width: 2; + pointer-events: auto; + cursor: grab; + touch-action: none; +} + +.motionControl:active { + cursor: grabbing; +} diff --git a/src/components/ai-edition/CursorPreviewLayer.test.tsx b/src/components/ai-edition/CursorPreviewLayer.test.tsx index 37b08a0ef..b33e478bd 100644 --- a/src/components/ai-edition/CursorPreviewLayer.test.tsx +++ b/src/components/ai-edition/CursorPreviewLayer.test.tsx @@ -1,5 +1,6 @@ import "@testing-library/jest-dom"; -import { act, cleanup, render } from "@testing-library/react"; +import { act, cleanup, fireEvent, render } from "@testing-library/react"; +import type { ComponentProps } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { CursorPreviewLayer } from "./CursorPreviewLayer"; @@ -113,7 +114,10 @@ const SAMPLE_DOC = { history: { revisions: [] }, }; -function renderWithVideo(): { host: HTMLDivElement; video: HTMLVideoElement } { +function renderWithVideo(props: Partial> = {}): { + host: HTMLDivElement; + video: HTMLVideoElement; +} { const stage = document.createElement("div"); const video = document.createElement("video"); stage.appendChild(video); @@ -134,7 +138,12 @@ function renderWithVideo(): { host: HTMLDivElement; video: HTMLVideoElement } { stage.appendChild(host); act(() => { render( - , + , { container: host }, ); }); @@ -182,22 +191,21 @@ describe("CursorPreviewLayer (shared)", () => { expect(host.querySelector("img")).toBeNull(); }); - it("queries the native bridge for cursor data when videoPath is set", async () => { - bridgeMocks.getRecordingData.mockImplementation(async () => null); - bridgeMocks.getTelemetry.mockImplementation(async () => []); - renderWithVideo(); + it("uses cursor data supplied by the parent without querying the native bridge", async () => { + renderWithVideo({ + cursorRecordingData: null, + cursorTelemetry: [{ timeMs: 500, cx: 0.4, cy: 0.6 }], + }); await act(async () => { await new Promise((resolve) => setTimeout(resolve, 0)); }); - expect(bridgeMocks.getRecordingData).toHaveBeenCalledWith("/tmp/recording.mp4"); - expect(bridgeMocks.getTelemetry).toHaveBeenCalledWith("/tmp/recording.mp4"); + expect(bridgeMocks.getRecordingData).not.toHaveBeenCalled(); + expect(bridgeMocks.getTelemetry).not.toHaveBeenCalled(); }); it("hides the native cursor when settings.cursorShow is false", async () => { - bridgeMocks.getRecordingData.mockImplementation(async () => null); - bridgeMocks.getTelemetry.mockImplementation(async () => []); const document = useProjectStore.getState().document; useProjectStore.setState({ document: { ...document!, legacyEditor: { cursorShow: false } }, @@ -213,4 +221,187 @@ describe("CursorPreviewLayer (shared)", () => { expect(img).toBeTruthy(); expect((img as HTMLImageElement).style.display).toBe("none"); }); + + it("shows recorded and edited paths with a draggable control for the selected region", async () => { + const onControlPointChange = vi.fn(); + const onControlPointCommit = vi.fn(); + const cursorRecordingData = { + version: 2, + provider: "native" as const, + assets: [ + { + id: "cursor-arrow", + platform: "win32", + imageDataUrl: "data:image/png;base64,AA==", + width: 32, + height: 32, + hotspotX: 0, + hotspotY: 0, + }, + ], + samples: [ + { timeMs: 0, cx: 0.2, cy: 0.5, visible: true, assetId: "cursor-arrow" }, + { timeMs: 1000, cx: 0.8, cy: 0.5, visible: true, assetId: "cursor-arrow" }, + ], + }; + const region = { + id: "motion_1", + clipId: "clip_1", + assetId: "asset_1", + startMs: 0, + endMs: 1000, + sourceStartMs: 0, + sourceEndMs: 1000, + startPoint: { cx: 0.2, cy: 0.5 }, + endPoint: { cx: 0.8, cy: 0.5 }, + controlPoints: [{ cx: 0.5, cy: 0.2 }], + startAnchor: "manual" as const, + endAnchor: "click" as const, + segmentKind: "move" as const, + preset: "arc" as const, + speed: 1, + cycles: 1, + easing: "linear" as const, + }; + const { host } = renderWithVideo({ + cursorRecordingData, + cursorTelemetry: [], + assetId: "asset_1", + clipId: "clip_1", + cursorMotionRegions: [region], + selectedCursorMotionRegionId: region.id, + onControlPointChange, + onControlPointCommit, + }); + + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); + + const overlay = host.querySelector('svg[aria-label="Selected cursor path"]'); + expect(overlay).toBeTruthy(); + expect(overlay?.querySelectorAll("polyline")).toHaveLength(2); + expect(overlay?.querySelectorAll("circle")).toHaveLength(3); + expect(overlay?.querySelectorAll("polyline")[0].getAttribute("points")).not.toBe( + overlay?.querySelectorAll("polyline")[1].getAttribute("points"), + ); + + const control = overlay?.querySelectorAll("circle")[2] as SVGCircleElement; + vi.spyOn(overlay as SVGSVGElement, "getBoundingClientRect").mockReturnValue({ + x: 0, + y: 0, + width: 100, + height: 100, + top: 0, + right: 100, + bottom: 100, + left: 0, + toJSON: () => "", + }); + Object.defineProperties(control, { + setPointerCapture: { value: vi.fn() }, + hasPointerCapture: { value: vi.fn(() => true) }, + releasePointerCapture: { value: vi.fn() }, + }); + + fireEvent.pointerDown(control, { pointerId: 1, clientX: 55, clientY: 35 }); + fireEvent.pointerMove(control, { pointerId: 1, clientX: 75, clientY: 45 }); + fireEvent.pointerUp(control, { pointerId: 1, clientX: 75, clientY: 45 }); + + expect(onControlPointChange).toHaveBeenLastCalledWith(region.id, 0, { + cx: 0.75, + cy: 0.45, + }); + expect(onControlPointCommit).toHaveBeenCalledTimes(1); + }); + + it("projects a cropped motion overlay and saves dragged controls in source coordinates", async () => { + const onControlPointChange = vi.fn(); + const cursorRecordingData = { + version: 2, + provider: "native" as const, + assets: [ + { + id: "cursor-arrow", + platform: "win32", + imageDataUrl: "data:image/png;base64,AA==", + width: 32, + height: 32, + hotspotX: 0, + hotspotY: 0, + }, + ], + samples: [ + { timeMs: 0, cx: 0.25, cy: 0.25, visible: true, assetId: "cursor-arrow" }, + { timeMs: 1000, cx: 0.75, cy: 0.75, visible: true, assetId: "cursor-arrow" }, + ], + }; + const region = { + id: "motion_crop", + clipId: "clip_1", + assetId: "asset_1", + startMs: 0, + endMs: 1000, + sourceStartMs: 0, + sourceEndMs: 1000, + startPoint: { cx: 0.25, cy: 0.25 }, + endPoint: { cx: 0.75, cy: 0.75 }, + controlPoints: [{ cx: 0.5, cy: 0.5 }], + startAnchor: "manual" as const, + endAnchor: "click" as const, + segmentKind: "move" as const, + preset: "arc" as const, + speed: 1, + cycles: 1, + easing: "linear" as const, + }; + const { host } = renderWithVideo({ + cursorRecordingData, + cursorTelemetry: [], + cropRegion: { x: 0.25, y: 0.25, width: 0.5, height: 0.5 }, + assetId: "asset_1", + clipId: "clip_1", + cursorMotionRegions: [region], + selectedCursorMotionRegionId: region.id, + onControlPointChange, + }); + + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); + + const overlay = host.querySelector('svg[aria-label="Selected cursor path"]'); + const circles = overlay?.querySelectorAll("circle"); + expect(circles).toHaveLength(3); + expect(circles?.[0]).toHaveAttribute("cx", "0"); + expect(circles?.[0]).toHaveAttribute("cy", "0"); + expect(circles?.[1]).toHaveAttribute("cx", "1"); + expect(circles?.[1]).toHaveAttribute("cy", "1"); + expect(circles?.[2]).toHaveAttribute("cx", "0.5"); + expect(circles?.[2]).toHaveAttribute("cy", "0.5"); + + vi.spyOn(overlay as SVGSVGElement, "getBoundingClientRect").mockReturnValue({ + x: 0, + y: 0, + width: 100, + height: 100, + top: 0, + right: 100, + bottom: 100, + left: 0, + toJSON: () => "", + }); + const control = circles?.[2] as SVGCircleElement; + Object.defineProperties(control, { + setPointerCapture: { value: vi.fn() }, + hasPointerCapture: { value: vi.fn(() => true) }, + releasePointerCapture: { value: vi.fn() }, + }); + fireEvent.pointerDown(control, { pointerId: 2, clientX: 80, clientY: 20 }); + + expect(onControlPointChange).toHaveBeenLastCalledWith(region.id, 0, { + cx: 0.65, + cy: 0.35, + }); + }); }); diff --git a/src/components/ai-edition/CursorPreviewLayer.tsx b/src/components/ai-edition/CursorPreviewLayer.tsx index 7a4d70a5f..7186eb9a5 100644 --- a/src/components/ai-edition/CursorPreviewLayer.tsx +++ b/src/components/ai-edition/CursorPreviewLayer.tsx @@ -12,8 +12,24 @@ // (which carries the additional zoom / crop / 3D transform math). import { Application, Container } from "pixi.js"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { + type PointerEvent as ReactPointerEvent, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import type { CropRegion, CursorTelemetryPoint } from "@/components/video-editor/types"; +import type { AxcutCursorMotionRegion } from "@/lib/ai-edition/schema"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; +import { + type CursorMotionOwner, + type CursorMotionPath, + findCursorMotionRegionAtSourceTime, + projectCursorMotionPointToCrop, + sampleCursorMotion, + unprojectCursorMotionPointFromCrop, +} from "@/lib/cursor/cursorMotion"; import { getSmoothedCursorPath } from "@/lib/cursor/cursorPathSmoothing"; import { createNativeCursorMotionBlurState, @@ -29,29 +45,74 @@ import { PixiCursorOverlay, preloadCursorAssets, } from "@/lib/cursor/pixiCursorRenderer"; -import { useCursorRecordingData } from "@/native/hooks/useCursorRecordingData"; -import { useCursorTelemetry } from "@/native/hooks/useCursorTelemetry"; +import type { CursorRecordingData } from "@/native/contracts"; import styles from "./CursorPreviewLayer.module.css"; -interface CursorPreviewLayerProps { +const IDENTITY_CURSOR_CROP: CropRegion = { x: 0, y: 0, width: 1, height: 1 }; + +export interface CursorPreviewLayerProps { videoPath: string | null; currentTimeSec: number; isPlaying: boolean; + cursorRecordingData?: CursorRecordingData | null; + cursorTelemetry?: CursorTelemetryPoint[]; + cropRegion?: CropRegion; + assetId?: string | null; + clipId?: string | null; + cursorMotionRegions?: AxcutCursorMotionRegion[]; + selectedCursorMotionRegionId?: string | null; + onControlPointChange?: (id: string, index: number, point: { cx: number; cy: number }) => void; + onControlPointCommit?: () => void; } export function CursorPreviewLayer({ videoPath, currentTimeSec, isPlaying, + cursorRecordingData = null, + cursorTelemetry = [], + cropRegion = IDENTITY_CURSOR_CROP, + assetId = null, + clipId = null, + cursorMotionRegions = [], + selectedCursorMotionRegionId = null, + onControlPointChange, + onControlPointCommit, }: CursorPreviewLayerProps) { const { settings } = useEditorSettings(); - const { data: cursorRecordingData } = useCursorRecordingData(videoPath); - const { samples: cursorTelemetry } = useCursorTelemetry(videoPath); const hasNativeCursor = useMemo( () => hasNativeCursorRecordingData(cursorRecordingData), [cursorRecordingData], ); + const recordedMotionPath = useMemo(() => { + const data = hasNativeCursor + ? cursorRecordingData + : cursorTelemetry.length > 0 + ? { + version: 1, + provider: "none" as const, + samples: cursorTelemetry as never, + assets: [], + } + : null; + const path = getSmoothedCursorPath(data, settings.cursor.smoothing); + return path ? { sampleAtSourceTime: (timeMs) => path.sampleAt(timeMs) } : null; + }, [cursorRecordingData, cursorTelemetry, hasNativeCursor, settings.cursor.smoothing]); + const owner = useMemo( + () => (assetId && clipId ? { assetId, clipId } : null), + [assetId, clipId], + ); + const selectedMotionRegion = useMemo( + () => + cursorMotionRegions.find( + (region) => + region.id === selectedCursorMotionRegionId && + region.assetId === assetId && + region.clipId === clipId, + ) ?? null, + [cursorMotionRegions, selectedCursorMotionRegionId, assetId, clipId], + ); // Layer ref — the DOM
that hosts the Pixi canvas + native-cursor // . Sized to the
); } +function CursorMotionEditorOverlay({ + region, + owner, + path, + cropRegion, + isPlaying, + onControlPointChange, + onControlPointCommit, +}: { + region: AxcutCursorMotionRegion; + owner: CursorMotionOwner; + path: CursorMotionPath; + cropRegion: CropRegion; + isPlaying: boolean; + onControlPointChange?: (id: string, index: number, point: { cx: number; cy: number }) => void; + onControlPointCommit?: () => void; +}) { + const draggingIndexRef = useRef(null); + const trajectories = useMemo(() => { + const recorded: Array<{ cx: number; cy: number }> = []; + const edited: Array<{ cx: number; cy: number }> = []; + const steps = 48; + for (let index = 0; index <= steps; index += 1) { + const timeMs = + region.sourceStartMs + ((region.sourceEndMs - region.sourceStartMs) * index) / steps; + const recordedPoint = path.sampleAtSourceTime(timeMs); + const projectedRecorded = recordedPoint + ? projectCursorMotionPointToCrop(recordedPoint, cropRegion) + : null; + if (projectedRecorded) recorded.push(projectedRecorded); + const editedPoint = sampleCursorMotion({ + path, + regions: [region], + owner, + sourceTimeMs: timeMs, + }); + const projectedEdited = editedPoint + ? projectCursorMotionPointToCrop(editedPoint, cropRegion) + : null; + if (projectedEdited) edited.push(projectedEdited); + } + return { recorded, edited }; + }, [cropRegion, owner, path, region]); + const points = (values: Array<{ cx: number; cy: number }>) => + values.map((point) => `${point.cx.toFixed(4)},${point.cy.toFixed(4)}`).join(" "); + const sourceControls = + region.controlPoints.length > 0 + ? region.controlPoints + : [ + { + cx: (region.startPoint.cx + region.endPoint.cx) / 2, + cy: (region.startPoint.cy + region.endPoint.cy) / 2, + }, + ]; + const controls = sourceControls + .map((point, index) => ({ + index, + point: projectCursorMotionPointToCrop(point, cropRegion), + })) + .filter( + (control): control is { index: number; point: { cx: number; cy: number } } => + control.point !== null, + ); + const projectedStart = projectCursorMotionPointToCrop(region.startPoint, cropRegion); + const projectedEnd = projectCursorMotionPointToCrop(region.endPoint, cropRegion); + + const updateControlPoint = (event: ReactPointerEvent, index: number) => { + const svg = event.currentTarget.ownerSVGElement; + if (!svg) return; + const bounds = svg.getBoundingClientRect(); + if (bounds.width <= 0 || bounds.height <= 0) return; + const sourcePoint = unprojectCursorMotionPointFromCrop( + { + cx: Math.min(1, Math.max(0, (event.clientX - bounds.left) / bounds.width)), + cy: Math.min(1, Math.max(0, (event.clientY - bounds.top) / bounds.height)), + }, + cropRegion, + ); + if (sourcePoint) onControlPointChange?.(region.id, index, sourcePoint); + }; + const finishDrag = (event: ReactPointerEvent) => { + if (draggingIndexRef.current === null) return; + draggingIndexRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + onControlPointCommit?.(); + }; + + return ( + + + + {projectedStart ? ( + + ) : null} + {projectedEnd ? ( + + ) : null} + {region.preset !== "recorded" && !isPlaying + ? controls.map((control) => ( + { + event.preventDefault(); + event.stopPropagation(); + draggingIndexRef.current = control.index; + event.currentTarget.setPointerCapture(event.pointerId); + updateControlPoint(event, control.index); + }} + onPointerMove={(event) => { + if (draggingIndexRef.current === control.index) { + updateControlPoint(event, control.index); + } + }} + onPointerUp={finishDrag} + onPointerCancel={finishDrag} + onLostPointerCapture={finishDrag} + /> + )) + : null} + + ); +} + interface NativeCursorFrameInputs { hasNativeCursor: boolean; - recordingData: ReturnType["data"]; - telemetry: ReturnType["samples"]; + recordingData: CursorRecordingData | null; + telemetry: CursorTelemetryPoint[]; timeMs: number; isPlaying: boolean; size: { width: number; height: number }; cursorSize: number; - cursorSmoothing: number; cursorClickBounce: number; cursorMotionBlur: number; cursorTheme: string; cursorClipToBounds: boolean; showCursor: boolean; + cropRegion: CropRegion; + motionPath: CursorMotionPath | null; + cursorMotionRegions: AxcutCursorMotionRegion[]; + owner: CursorMotionOwner | null; imageRef: React.MutableRefObject; clipRef: React.MutableRefObject; lastImageIdRef: React.MutableRefObject; @@ -333,12 +591,15 @@ function renderNativeCursorFrame(inputs: NativeCursorFrameInputs) { isPlaying, size, cursorSize, - cursorSmoothing, cursorClickBounce, cursorMotionBlur, cursorTheme, cursorClipToBounds, showCursor, + cropRegion, + motionPath, + cursorMotionRegions, + owner, imageRef, clipRef, lastImageIdRef, @@ -367,18 +628,27 @@ function renderNativeCursorFrame(inputs: NativeCursorFrameInputs) { hide(); return; } - const path = getSmoothedCursorPath( - { version: 1, provider: "none", samples: samples as never, assets: [] }, - cursorSmoothing, - ); - const pos = path?.sampleAt(timeMs); + const pos = + owner && motionPath + ? sampleCursorMotion({ + path: motionPath, + regions: cursorMotionRegions, + owner, + sourceTimeMs: timeMs, + }) + : motionPath?.sampleAtSourceTime(timeMs); if (!pos || !imageEl) { hide(); return; } + const projectedPosition = projectCursorMotionPointToCrop(pos, cropRegion); + if (!projectedPosition) { + hide(); + return; + } const radius = Math.max(4, 28 * Math.max(0, cursorSize)); - const x = pos.cx * size.width; - const y = pos.cy * size.height; + const x = projectedPosition.cx * size.width; + const y = projectedPosition.cy * size.height; if (lastImageIdRef.current !== "fallback-dot") { imageEl.src = "data:image/svg+xml;utf8," + @@ -400,14 +670,27 @@ function renderNativeCursorFrame(inputs: NativeCursorFrameInputs) { hide(); return; } - const pos = getSmoothedCursorPath(recordingData, cursorSmoothing)?.sampleAt(timeMs); + const pos = + owner && motionPath + ? sampleCursorMotion({ + path: motionPath, + regions: cursorMotionRegions, + owner, + sourceTimeMs: timeMs, + }) + : motionPath?.sampleAtSourceTime(timeMs); const displaySample = pos ? { ...frame.sample, cx: pos.cx, cy: pos.cy } : frame.sample; + const projectedPosition = projectCursorMotionPointToCrop(displaySample, cropRegion); + if (!projectedPosition) { + hide(); + return; + } const renderAsset = resolveNativeCursorRenderAsset(frame.asset, 1, displaySample, cursorTheme); const bounceProgress = getNativeCursorClickBounceProgress(recordingData, timeMs); const scale = Math.max(0, cursorSize) * getNativeCursorClickBounceScale(cursorClickBounce, bounceProgress); - const x = displaySample.cx * size.width; - const y = displaySample.cy * size.height; + const x = projectedPosition.cx * size.width; + const y = projectedPosition.cy * size.height; const motionBlurPx = isPlaying ? getNativeCursorMotionBlurPx({ motionBlur: cursorMotionBlur, diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx index 17aaea7cc..002aabdca 100644 --- a/src/components/ai-edition/ExportDialog.tsx +++ b/src/components/ai-edition/ExportDialog.tsx @@ -18,6 +18,7 @@ import { exportAxcutDocument, } from "@/lib/ai-edition/exporter/documentExporter"; import type { AxcutDocument } from "@/lib/ai-edition/schema"; +import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { type ExportFormat, type ExportProgress, @@ -182,23 +183,44 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { return; } - setPhase("rendering"); - const options: DocumentExportOptions = { - format, - quality, - frameRate: fps, - codec, - gifFrameRate, - gifLoop, - gifSizePreset: gifSize, - // Size the output to the largest clip on the timeline (see referenceSource), - // not just the primary asset, so "Source" matches the shown size. - sourceWidth: referenceSource?.width ?? asset.video?.width, - sourceHeight: referenceSource?.height ?? asset.video?.height, - onProgress: (p) => setProgress(p), - }; - try { + setPhase("rendering"); + const usedAssetIds = new Set(document.timeline.clips.map((clip) => clip.assetId)); + const exportAssets = document.assets.filter( + (candidate) => usedAssetIds.size === 0 || usedAssetIds.has(candidate.id), + ); + const recordingDataByAssetId = new Map( + await Promise.all( + exportAssets.map( + async (candidate) => + [ + candidate.id, + await nativeBridgeClient.cursor.getRecordingData(candidate.originalPath), + ] as const, + ), + ), + ); + const editorSettings = getEditorSettings(document); + const options: DocumentExportOptions = { + format, + quality, + frameRate: fps, + codec, + gifFrameRate, + gifLoop, + gifSizePreset: gifSize, + // Size the output to the largest clip on the timeline (see referenceSource), + // not just the primary asset, so "Source" matches the shown size. + sourceWidth: referenceSource?.width ?? asset.video?.width, + sourceHeight: referenceSource?.height ?? asset.video?.height, + recordingDataByAssetId, + cursorScale: editorSettings.cursorShow ? editorSettings.cursor.size : 0, + cursorSmoothing: editorSettings.cursor.smoothing, + cursorMotionBlur: editorSettings.cursor.motionBlur, + cursorClickBounce: editorSettings.cursor.clickBounce, + cursorClipToBounds: editorSettings.cursor.clipToBounds, + onProgress: (p) => setProgress(p), + }; const result = await exportAxcutDocument(document, options); if (!result.success || !result.blob) { throw new Error(result.error ?? t("exportDialog.exportFailed")); @@ -223,9 +245,6 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { }, }, ); - // Touch the bridge so it stays referenced even when export - // is invoked from a non-Electron shim. - void nativeBridgeClient.aiEdition.llmGetSnapshot; } catch (err) { setError(err instanceof Error ? err.message : String(err)); setPhase("error"); diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index dd698d4f0..297105baa 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -760,6 +760,7 @@ export function NewEditorShell() { const handleCopyRegion = useCallback(async () => { const doc = useProjectStore.getState().document; if (!doc || !tl.selection) return; + if (tl.selection.kind === "cursorMotion") return; const { copyRegion } = await import("@/lib/ai-edition/store/regionClipboard"); const region = tl.selection.kind === "zoom" @@ -1185,6 +1186,12 @@ export function NewEditorShell() { speedRegions={tl.speedRegions} cameraFullscreenRegions={tl.cameraFullscreenRegions} trimRanges={tl.trimRanges} + cursorMotionRegions={tl.cursorMotionRegions} + selectedCursorMotionRegionId={ + tl.selection?.kind === "cursorMotion" ? tl.selection.id : null + } + onCursorMotionControlPointChange={tl.updateCursorMotionControlPointLive} + onCursorMotionControlPointCommit={() => void tl.commitCursorMotionChange()} selectedZoomRegionId={tl.selection?.kind === "zoom" ? tl.selection.id : null} onZoomFocusChange={tl.updateZoomFocusLive} onZoomFocusCommit={() => void tl.commitZoomFocus()} diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index 5e1c75642..79396f0f0 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -3,6 +3,7 @@ import type { CameraFullscreenRegion, ZoomFocus } from "@/components/video-edito import type { AxcutAnnotationRegion, AxcutClip, + AxcutCursorMotionRegion, AxcutTrimRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; @@ -22,6 +23,14 @@ interface PreviewProps { speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; + cursorMotionRegions?: AxcutCursorMotionRegion[]; + selectedCursorMotionRegionId?: string | null; + onCursorMotionControlPointChange?: ( + id: string, + index: number, + point: { cx: number; cy: number }, + ) => void; + onCursorMotionControlPointCommit?: () => void; selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; @@ -54,6 +63,10 @@ export function Preview({ speedRegions, cameraFullscreenRegions, trimRanges, + cursorMotionRegions, + selectedCursorMotionRegionId, + onCursorMotionControlPointChange, + onCursorMotionControlPointCommit, selectedZoomRegionId, onZoomFocusChange, onZoomFocusCommit, @@ -102,6 +115,10 @@ export function Preview({ speedRegions={speedRegions} cameraFullscreenRegions={cameraFullscreenRegions} trimRanges={trimRanges} + cursorMotionRegions={cursorMotionRegions} + selectedCursorMotionRegionId={selectedCursorMotionRegionId} + onCursorMotionControlPointChange={onCursorMotionControlPointChange} + onCursorMotionControlPointCommit={onCursorMotionControlPointCommit} selectedZoomRegionId={selectedZoomRegionId} onZoomFocusChange={onZoomFocusChange} onZoomFocusCommit={onZoomFocusCommit} diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index 6c1d5dec2..0335f8c4e 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -36,6 +36,7 @@ import { computeCameraFullscreenProgress } from "@/components/video-editor/video import type { AxcutAnnotationRegion, AxcutClip, + AxcutCursorMotionRegion, AxcutTrimRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; @@ -70,6 +71,14 @@ interface PreviewCanvasProps { speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; trimRanges?: AxcutTrimRange[]; + cursorMotionRegions?: AxcutCursorMotionRegion[]; + selectedCursorMotionRegionId?: string | null; + onCursorMotionControlPointChange?: ( + id: string, + index: number, + point: { cx: number; cy: number }, + ) => void; + onCursorMotionControlPointCommit?: () => void; selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; diff --git a/src/components/ai-edition/VirtualPreview.test.tsx b/src/components/ai-edition/VirtualPreview.test.tsx new file mode 100644 index 000000000..622d9df0e --- /dev/null +++ b/src/components/ai-edition/VirtualPreview.test.tsx @@ -0,0 +1,182 @@ +import "@testing-library/jest-dom"; +import { cleanup, render, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutClip, AxcutZoomRegion } from "@/lib/ai-edition/schema"; +import { VirtualPreview } from "./VirtualPreview"; + +const mocks = vi.hoisted(() => ({ + computeZoomPreviewTransform: vi.fn(() => ({ + scale: 1, + translateXPercent: 0, + translateYPercent: 0, + })), + getRecordingData: vi.fn(), + getTelemetry: vi.fn(), + cursorLayerProps: vi.fn(), +})); + +vi.mock("@/lib/ai-edition/timeline/zoom-preview", () => ({ + IDENTITY_ZOOM_TRANSFORM: { scale: 1, translateXPercent: 0, translateYPercent: 0 }, + computeZoomPreviewTransform: mocks.computeZoomPreviewTransform, +})); + +vi.mock("@/native/client", () => ({ + nativeBridgeClient: { + cursor: { + getRecordingData: mocks.getRecordingData, + getTelemetry: mocks.getTelemetry, + }, + }, +})); + +vi.mock("@/lib/ai-edition/store/useEditorSettings", () => ({ + useEditorSettings: () => ({ settings: { cursorShow: true } }), +})); + +vi.mock("./CursorPreviewLayer", () => ({ + CursorPreviewLayer: (props: Record) => { + mocks.cursorLayerProps(props); + return
; + }, +})); + +const clips: AxcutClip[] = [ + { + id: "clip-a", + assetId: "asset-a", + sourceStartSec: 5, + sourceEndSec: 6, + timelineStartSec: 0, + timelineEndSec: 1, + wordRefs: [], + origin: "system", + reason: "", + }, + { + id: "clip-b", + assetId: "asset-b", + sourceStartSec: 20, + sourceEndSec: 21, + timelineStartSec: 1, + timelineEndSec: 2, + wordRefs: [], + origin: "system", + reason: "", + }, +]; + +const zoomRegions: AxcutZoomRegion[] = [ + { + id: "zoom-auto", + startMs: 0, + endMs: 2000, + depth: 3, + focus: { cx: 0.5, cy: 0.5 }, + focusMode: "auto", + }, +]; + +describe("VirtualPreview cursor data", () => { + beforeEach(() => { + for (const mock of Object.values(mocks)) mock.mockClear(); + }); + + afterEach(() => cleanup()); + + it("loads each active asset once and never applies the previous asset's telemetry", async () => { + let resolveRecordingB: ((value: unknown) => void) | undefined; + let resolveTelemetryB: ((value: unknown) => void) | undefined; + mocks.getRecordingData.mockImplementation((path: string) => { + if (path.includes("b.mp4")) { + return new Promise((resolve) => { + resolveRecordingB = resolve; + }); + } + return Promise.resolve({ + version: 2, + provider: "native", + assets: [], + samples: [{ timeMs: 5000, cx: 0.2, cy: 0.4 }], + }); + }); + mocks.getTelemetry.mockImplementation((path: string) => { + if (path.includes("b.mp4")) { + return new Promise((resolve) => { + resolveTelemetryB = resolve; + }); + } + return Promise.resolve([{ timeMs: 5000, cx: 0.25, cy: 0.45 }]); + }); + + const sources = [ + { id: "asset-a", src: "file:///a.mp4", label: "A" }, + { id: "asset-b", src: "file:///b.mp4", label: "B" }, + ]; + const view = render( + , + ); + + await waitFor(() => { + expect( + mocks.computeZoomPreviewTransform.mock.calls.some( + (call) => (call[2] as Array<{ cx: number }>)[0]?.cx === 0.2, + ), + ).toBe(true); + }); + const recordingCallsForA = mocks.getRecordingData.mock.calls.filter(([path]) => + String(path).includes("a.mp4"), + ).length; + const telemetryCallsForA = mocks.getTelemetry.mock.calls.filter(([path]) => + String(path).includes("a.mp4"), + ).length; + expect(recordingCallsForA).toBeGreaterThan(0); + expect(telemetryCallsForA).toBeGreaterThan(0); + + view.rerender( + , + ); + + await waitFor(() => { + expect( + mocks.getRecordingData.mock.calls.some(([path]) => String(path).includes("b.mp4")), + ).toBe(true); + expect(mocks.getTelemetry.mock.calls.some(([path]) => String(path).includes("b.mp4"))).toBe( + true, + ); + }); + expect( + mocks.getRecordingData.mock.calls.filter(([path]) => String(path).includes("a.mp4")), + ).toHaveLength(recordingCallsForA); + expect( + mocks.getTelemetry.mock.calls.filter(([path]) => String(path).includes("a.mp4")), + ).toHaveLength(telemetryCallsForA); + await waitFor(() => { + const lastCall = mocks.computeZoomPreviewTransform.mock.calls.at(-1); + expect(lastCall?.[2]).toEqual([]); + }); + + resolveRecordingB?.({ + version: 2, + provider: "native", + assets: [], + samples: [{ timeMs: 20000, cx: 0.8, cy: 0.6 }], + }); + resolveTelemetryB?.([{ timeMs: 20000, cx: 0.75, cy: 0.55 }]); + + await waitFor(() => { + const lastCall = mocks.computeZoomPreviewTransform.mock.calls.at(-1); + expect((lastCall?.[2] as Array<{ cx: number }>)[0]?.cx).toBe(0.8); + }); + expect( + mocks.getRecordingData.mock.calls.filter(([path]) => String(path).includes("a.mp4")), + ).toHaveLength(recordingCallsForA); + expect( + mocks.getTelemetry.mock.calls.filter(([path]) => String(path).includes("a.mp4")), + ).toHaveLength(telemetryCallsForA); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx index 7102b9252..ee24c1f28 100644 --- a/src/components/ai-edition/VirtualPreview.tsx +++ b/src/components/ai-edition/VirtualPreview.tsx @@ -2,10 +2,16 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { fromFileUrl } from "@/components/video-editor/projectPersistence"; import { type CropRegion, + type CursorTelemetryPoint, DEFAULT_CROP_REGION, MAX_NATIVE_PLAYBACK_RATE, } from "@/components/video-editor/types"; -import type { AxcutClip, AxcutTrimRange, AxcutZoomRegion } from "@/lib/ai-edition/schema"; +import type { + AxcutClip, + AxcutCursorMotionRegion, + AxcutTrimRange, + AxcutZoomRegion, +} from "@/lib/ai-edition/schema"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { PlaybackClockRef } from "@/lib/ai-edition/timeline/playback-clock"; import { findActiveSpeedRegion, type SpeedRegion } from "@/lib/ai-edition/timeline/speed"; @@ -19,9 +25,46 @@ import { computeZoomPreviewTransform, IDENTITY_ZOOM_TRANSFORM, } from "@/lib/ai-edition/timeline/zoom-preview"; -import { CursorPreviewLayer } from "./CursorPreviewLayer"; +import { useCursorRecordingData } from "@/native/hooks/useCursorRecordingData"; +import { useCursorTelemetry } from "@/native/hooks/useCursorTelemetry"; +import { CursorPreviewLayer, type CursorPreviewLayerProps } from "./CursorPreviewLayer"; import styles from "./VirtualPreview.module.css"; +const EMPTY_CURSOR_TELEMETRY: CursorTelemetryPoint[] = []; + +interface ActiveCursorLayerProps + extends Omit { + onFocusTelemetryChange: (videoPath: string | null, samples: CursorTelemetryPoint[]) => void; +} + +function ActiveCursorLayer({ onFocusTelemetryChange, ...props }: ActiveCursorLayerProps) { + const { data: cursorRecordingData } = useCursorRecordingData(props.videoPath); + const { samples: cursorTelemetry } = useCursorTelemetry(props.videoPath); + const focusTelemetry = useMemo(() => { + const samples = + cursorRecordingData && cursorRecordingData.samples.length > 0 + ? cursorRecordingData.samples + : cursorTelemetry; + return samples.map((sample) => ({ + timeMs: sample.timeMs, + cx: sample.cx, + cy: sample.cy, + })); + }, [cursorRecordingData, cursorTelemetry]); + + useEffect(() => { + onFocusTelemetryChange(props.videoPath, focusTelemetry); + }, [focusTelemetry, onFocusTelemetryChange, props.videoPath]); + + return ( + + ); +} + export interface VideoSource { id: string; src: string; @@ -34,6 +77,14 @@ interface VirtualPreviewProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; trimRanges?: AxcutTrimRange[]; + cursorMotionRegions?: AxcutCursorMotionRegion[]; + selectedCursorMotionRegionId?: string | null; + onCursorMotionControlPointChange?: ( + id: string, + index: number, + point: { cx: number; cy: number }, + ) => void; + onCursorMotionControlPointCommit?: () => void; seekTarget?: { timeSec: number; isSource?: boolean; requestId: number } | null; onTimeChange?: (timeSec: number) => void; onLoadedMetadata?: ( @@ -63,6 +114,10 @@ export function VirtualPreview({ zoomRegions = [], speedRegions = [], trimRanges = [], + cursorMotionRegions = [], + selectedCursorMotionRegionId, + onCursorMotionControlPointChange, + onCursorMotionControlPointCommit, seekTarget, onTimeChange, onLoadedMetadata, @@ -113,12 +168,31 @@ export function VirtualPreview({ const virtualDurationSec = useMemo(() => totalVirtualDuration(clips), [clips]); const activeSource = videoSources[sourceIndex] ?? null; + const activeVideoPath = activeSource ? fromFileUrl(activeSource.src) : null; + const activeClip = useMemo( + () => locateVirtualPosition(clips, virtualTimeSec)?.clip ?? null, + [clips, virtualTimeSec], + ); // ponytail: the cursor overlay wants source-media time (the recorded // cursor samples live on the original mp4 timeline, not the edited // virtual timeline). `setSourceTimeSec` is called from the 60 Hz rAF // below so the cursor follows the playhead even when the user scrubs. const [sourceTimeSec, setSourceTimeSec] = useState(0); + const [cursorFocusData, setCursorFocusData] = useState<{ + videoPath: string | null; + samples: CursorTelemetryPoint[]; + }>({ videoPath: null, samples: EMPTY_CURSOR_TELEMETRY }); + const handleFocusTelemetryChange = useCallback( + (videoPath: string | null, samples: CursorTelemetryPoint[]) => { + setCursorFocusData({ videoPath, samples }); + }, + [], + ); + const activeCursorTelemetry = + cursorFocusData.videoPath === activeVideoPath + ? cursorFocusData.samples + : EMPTY_CURSOR_TELEMETRY; // Drive the virtual-time preview clock at 60 Hz (the
)} - ) : ( diff --git a/src/components/ai-edition/v4/EditorShellV4.module.css b/src/components/ai-edition/v4/EditorShellV4.module.css index bd02e784d..dacbca94c 100644 --- a/src/components/ai-edition/v4/EditorShellV4.module.css +++ b/src/components/ai-edition/v4/EditorShellV4.module.css @@ -963,6 +963,21 @@ background: var(--surface-3); color: var(--fg); } +.tlToolBtn[aria-pressed="true"] { + background: var(--accent-soft); + color: var(--accent); +} +.tlToolBtnLabeled { + width: auto; + display: inline-flex; + gap: 5px; + padding: 0 8px; +} +.tlToolBtnText { + font-size: 11px; + font-weight: 600; + white-space: nowrap; +} .tlHints { margin-left: auto; display: flex; @@ -1161,6 +1176,10 @@ border-color: var(--accent); background: var(--accent-soft); } +.laneCursorMotion { + border-color: #8b5cf6; + background: rgba(139, 92, 246, 0.14); +} .laneCameraFullscreen { border-color: #a855f7; background: rgba(168, 85, 247, 0.14); diff --git a/src/components/ai-edition/v4/FloatingInspector.test.tsx b/src/components/ai-edition/v4/FloatingInspector.test.tsx new file mode 100644 index 000000000..57f2c0b6d --- /dev/null +++ b/src/components/ai-edition/v4/FloatingInspector.test.tsx @@ -0,0 +1,54 @@ +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { FloatingInspector } from "./FloatingInspector"; + +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); + +vi.mock("@/lib/ai-edition/store/useEditorSettings", () => ({ + useEditorSettings: () => ({ settings: { autoFocusAll: false } }), +})); + +describe("FloatingInspector zoom Auto-Focus", () => { + afterEach(cleanup); + + it("switches the selected zoom between manual and cursor-follow modes", () => { + const updateZoomFocusMode = vi.fn(); + const tl = { + selection: { kind: "zoom", id: "zoom_1" }, + zoomRegions: [ + { + id: "zoom_1", + startMs: 1000, + endMs: 3000, + depth: 3, + focus: { cx: 0.5, cy: 0.5 }, + focusMode: "manual", + }, + ], + updateZoomFocusMode, + clearSelection: vi.fn(), + }; + + render( + , + ); + + fireEvent.change(screen.getByLabelText("zoom.focusMode.title"), { + target: { value: "auto" }, + }); + expect(updateZoomFocusMode).toHaveBeenCalledWith("zoom_1", "auto"); + }); +}); diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index de5cf856d..2d5a6dff2 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -17,6 +17,7 @@ import type { ComponentProps } from "react"; import { useState } from "react"; import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutClip } from "@/lib/ai-edition/schema"; +import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { coalescedTrimGroups } from "@/lib/ai-edition/timeline/trim-mapping"; import { formatSeconds } from "@/lib/ai-edition/timeline/virtual-preview"; @@ -273,12 +274,16 @@ function paneRow(label: string, control: React.ReactNode) { const ZOOM_DEPTHS = [1, 2, 3, 4, 5, 6] as const; const SPEED_VALUES = [0.5, 0.75, 1, 1.25, 1.5, 2, 3]; +const CURSOR_SPEED_VALUES = [1, 1.5, 2, 2.5, 3, 3.5, 4]; +const CURSOR_PRESETS = ["recorded", "straight", "arc", "wave", "loop", "overshoot"] as const; +const CURSOR_EASINGS = ["linear", "ease-in", "ease-out", "ease-in-out"] as const; function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void }) { const ts = useScopedT("settings"); const tt = useScopedT("timeline"); const tc = useScopedT("common"); const te = useScopedT("editor"); + const { settings } = useEditorSettings(); const selection = tl.selection; if (!selection) return null; @@ -314,6 +319,31 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void }
{paneHeader(, tt("labels.zoom"), onClose, tc("actions.close"))}
+ {paneRow( + ts("zoom.focusMode.title"), + , + )} + {region.focusMode === "auto" ? ( +

+ {ts("zoom.focusMode.autoDescription")} +

+ ) : null} + {settings.autoFocusAll ? ( +

+ {ts("zoom.focusMode.lockedDisclaimer")} +

+ ) : null} {paneRow( ts("zoom.level"), + void tl.updateCursorMotionSettings(region.id, { + preset: event.target.value as (typeof CURSOR_PRESETS)[number], + }) + } + style={selectStyle} + > + {CURSOR_PRESETS.map((preset) => ( + + ))} + , + )} + {paneRow( + ts("speed.playbackSpeed"), + , + )} + {paneRow( + "Easing", + , + )} + {region.preset === "wave" || region.preset === "loop" + ? paneRow( + "Cycles", + , + ) + : null} +

+ {region.preset === "recorded" + ? "Recorded keeps the captured cursor unchanged. Choose another motion to edit this segment." + : "Drag the purple control point in the preview to reshape this segment."} +

+ + +
+
+ ); + } + if (selection.kind === "cameraFullscreen") { const region = tl.cameraFullscreenRegions.find((c) => c.id === selection.id); if (!region) return null; diff --git a/src/components/ai-edition/v4/V4Timeline.test.tsx b/src/components/ai-edition/v4/V4Timeline.test.tsx new file mode 100644 index 000000000..7235655d3 --- /dev/null +++ b/src/components/ai-edition/v4/V4Timeline.test.tsx @@ -0,0 +1,251 @@ +import "@testing-library/jest-dom"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { V4Timeline } from "./V4Timeline"; + +const bridgeMocks = vi.hoisted(() => ({ + getRecordingData: vi.fn(), + getTelemetry: vi.fn(), +})); + +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); + +vi.mock("@/hooks/useAudioPeaks", () => ({ useAudioPeaks: () => null })); + +vi.mock("@/native/client", () => ({ + nativeBridgeClient: { + cursor: { + getRecordingData: bridgeMocks.getRecordingData, + getTelemetry: bridgeMocks.getTelemetry, + }, + }, +})); + +vi.mock("@/lib/ai-edition/store/useEditorSettings", () => ({ + useEditorSettings: () => ({ + settings: { + aspectRatio: "16:9", + autoZoomEnabled: true, + autoFocusAll: false, + cursor: { smoothing: 0 }, + }, + set: vi.fn(), + }), +})); + +vi.mock("@/components/ui/popover", () => ({ + Popover: ({ children }: { children: ReactNode }) => children, + PopoverContent: ({ children }: { children: ReactNode }) => children, + PopoverTrigger: ({ children }: { children: ReactNode }) => children, +})); + +vi.mock("../TransportBar", () => ({ TransportBar: () => null })); + +function createTimeline() { + return { + clips: [ + { + id: "clip_1", + assetId: "asset_1", + timelineStartSec: 0, + timelineEndSec: 10, + sourceStartSec: 0, + sourceEndSec: 10, + }, + ], + assets: [{ id: "asset_1", label: "Recording", durationSec: 10 }], + annotationRegions: [], + speedRegions: [], + trimRanges: [], + zoomRegions: [ + { + id: "zoom_1", + startMs: 1000, + endMs: 3000, + depth: 3, + mode: "manual", + focus: { cx: 0.5, cy: 0.5 }, + }, + ], + cursorMotionRegions: [ + { + id: "motion_1", + clipId: "clip_1", + assetId: "asset_1", + startMs: 3000, + endMs: 5000, + sourceStartMs: 3000, + sourceEndMs: 5000, + startPoint: { cx: 0.2, cy: 0.3 }, + endPoint: { cx: 0.8, cy: 0.7 }, + controlPoints: [], + startAnchor: "manual", + endAnchor: "click", + segmentKind: "move", + preset: "arc", + speed: 1, + cycles: 1, + easing: "ease-in-out", + }, + ], + cameraFullscreenRegions: [], + selection: null, + multiSelection: [], + clipSelection: null, + selectRegion: vi.fn(), + clearSelection: vi.fn(), + selectClip: vi.fn(), + addTrim: vi.fn(), + addAnnotation: vi.fn(), + addSpeed: vi.fn(), + addZoom: vi.fn(), + addCameraFullscreen: vi.fn(), + addZoomsBulk: vi.fn(), + completePendingAutoZoom: vi.fn(), + setAutoZoomEnabled: vi.fn(), + setAutoFocusAll: vi.fn(), + addCursorMotionRegions: vi.fn(), + moveClip: vi.fn(), + removeClip: vi.fn(), + setTrimEntries: vi.fn(), + updateAnnotationSpan: vi.fn(), + updateCameraFullscreenSpan: vi.fn(), + updateSpeedSpan: vi.fn(), + updateZoomSpan: vi.fn(), + }; +} + +describe("V4Timeline region selection", () => { + afterEach(() => { + cleanup(); + bridgeMocks.getRecordingData.mockReset(); + bridgeMocks.getTelemetry.mockReset(); + }); + + it("keeps existing zoom pills selectable and selects cursor motion pills", () => { + const tl = createTimeline(); + const setCurrentTime = vi.fn(); + render( + , + ); + + fireEvent.pointerDown(screen.getByTitle("1.80×"), { button: 0, pointerId: 1 }); + expect(tl.selectRegion).toHaveBeenLastCalledWith("zoom", "zoom_1", { additive: false }); + fireEvent.pointerUp(window, { pointerId: 1 }); + + fireEvent.pointerDown(screen.getByTitle("arc"), { button: 0, pointerId: 2 }); + expect(tl.selectRegion).toHaveBeenLastCalledWith("cursorMotion", "motion_1", { + additive: false, + }); + expect(setCurrentTime).toHaveBeenCalledWith(3.001); + }); + + it("keeps auto zoom available while cursor telemetry is loading", async () => { + let resolveRecordingData: (value: null) => void = () => undefined; + bridgeMocks.getRecordingData.mockReturnValue( + new Promise((resolve) => { + resolveRecordingData = resolve; + }), + ); + bridgeMocks.getTelemetry.mockResolvedValue([]); + const tl = createTimeline(); + render( + , + ); + + const cursorButton = screen.getByLabelText("cursorMotion.create"); + fireEvent.click(cursorButton); + await waitFor(() => expect(cursorButton).toBeDisabled()); + expect(screen.getByLabelText("toolbar.autoEnhance")).not.toBeDisabled(); + + await act(async () => resolveRecordingData(null)); + await waitFor(() => expect(cursorButton).not.toBeDisabled()); + }); + + it("restores direct automatic zoom and Auto-Focus controls", () => { + const tl = createTimeline(); + render( + , + ); + + fireEvent.click(screen.getByLabelText("buttons.autoZoomOn")); + expect(tl.setAutoZoomEnabled).toHaveBeenCalledWith(false); + + fireEvent.click(screen.getByLabelText("buttons.autoFocusAllOff")); + expect(tl.setAutoFocusAll).toHaveBeenCalledWith(true); + }); + + it("automatically processes a newly imported pending recording once telemetry is ready", async () => { + bridgeMocks.getTelemetry.mockResolvedValue([ + { timeMs: 1000, cx: 0.25, cy: 0.4 }, + { timeMs: 1600, cx: 0.25, cy: 0.4 }, + ]); + const tl = createTimeline(); + tl.zoomRegions = []; + tl.assets[0] = { ...tl.assets[0], autoZoomState: "pending" } as never; + render( + , + ); + + await waitFor(() => expect(tl.completePendingAutoZoom).toHaveBeenCalledTimes(1)); + expect(tl.completePendingAutoZoom).toHaveBeenCalledWith( + "asset_1", + expect.arrayContaining([ + expect.objectContaining({ focus: expect.objectContaining({ cx: 0.25, cy: 0.4 }) }), + ]), + ); + }); +}); diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index 9df1cc6ab..4c65782d0 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -4,7 +4,9 @@ import { Loader2, Maximize2, MessageSquare, + MousePointer2, Pencil, + ScanEye, Scissors, Sparkles, SplitSquareHorizontal, @@ -27,7 +29,7 @@ import { ZOOM_DEPTH_SCALES } from "@/components/video-editor/types"; import { useScopedT } from "@/contexts/I18nContext"; import { useAudioPeaks } from "@/hooks/useAudioPeaks"; import { createId } from "@/lib/ai-edition/document/ids"; -import type { AxcutClip } from "@/lib/ai-edition/schema"; +import type { AxcutClip, AxcutCursorMotionRegion } from "@/lib/ai-edition/schema"; import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; @@ -37,7 +39,10 @@ import { resolveTimelineSpanToTrim, ventilateTimelineSpanToTrims, } from "@/lib/ai-edition/timeline/trim-mapping"; -import { buildAutoZoomSuggestions } from "@/lib/ai-edition/timeline/zoom-suggestions"; +import { locateVirtualPosition } from "@/lib/ai-edition/timeline/virtual-preview"; +import { buildAutoZoomSuggestionsForClips } from "@/lib/ai-edition/timeline/zoom-suggestions"; +import { buildCursorMotionRegionDrafts } from "@/lib/cursor/cursorMotion"; +import { getSmoothedCursorPath } from "@/lib/cursor/cursorPathSmoothing"; import { nativeBridgeClient } from "@/native/client"; import { ASPECT_RATIOS } from "@/utils/aspectRatioUtils"; import { TransportBar } from "../TransportBar"; @@ -49,6 +54,8 @@ import styles from "./EditorShellV4.module.css"; const AI_ENHANCE_PROMPT = "Automatically enhance this recording: (1) add smart zoom-ins on the moments where the cursor dwells or interacts with the UI, each focused on the cursor's location; and (2) cut the dead time — long pauses, silences, and idle stretches where nothing happens — to keep the pacing tight and natural. Apply the edits directly to the timeline."; +const AUTO_ZOOM_IN_FLIGHT_ASSET_IDS = new Set(); + type TimelineApi = ReturnType; const ASSET_MIME = "application/x-axcut-asset"; @@ -140,7 +147,7 @@ function ClipWaveform({ interface LanePill { id: string; - kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen"; + kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen" | "cursorMotion"; start: number; end: number; label: string; @@ -214,6 +221,7 @@ export function V4Timeline({ const [aspectMenuOpen, setAspectMenuOpen] = useState(false); const [autoEnhanceOpen, setAutoEnhanceOpen] = useState(false); const [autoBusy, setAutoBusy] = useState(false); + const [cursorMotionBusy, setCursorMotionBusy] = useState(false); const clips = tl.clips; const total = useMemo( @@ -227,6 +235,55 @@ export function V4Timeline({ const pctOf = useCallback((sec: number) => (sec / total) * 100, [total]); const showLanes = variant === "edit"; + useEffect(() => { + if (!settings.autoZoomEnabled) return; + for (const asset of tl.assets) { + if (asset.autoZoomState !== "pending" || !asset.durationSec || asset.durationSec <= 0) { + continue; + } + const assetDurationSec = asset.durationSec; + const source = videoSources.find((candidate) => candidate.id === asset.id); + const assetClips = clips.filter( + (clip) => clip.assetId === asset.id && clip.sourceEndSec !== undefined, + ); + if (!source || assetClips.length === 0 || AUTO_ZOOM_IN_FLIGHT_ASSET_IDS.has(asset.id)) { + continue; + } + + AUTO_ZOOM_IN_FLIGHT_ASSET_IDS.add(asset.id); + void (async () => { + try { + const telemetry = + (await nativeBridgeClient.cursor.getTelemetry(fromFileUrl(source.src))) ?? []; + const suggestions = buildAutoZoomSuggestionsForClips({ + cursorTelemetry: telemetry, + clips: assetClips, + existingRegions: tl.zoomRegions.map((region) => ({ + startMs: region.startMs, + endMs: region.endMs, + })), + defaultDurationMs: Math.max(1000, Math.round(assetDurationSec * 50)), + }); + await tl.completePendingAutoZoom(asset.id, suggestions); + } catch (error) { + toast.error(t("toolbar.autoZoomFailed"), { + description: error instanceof Error ? error.message : String(error), + }); + } finally { + AUTO_ZOOM_IN_FLIGHT_ASSET_IDS.delete(asset.id); + } + })(); + } + }, [ + settings.autoZoomEnabled, + tl.assets, + tl.zoomRegions, + tl.completePendingAutoZoom, + clips, + videoSources, + t, + ]); + // ── region lanes ──────────────────────────────────────────────── // zoom/speed/annotation: one pill per row, never coalesced — each carries // distinct per-instance content (depth/focus, speed value, text) that two @@ -266,6 +323,17 @@ export function V4Timeline({ label: `${(z.customScale ?? ZOOM_DEPTH_SCALES[z.depth]).toFixed(2)}×`, sourceIds: [z.id], })); + const cursorMotionPills: LanePill[] = tl.cursorMotionRegions.map((region) => ({ + id: region.id, + kind: "cursorMotion", + start: region.startMs / 1000, + end: region.endMs / 1000, + label: + region.preset === "wave" || region.preset === "loop" + ? `${region.preset} ×${region.cycles}` + : region.preset, + sourceIds: [region.id], + })); // trims: content-free (no per-instance text/settings), so touching rows — // inevitable once a trim is ventilated across a clip boundary — are // coalesced into one pill. This is what makes growing a trim across a @@ -339,6 +407,10 @@ export function V4Timeline({ e.preventDefault(); e.stopPropagation(); tl.selectRegion(pill.kind, pill.id, { additive: e.shiftKey }); + if (pill.kind === "cursorMotion") { + setCurrentTime(pill.start + Math.min(0.001, (pill.end - pill.start) / 2)); + return; + } // Scale drag deltas against the canvas (full zoomed timeline) width, so a // drag tracks the cursor exactly regardless of padding, scrollbar or zoom. const el = canvasRef.current; @@ -425,7 +497,7 @@ export function V4Timeline({ window.addEventListener("pointermove", move); window.addEventListener("pointerup", up); }, - [tl, total, clips], + [tl, total, clips, setCurrentTime], ); const startNavDrag = useCallback( @@ -514,9 +586,11 @@ export function V4Timeline({ ? styles.laneSpeed : kind === "trim" ? styles.laneTrim - : kind === "cameraFullscreen" - ? styles.laneCameraFullscreen - : styles.laneZoom; + : kind === "cursorMotion" + ? styles.laneCursorMotion + : kind === "cameraFullscreen" + ? styles.laneCameraFullscreen + : styles.laneZoom; const pillIcon = (kind: LanePill["kind"]) => kind === "annotation" ? ( @@ -526,6 +600,8 @@ export function V4Timeline({ ) : kind === "cameraFullscreen" ? ( + ) : kind === "cursorMotion" ? ( + ) : ( ); @@ -632,23 +708,25 @@ export function V4Timeline({ ]; // Auto-enhance option 1 — the deterministic cursor-telemetry auto-zoom - // (ported from main; NOT AI). Reads the recorded cursor movement for the - // primary asset and drops zoom-ins on the dwell moments. + // The active clip supplies the asset source-time window; generated regions + // are projected back to the virtual timeline. const runAutoZooms = useCallback(async () => { setAutoEnhanceOpen(false); - const source = videoSources[0]; - const asset = tl.assets.find((a) => a.id === source?.id) ?? tl.assets[0]; - if (!source || !asset) { + const position = locateVirtualPosition(clips, currentTimeSec); + const source = videoSources.find((candidate) => candidate.id === position?.clip.assetId); + const asset = tl.assets.find((candidate) => candidate.id === position?.clip.assetId); + if (!position || !source || !asset) { toast.error(t("toolbar.importRecordingFirst")); return; } setAutoBusy(true); try { + if (!settings.autoZoomEnabled) await tl.setAutoZoomEnabled(true); const telemetry = (await nativeBridgeClient.cursor.getTelemetry(fromFileUrl(source.src))) ?? []; - const suggestions = buildAutoZoomSuggestions({ + const suggestions = buildAutoZoomSuggestionsForClips({ cursorTelemetry: telemetry, - totalMs: (asset.durationSec ?? 0) * 1000, + clips: [position.clip], existingRegions: tl.zoomRegions.map((z) => ({ startMs: z.startMs, endMs: z.endMs })), defaultDurationMs: 2000, }); @@ -669,7 +747,77 @@ export function V4Timeline({ } finally { setAutoBusy(false); } - }, [videoSources, tl, t]); + }, [clips, currentTimeSec, settings.autoZoomEnabled, videoSources, tl, t]); + + const toggleAutoZoom = useCallback(() => { + if (settings.autoZoomEnabled) { + void tl.setAutoZoomEnabled(false); + return; + } + void runAutoZooms(); + }, [settings.autoZoomEnabled, runAutoZooms, tl]); + + const createCursorMotionToNextClick = useCallback(async () => { + const position = locateVirtualPosition(clips, currentTimeSec); + if (!position) { + toast.error(t("cursorMotion.placePlayhead")); + return; + } + const clip = position.clip; + const source = videoSources.find((candidate) => candidate.id === clip.assetId); + const asset = tl.assets.find((candidate) => candidate.id === clip.assetId); + if (!source || !asset) { + toast.error(t("cursorMotion.dataUnavailable")); + return; + } + + setCursorMotionBusy(true); + try { + const videoPath = fromFileUrl(source.src); + const recordingData = await nativeBridgeClient.cursor + .getRecordingData(videoPath) + .catch(() => null); + const telemetry = + recordingData && recordingData.samples.length > 0 + ? recordingData.samples + : ((await nativeBridgeClient.cursor.getTelemetry(videoPath)) ?? []); + const smoothedPath = recordingData + ? getSmoothedCursorPath(recordingData, settings.cursor.smoothing) + : null; + const clipSourceEndMs = + (clip.sourceEndSec ?? asset.durationSec ?? clip.sourceStartSec) * 1000; + const drafts = buildCursorMotionRegionDrafts({ + owner: { clipId: clip.id, assetId: clip.assetId }, + currentSourceTimeMs: position.sourceTimeSec * 1000, + currentVirtualTimeMs: position.virtualTimeSec * 1000, + clipSourceEndMs, + samples: telemetry, + path: smoothedPath + ? { sampleAtSourceTime: (sourceTimeMs) => smoothedPath.sampleAt(sourceTimeMs) } + : null, + }); + if (drafts.length === 0) { + toast.info(t("cursorMotion.noNextClick")); + return; + } + const regions = drafts.map((draft) => ({ + ...draft, + id: createId("cursor-motion"), + })) as AxcutCursorMotionRegion[]; + const added = await tl.addCursorMotionRegions(regions); + toast.success( + t(added === 1 ? "cursorMotion.created" : "cursorMotion.createdPlural", { + count: added, + }), + ); + } catch (error) { + toast.error(t("cursorMotion.createFailed"), { + description: error instanceof Error ? error.message : String(error), + }); + } finally { + setCursorMotionBusy(false); + } + }, [clips, currentTimeSec, settings.cursor.smoothing, videoSources, tl, t]); // Auto-enhance option 2 — hand a generic prompt to the AI agent (smart // zooms + cuts) via the chat prompt-bus. @@ -745,7 +893,7 @@ export function V4Timeline({ onPointerDown={seg.interactive ? (e) => startPillDrag(e, p, "move") : undefined} title={p.label} > - {seg.interactive ? ( + {seg.interactive && p.kind !== "cursorMotion" ? ( {p.label} ) : null} - {seg.interactive ? ( + {seg.interactive && p.kind !== "cursorMotion" ? ( void tl.addZoom()} > + + +