diff --git a/Cargo.lock b/Cargo.lock index c9d44f3..dd4aa3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -781,6 +781,7 @@ dependencies = [ name = "flowproof-agent" version = "0.12.2" dependencies = [ + "base64", "chrono", "flowproof-driver", "flowproof-trace", diff --git a/Cargo.toml b/Cargo.toml index 0aaa02c..cc47f36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ flowproof-adapters = { path = "crates/flowproof-adapters" } flowproof-cli = { path = "crates/flowproof-cli" } anyhow = "1" +base64 = "0.22" chrono = { version = "0.4", default-features = false, features = ["clock"] } clap = { version = "4", features = ["derive"] } serde = { version = "1", features = ["derive"] } diff --git a/crates/flowproof-agent/Cargo.toml b/crates/flowproof-agent/Cargo.toml index d20f0f0..6f0781d 100644 --- a/crates/flowproof-agent/Cargo.toml +++ b/crates/flowproof-agent/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true authors.workspace = true [dependencies] +base64 = { workspace = true } chrono = { workspace = true } flowproof-driver = { workspace = true, features = ["oob"] } flowproof-trace = { workspace = true } diff --git a/crates/flowproof-agent/src/lib.rs b/crates/flowproof-agent/src/lib.rs index 87b1158..febfd06 100644 --- a/crates/flowproof-agent/src/lib.rs +++ b/crates/flowproof-agent/src/lib.rs @@ -12,6 +12,7 @@ pub mod llm; pub mod recorder; pub mod rules; pub mod spec; +pub mod video_author; pub use clarify::{Clarification, ClarifyStage}; pub use heal::{heal, heal_with_author, HealError, HealReport}; diff --git a/crates/flowproof-agent/src/video_author.rs b/crates/flowproof-agent/src/video_author.rs new file mode 100644 index 0000000..69c1713 --- /dev/null +++ b/crates/flowproof-agent/src/video_author.rs @@ -0,0 +1,699 @@ +//! SPIKE: draft `.flow.yaml` authoring from a screen-recording video. +//! +//! docs/recording.md and issue #227 both say video should never be a +//! machine-parsed input - video is the human surface, not the machine +//! surface. This exists anyway, as a scoped experiment: extract key +//! frames, ask a vision model what single action bridges each pair (in +//! flowproof's existing step vocabulary), and write a DRAFT. It never +//! infers `assert:` steps - a recording shows behaviour, not a belief +//! about correctness - so a human adds at least one before the draft +//! can survive the live `flowproof record` pass every spec still needs. +//! +//! A transition video cannot explain (most often an invisible keypress +//! like Enter) is never silently dropped and never guessed at either - +//! it becomes a flagged, freeform step (see [`FLAGGED_MARKER`]) that the +//! *existing* live authoring agent resolves against the real screen +//! during that same `record` pass, the same way it already resolves any +//! other freeform step. Two OS-level ways to detect the keypress +//! directly instead (`GetAsyncKeyState` polling, then a `WH_KEYBOARD_LL` +//! hook — see [`flowproof_driver::input_log`]) were tried first and both +//! failed to see a real physical keystroke in a UTM VM, apparently +//! because that hypervisor's virtual keyboard bypasses the input layer +//! both depend on; [`record_session`] and [`flowproof_driver::input_log`] +//! stay in as a +//! best-effort enhancement for real hardware, where they should work. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use base64::Engine; +use serde_json::json; + +use crate::{AgentError, BackendConfig, BackendKind, FlowSpec}; + +#[derive(Debug, thiserror::Error)] +pub enum VideoAuthorError { + #[error("ffmpeg not found on PATH: install ffmpeg to extract frames from a video")] + FfmpegMissing, + #[error("ffmpeg failed extracting frames: {0}")] + FfmpegFailed(String), + #[error("no steps were inferred from the video's frames — nothing to draft")] + NoStepsInferred, + #[error("key event capture failed: {0}")] + KeyCapture(#[from] flowproof_driver::DriverError), + #[error("model produced a draft that does not parse as a flow spec: {0}")] + DraftInvalid(#[from] crate::spec::SpecError), + #[error(transparent)] + Agent(#[from] AgentError), + #[error("io error at '{path}': {source}")] + Io { + path: String, + source: std::io::Error, + }, +} + +fn io_err(path: &Path, source: std::io::Error) -> VideoAuthorError { + VideoAuthorError::Io { + path: path.display().to_string(), + source, + } +} + +/// One frame every `interval_ms` from `video` into `out_dir`, via a +/// system `ffmpeg` (never bundled, never run in CI — local only). +pub fn extract_keyframes( + video: &Path, + interval_ms: u64, + out_dir: &Path, +) -> Result, VideoAuthorError> { + check_ffmpeg_available()?; + std::fs::create_dir_all(out_dir).map_err(|e| io_err(out_dir, e))?; + + let fps = 1000.0 / interval_ms.max(1) as f64; + let pattern = out_dir.join("frame_%05d.png"); + let output = Command::new("ffmpeg") + .arg("-y") + .arg("-i") + .arg(video) + .args(["-vf", &format!("fps={fps}")]) + .arg(&pattern) + .output() + .map_err(|e| VideoAuthorError::FfmpegFailed(e.to_string()))?; + if !output.status.success() { + return Err(VideoAuthorError::FfmpegFailed( + String::from_utf8_lossy(&output.stderr).trim().to_string(), + )); + } + + // Empty (too-short video) is not an error here; assemble_draft_spec + // refuses an empty result downstream, with one message for that. + let mut frames: Vec = std::fs::read_dir(out_dir) + .map_err(|e| io_err(out_dir, e))? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("png")) + .collect(); + frames.sort(); + Ok(frames) +} + +/// Record a fresh screen-and-keypress session for `seconds`, writing the +/// video to `out_video`. The two capture concurrently (ffmpeg as a +/// spawned child process, key polling on this thread), so a keypress's +/// logged offset lines up with the same instant in the video. Requires +/// ffmpeg on PATH; only ever runs locally, interactively — never in CI. +pub fn record_session( + out_video: &Path, + seconds: u64, +) -> Result, VideoAuthorError> { + check_ffmpeg_available()?; + let mut child = Command::new("ffmpeg") + .arg("-y") + .args(["-f", "gdigrab"]) + .args(["-framerate", "10"]) + .args(["-i", "desktop"]) + .args(["-t", &seconds.to_string()]) + .arg(out_video) + .spawn() + .map_err(|e| VideoAuthorError::FfmpegFailed(e.to_string()))?; + let events = flowproof_driver::input_log::capture_for(Duration::from_secs(seconds))?; + let status = child + .wait() + .map_err(|e| VideoAuthorError::FfmpegFailed(e.to_string()))?; + if !status.success() { + return Err(VideoAuthorError::FfmpegFailed(format!( + "ffmpeg exited with {status}" + ))); + } + Ok(events) +} + +fn check_ffmpeg_available() -> Result<(), VideoAuthorError> { + Command::new("ffmpeg") + .arg("-version") + .output() + .map(|_| ()) + .map_err(|_| VideoAuthorError::FfmpegMissing) +} + +const TRANSITION_SYSTEM_PROMPT: &str = "\ +Two frames from a screen recording: FIRST is before an action, SECOND is \ +after. You also see the flow's steps already inferred so far, in order. + +Describe the action(s) that bridge FIRST to SECOND, each on its own \ +line, using ONLY these forms: +- \"Go to /nVA01\" (a transaction code in the command field) +- \"Type into the