diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 00000000..7eb24102 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,12 @@ +{ + "mcpServers": { + "filmcraft": { + "command": "target/release/filmcraft-cli", + "args": ["mcp", "--bridge", "127.0.0.1:9876"] + }, + "filmcraft-headless": { + "command": "target/release/filmcraft-cli", + "args": ["mcp", "--demo"] + } + } +} diff --git a/Cargo.lock b/Cargo.lock index 9895f4bb..09194c09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -417,6 +417,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bit-set" version = "0.8.0" @@ -618,6 +624,18 @@ dependencies = [ "libc", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + [[package]] name = "clipboard-win" version = "5.4.1" @@ -804,6 +822,40 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.6", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.6", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -858,6 +910,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ecolor" version = "0.36.2" @@ -1163,6 +1221,22 @@ dependencies = [ "serde_json", ] +[[package]] +name = "filmcraft-automation" +version = "0.1.0" +dependencies = [ + "base64", + "filmcraft-engine", + "filmcraft-time", + "image", + "rmcp", + "schemars", + "serde", + "serde_json", + "thiserror 2.0.21", + "tokio", +] + [[package]] name = "filmcraft-bitstream" version = "0.1.0" @@ -1170,6 +1244,19 @@ dependencies = [ "proptest", ] +[[package]] +name = "filmcraft-cli" +version = "0.1.0" +dependencies = [ + "filmcraft-automation", + "filmcraft-codecs", + "filmcraft-engine", + "filmcraft-media", + "filmcraft-time", + "serde_json", + "tokio", +] + [[package]] name = "filmcraft-codecs" version = "0.1.0" @@ -1387,12 +1474,48 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.34" @@ -1423,6 +1546,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + [[package]] name = "futures-task" version = "0.3.34" @@ -1435,9 +1564,13 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -1669,6 +1802,36 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.62.2", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "image" version = "0.25.10" @@ -1706,6 +1869,8 @@ checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -1979,6 +2144,17 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -2524,6 +2700,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "peniko" version = "0.6.1" @@ -2924,6 +3106,26 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "regex-syntax" version = "0.8.11" @@ -2963,6 +3165,41 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rmcp" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fae7019994ae0fe4ada40b732f798f3ff26f0f04facb1477f1bf37eb4f18a2d3" +dependencies = [ + "chrono", + "futures", + "indexmap", + "pastey", + "pin-project-lite", + "rmcp-macros", + "schemars", + "serde", + "serde_json", + "thiserror 2.0.21", + "tokio", + "tokio-util", + "tracing", + "uuid", +] + +[[package]] +name = "rmcp-macros" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e66877136e969d5a2d561ee5841b17717a0619aebd64abb47a2a2f46e34683" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 3.0.6", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -3037,6 +3274,32 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "chrono", + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.6", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3091,6 +3354,17 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "serde_json" version = "1.0.151" @@ -3262,6 +3536,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -3277,6 +3561,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "symphonia" version = "0.5.5" @@ -3539,7 +3829,38 @@ version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ + "bytes", + "libc", + "mio", "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", ] [[package]] @@ -3660,6 +3981,7 @@ version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -3717,6 +4039,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" diff --git a/Cargo.toml b/Cargo.toml index 3554e91f..dc9ce759 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ filmcraft-edit = { path = "crates/edit" } filmcraft-render = { path = "crates/render" } filmcraft-engine = { path = "crates/engine" } filmcraft-ui-egui = { path = "crates/ui-egui" } +filmcraft-automation = { path = "crates/automation" } # foundation thiserror = "2" diff --git a/apps/filmcraft-cli/Cargo.toml b/apps/filmcraft-cli/Cargo.toml new file mode 100644 index 00000000..3fd8641b --- /dev/null +++ b/apps/filmcraft-cli/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "filmcraft-cli" +description = "FilmCraft headless CLI and MCP server" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "filmcraft-cli" +path = "src/main.rs" + +[dependencies] +filmcraft-engine = { workspace = true } +filmcraft-media = { workspace = true } +filmcraft-codecs = { workspace = true } +filmcraft-time = { workspace = true } +filmcraft-automation = { workspace = true } +serde_json = { workspace = true } +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } diff --git a/apps/filmcraft-cli/src/main.rs b/apps/filmcraft-cli/src/main.rs new file mode 100644 index 00000000..9a7d9ce5 --- /dev/null +++ b/apps/filmcraft-cli/src/main.rs @@ -0,0 +1,118 @@ +//! FilmCraft headless CLI. +//! +//! ```text +//! filmcraft-cli probe media info as JSON +//! filmcraft-cli commands [filter] list engine commands +//! filmcraft-cli render [--project p.fcproj|--demo] --seconds S --out frame.png [--scale 0.5] +//! filmcraft-cli run [--project p.fcproj|--demo] run commands (one {"id","params"} per line) +//! filmcraft-cli mcp [--bridge 127.0.0.1:9876] [--demo] MCP server on stdio +//! ``` + +use filmcraft_engine::Session; +use serde_json::{Value, json}; + +fn usage() -> ! { + eprintln!("usage: filmcraft-cli … (see --help in source header)"); + std::process::exit(2) +} + +fn session_from(args: &[String]) -> Session { + let mut s = Session::default(); + if let Some(i) = args.iter().position(|a| a == "--project") { + let p = args.get(i + 1).cloned().unwrap_or_else(|| usage()); + if let Err(e) = s.execute("file.open", json!({"path": p})) { + eprintln!("filmcraft-cli: {e}"); + std::process::exit(1); + } + } else if args.iter().any(|a| a == "--demo") { + let _ = s.execute("file.openDemoProject", json!({})); + } + s +} + +fn opt<'a>(args: &'a [String], k: &str) -> Option<&'a str> { + args.iter().position(|a| a == k).and_then(|i| args.get(i + 1)).map(String::as_str) +} + +#[tokio::main] +async fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let Some(cmd) = args.first() else { usage() }; + match cmd.as_str() { + "probe" => { + let path = args.get(1).unwrap_or_else(|| usage()); + let bytes = std::fs::read(path).unwrap_or_else(|e| { + eprintln!("{path}: {e}"); + std::process::exit(1) + }); + match filmcraft_codecs::open_bytes(path, bytes.into()) { + Ok(src) => println!("{}", serde_json::to_string_pretty(src.info()).unwrap_or_default()), + Err(e) => { + eprintln!("{path}: {e}"); + std::process::exit(1) + } + } + } + "commands" => { + let f = args.get(1).map(|s| s.to_ascii_lowercase()).unwrap_or_default(); + for c in filmcraft_engine::command_specs() { + if f.is_empty() || c.id.to_ascii_lowercase().contains(&f) { + println!("{:<32} {:<34} {:<14} {}", c.id, c.label, c.shortcut.unwrap_or(""), c.params); + } + } + } + "render" => { + let mut s = session_from(&args); + let secs: f64 = opt(&args, "--seconds").and_then(|v| v.parse().ok()).unwrap_or(0.0); + let scale: f32 = opt(&args, "--scale").and_then(|v| v.parse().ok()).unwrap_or(1.0); + let out = opt(&args, "--out").unwrap_or("frame.png"); + s.set_playhead(filmcraft_time::Tick::from_seconds_f64(secs)); + let t0 = std::time::Instant::now(); + let Some(img) = s.render_program(scale) else { + eprintln!("no sequence"); + std::process::exit(1) + }; + let dt = t0.elapsed(); + let png = filmcraft_automation::png_rgba(img.w as u32, img.h as u32, img.over_black_rgba8(), 0).expect("png"); + std::fs::write(out, png).expect("write"); + eprintln!("rendered {}x{} in {:.1} ms → {out}", img.w, img.h, dt.as_secs_f64() * 1000.0); + } + "run" => { + let mut s = session_from(&args); + let script = args.last().unwrap_or_else(|| usage()); + let text = std::fs::read_to_string(script).unwrap_or_else(|e| { + eprintln!("{script}: {e}"); + std::process::exit(1) + }); + for line in text.lines().filter(|l| !l.trim().is_empty() && !l.trim_start().starts_with('#')) { + let v: Value = serde_json::from_str(line).unwrap_or_else(|e| { + eprintln!("bad line `{line}`: {e}"); + std::process::exit(1) + }); + let id = v["id"].as_str().unwrap_or(""); + match s.execute(id, v.get("params").cloned().unwrap_or(json!({}))) { + Ok(r) => println!("{id}: {r}"), + Err(e) => { + eprintln!("{id}: {e}"); + std::process::exit(1) + } + } + } + } + "mcp" => { + let server = match opt(&args, "--bridge") { + Some(addr) => filmcraft_automation::FilmcraftMcp::bridge(addr).unwrap_or_else(|e| { + eprintln!("{e}"); + std::process::exit(1) + }), + None => filmcraft_automation::FilmcraftMcp::headless(session_from(&args)), + }; + if let Err(e) = server.serve_stdio().await { + eprintln!("filmcraft-cli mcp: {e}"); + std::process::exit(1) + } + } + _ => usage(), + } + let _ = filmcraft_media::MediaKind::Movie; +} diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml new file mode 100644 index 00000000..14b090e7 --- /dev/null +++ b/crates/automation/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "filmcraft-automation" +description = "MCP server (rmcp) over the FilmCraft engine, plus a bridge to the running app's control channel" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +filmcraft-engine = { workspace = true } +filmcraft-time = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +rmcp = { version = "3.5", default-features = false, features = ["server", "macros", "transport-io"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-std", "sync", "net", "io-util", "time"] } +base64 = "0.22" +schemars = "1" +image = { version = "0.25", default-features = false, features = ["png"] } diff --git a/crates/automation/src/bridge.rs b/crates/automation/src/bridge.rs new file mode 100644 index 00000000..b6c46bee --- /dev/null +++ b/crates/automation/src/bridge.rs @@ -0,0 +1,74 @@ +//! Client for the desktop app's JSON-lines control protocol. One connection, reconnect on failure. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use serde_json::{Value, json}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::TcpStream; +use tokio::sync::Mutex; + +use crate::AutomationError; + +type Conn = (BufReader, tokio::net::tcp::OwnedWriteHalf); + +pub struct BridgeClient { + addr: String, + conn: Mutex>, + next_id: AtomicU64, +} + +impl BridgeClient { + /// `addr` such as `127.0.0.1:9876` (loopback only). + pub fn new(addr: impl Into) -> Result { + let addr = addr.into(); + let host = addr.rsplit_once(':').map(|(h, _)| h).unwrap_or(&addr); + if !matches!(host, "127.0.0.1" | "localhost" | "[::1]" | "::1") { + return Err(AutomationError::BadRequest(format!("bridge address must be loopback, got `{addr}`"))); + } + Ok(Self { addr, conn: Mutex::new(None), next_id: AtomicU64::new(1) }) + } + + /// Call a control method; returns `result` or the app's error. + pub async fn call(&self, method: &str, params: Value) -> Result { + let mut guard = self.conn.lock().await; + for attempt in 0..2 { + if guard.is_none() { + let s = tokio::time::timeout(Duration::from_secs(5), TcpStream::connect(&self.addr)) + .await + .map_err(|_| AutomationError::Bridge(format!("timed out connecting to {}", self.addr)))? + .map_err(|e| AutomationError::Bridge(format!("cannot connect to {} ({e}); start the app with `filmcraft --control `", self.addr)))?; + let (r, w) = s.into_split(); + *guard = Some((BufReader::new(r), w)); + } + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let conn = guard.as_mut().expect("connected"); + let line = format!("{}\n", json!({"id": id, "method": method, "params": params})); + let res: Result = async { + conn.1.write_all(line.as_bytes()).await.map_err(|e| AutomationError::Bridge(e.to_string()))?; + let mut buf = String::new(); + tokio::time::timeout(Duration::from_secs(90), conn.0.read_line(&mut buf)) + .await + .map_err(|_| AutomationError::Bridge("timeout".into()))? + .map_err(|e| AutomationError::Bridge(e.to_string()))?; + serde_json::from_str(&buf).map_err(|e| AutomationError::Bridge(format!("bad reply: {e}"))) + } + .await; + match res { + Ok(v) => { + return if v.get("ok").and_then(Value::as_bool) == Some(true) { + Ok(v.get("result").cloned().unwrap_or(Value::Null)) + } else { + Err(AutomationError::App(v.get("error").and_then(Value::as_str).unwrap_or("error").to_string())) + }; + } + Err(e) if attempt == 0 => { + *guard = None; + let _ = e; + } + Err(e) => return Err(e), + } + } + Err(AutomationError::Bridge("unreachable".into())) + } +} diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs new file mode 100644 index 00000000..76428c9e --- /dev/null +++ b/crates/automation/src/lib.rs @@ -0,0 +1,41 @@ +//! # filmcraft-automation +//! +//! The FilmCraft MCP server (official `rmcp` SDK, stdio transport). +//! +//! * **Headless** mode drives an in-process [`filmcraft_engine::Session`]: every engine command, +//! project/sequence inspection and frame rendering (PNG) — no window needed. +//! * **Bridge** mode forwards to a running desktop app over the JSON-lines control channel +//! (`filmcraft --control `), so agents can also inspect, screenshot, click, drag, scroll +//! and type in the live UI — every menu, panel, timeline gesture and keystroke. + +pub mod bridge; +pub mod server; + +pub use bridge::BridgeClient; +pub use server::FilmcraftMcp; + +#[derive(Debug, thiserror::Error)] +pub enum AutomationError { + #[error("{0}")] + BadRequest(String), + #[error(transparent)] + Engine(#[from] filmcraft_engine::EngineError), + #[error("bridge: {0}")] + Bridge(String), + #[error("app: {0}")] + App(String), + #[error("{0}")] + Other(String), +} + +/// Encode RGBA8 as PNG, downscaled so the longest side is at most `max_side` (0 = no limit). +pub fn png_rgba(w: u32, h: u32, rgba: Vec, max_side: u32) -> Result, AutomationError> { + let mut img = image::RgbaImage::from_raw(w, h, rgba).ok_or_else(|| AutomationError::Other("bad image".into()))?; + if max_side > 0 && w.max(h) > max_side { + let s = max_side as f32 / w.max(h) as f32; + img = image::imageops::resize(&img, ((w as f32 * s) as u32).max(1), ((h as f32 * s) as u32).max(1), image::imageops::FilterType::Triangle); + } + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, image::ImageFormat::Png).map_err(|e| AutomationError::Other(e.to_string()))?; + Ok(out.into_inner()) +} diff --git a/crates/automation/src/server.rs b/crates/automation/src/server.rs new file mode 100644 index 00000000..816f6606 --- /dev/null +++ b/crates/automation/src/server.rs @@ -0,0 +1,374 @@ +//! The MCP server. Tool names are snake_case. + +use std::sync::{Arc, Mutex}; + +use base64::Engine as _; +use filmcraft_engine::Session; +use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::{CallToolResult, ContentBlock as Content}; +use rmcp::{ErrorData as McpError, ServerHandler, ServiceExt, tool, tool_handler, tool_router}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::{AutomationError, BridgeClient, png_rgba}; + +pub enum Backend { + Headless(Arc>), + Bridge(Arc), +} + +#[derive(Clone)] +pub struct FilmcraftMcp { + backend: Arc, + tool_router: ToolRouter, +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct ListParams { + /// Only commands whose id or label contains this text (case-insensitive). + #[serde(default)] + pub filter: Option, + /// Only commands that can run right now. + #[serde(default)] + pub enabled_only: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct RunParams { + /// Command id, e.g. `sequence.addEdit`, `timeline.trim`, `effects.apply`, `file.import`. + pub id: String, + /// Parameters object (see `params` in `command_list`). + #[serde(default)] + pub params: Option, +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct RenderParams { + /// Timeline time in seconds (default: the playhead). + #[serde(default)] + pub seconds: Option, + /// Longest side of the PNG (default 960). + #[serde(default)] + pub max_side: Option, +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct ShotParams { + /// Panel to crop to (e.g. `Timeline`, `Program`); omit for the whole window. + #[serde(default)] + pub panel: Option, + /// Longest side of the PNG (default 1600). + #[serde(default)] + pub max_side: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ClickParams { + /// Element id from `ui_elements` (e.g. `timeline.track.V1.lock`, `tools.Razor`, `project.item.12`). + #[serde(default)] + pub id: Option, + /// Or screen coordinates in points. + #[serde(default)] + pub x: Option, + #[serde(default)] + pub y: Option, + /// left | right | middle + #[serde(default)] + pub button: Option, + /// 2 for double-click. + #[serde(default)] + pub count: Option, + /// {"shift":bool,"alt":bool,"command":bool,"ctrl":bool} + #[serde(default)] + pub modifiers: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct DragParams { + /// Start: {"id":..} or {"x":..,"y":..} (optionally "fx"/"fy" fractions inside the element). + pub from: Value, + /// End point, same forms. + pub to: Value, + #[serde(default)] + pub steps: Option, + #[serde(default)] + pub modifiers: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct KeyParams { + /// Key with optional modifiers, e.g. `Space`, `Cmd+K`, `Shift+Delete`, `I`, `L`. + pub key: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct TypeParams { + pub text: String, +} + +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct ElementsParams { + /// Only element ids starting with this prefix (e.g. `timeline.clip.`). + #[serde(default)] + pub prefix: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ControlParams { + /// Any control-channel method (e.g. `ui.set`, `ui.scroll`, `ui.timeline.hit`, `ui.playback`). + pub method: String, + #[serde(default)] + pub params: Option, +} + +fn ok_json(v: &Value) -> CallToolResult { + CallToolResult::success(vec![Content::text(serde_json::to_string_pretty(v).unwrap_or_default())]) +} +fn fail(e: impl std::fmt::Display) -> CallToolResult { + CallToolResult::error(vec![Content::text(e.to_string())]) +} +fn png_result(png: &[u8], note: &str) -> CallToolResult { + CallToolResult::success(vec![Content::image(base64::engine::general_purpose::STANDARD.encode(png), "image/png"), Content::text(note.to_string())]) +} +fn wrap(r: Result) -> Result { + Ok(match r { + Ok(v) => ok_json(&v), + Err(e) => fail(e), + }) +} + +impl FilmcraftMcp { + pub fn headless(session: Session) -> Self { + Self { backend: Arc::new(Backend::Headless(Arc::new(Mutex::new(session)))), tool_router: Self::tool_router() } + } + pub fn bridge(addr: &str) -> Result { + Ok(Self { backend: Arc::new(Backend::Bridge(Arc::new(BridgeClient::new(addr)?))), tool_router: Self::tool_router() }) + } + + pub async fn serve_stdio(self) -> Result<(), AutomationError> { + let running = self.serve(rmcp::transport::stdio()).await.map_err(|e| AutomationError::Other(format!("MCP init: {e}")))?; + running.waiting().await.map_err(|e| AutomationError::Other(e.to_string()))?; + Ok(()) + } + + /// Run an engine command on whichever backend. + pub async fn run(&self, id: &str, params: Value) -> Result { + match &*self.backend { + Backend::Headless(s) => { + let s = s.clone(); + let id = id.to_string(); + tokio::task::spawn_blocking(move || { + let mut g = s.lock().map_err(|_| AutomationError::Other("session lock poisoned".into()))?; + g.execute(&id, params).map_err(AutomationError::from) + }) + .await + .map_err(|e| AutomationError::Other(e.to_string()))? + } + Backend::Bridge(b) => b.call("engine.execute", json!({"command": id, "params": params})).await, + } + } + + fn bridge_client(&self) -> Option> { + match &*self.backend { + Backend::Bridge(b) => Some(b.clone()), + Backend::Headless(_) => None, + } + } + + async fn ui(&self, method: &str, params: Value) -> Result { + match self.bridge_client() { + Some(b) => wrap(b.call(method, params).await), + None => Ok(fail( + "this tool drives the live app: run `filmcraft-cli mcp --bridge 127.0.0.1:` with the app started as `filmcraft --control `", + )), + } + } +} + +#[tool_router] +impl FilmcraftMcp { + #[tool( + description = "List every command (id, label, menu path, shortcut, parameter doc, enabled now). Menus, shortcuts, panels and timeline gestures all map to these ids." + )] + async fn command_list(&self, Parameters(p): Parameters) -> Result { + let r = self.run("command.list", json!({})).await.map(|v| { + let f = p.filter.unwrap_or_default().to_ascii_lowercase(); + let only = p.enabled_only.unwrap_or(false); + Value::Array( + v.as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .filter(|c| { + let id = c["id"].as_str().unwrap_or("").to_ascii_lowercase(); + let label = c["label"].as_str().unwrap_or("").to_ascii_lowercase(); + (f.is_empty() || id.contains(&f) || label.contains(&f)) && (!only || c["enabled"].as_bool() == Some(true)) + }) + .collect(), + ) + }); + wrap(r) + } + + #[tool( + description = "Run a command by id with JSON params, e.g. {\"id\":\"timeline.razor\",\"params\":{\"seconds\":3.5}} or {\"id\":\"effects.apply\",\"params\":{\"effect\":\"Gaussian Blur\"}}. Undoable edits land in History." + )] + async fn command_run(&self, Parameters(p): Parameters) -> Result { + wrap(self.run(&p.id, p.params.unwrap_or(json!({}))).await) + } + + #[tool(description = "Project tree (bins and items with ids, types, durations), active sequence, source clip.")] + async fn project_inspect(&self) -> Result { + wrap(self.run("project.inspect", json!({})).await) + } + + #[tool( + description = "The active sequence as JSON: settings, tracks, clips (ids, start/duration in ticks and frames, effects), transitions, markers, in/out, playhead, selection. 254016000000 ticks = 1 second." + )] + async fn sequence_inspect(&self) -> Result { + wrap(self.run("sequence.inspect", json!({})).await) + } + + #[tool(description = "Import media files by absolute path (MP4/MOV, WAV/MP3/FLAC/AIFF/Ogg, PNG/JPEG/…).")] + async fn media_import(&self, Parameters(p): Parameters) -> Result { + let paths: Vec<&str> = p.text.split('\n').map(str::trim).filter(|s| !s.is_empty()).collect(); + wrap(self.run("file.import", json!({"paths": paths})).await) + } + + #[tool(description = "Render the active sequence's frame (at `seconds` or the playhead) and return it as PNG.")] + async fn render_frame(&self, Parameters(p): Parameters) -> Result { + let max = p.max_side.unwrap_or(960); + match &*self.backend { + Backend::Headless(s) => { + let s = s.clone(); + let r = tokio::task::spawn_blocking(move || -> Result, AutomationError> { + let mut g = s.lock().map_err(|_| AutomationError::Other("lock".into()))?; + if let Some(sec) = p.seconds { + g.set_playhead(filmcraft_time::Tick::from_seconds_f64(sec)); + } + let (w, h) = + g.active_sequence().map(|q| (q.settings.width, q.settings.height)).ok_or_else(|| AutomationError::Other("no sequence".into()))?; + let scale = (max as f32 / w.max(h) as f32).min(1.0); + let img = g.render_program(scale).ok_or_else(|| AutomationError::Other("no sequence".into()))?; + png_rgba(img.w as u32, img.h as u32, img.over_black_rgba8(), max) + }) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None))?; + Ok(match r { + Ok(png) => png_result(&png, "rendered frame"), + Err(e) => fail(e), + }) + } + Backend::Bridge(b) => { + if let Some(sec) = p.seconds + && let Err(e) = b.call("engine.execute", json!({"command": "playhead.set", "params": {"seconds": sec}})).await + { + return Ok(fail(e)); + } + self.screenshot(&b.clone(), Some("Program".into()), max).await + } + } + } + + #[tool(description = "Live app: UI state (tool, workspace, panels, timeline zoom, playback, selection, fps).")] + async fn ui_inspect(&self) -> Result { + self.ui("ui.inspect", json!({})).await + } + + #[tool(description = "Live app: every on-screen interactive element with id, label and rect (points). Use ids with ui_click/ui_drag.")] + async fn ui_elements(&self, Parameters(p): Parameters) -> Result { + self.ui("ui.elements", json!({"prefix": p.prefix.unwrap_or_default()})).await + } + + #[tool(description = "Live app: click an element by id or at x,y (points). Supports right/middle button, double-click (count=2) and modifiers.")] + async fn ui_click(&self, Parameters(p): Parameters) -> Result { + let mut v = json!({"button": p.button, "count": p.count, "modifiers": p.modifiers}); + if let Some(id) = p.id { + v["id"] = json!(id); + } else { + v["x"] = json!(p.x); + v["y"] = json!(p.y); + } + self.ui("ui.click", v).await + } + + #[tool( + description = "Live app: press-drag-release from one point/element to another (move clips, trim edges, scrub, resize panels, drop project items onto tracks)." + )] + async fn ui_drag(&self, Parameters(p): Parameters) -> Result { + self.ui("ui.drag", json!({"from": p.from, "to": p.to, "steps": p.steps, "modifiers": p.modifiers})).await + } + + #[tool(description = "Live app: press a key or shortcut (e.g. `Space`, `L`, `Cmd+K`, `Shift+Delete`, `Cmd+Z`).")] + async fn ui_key(&self, Parameters(p): Parameters) -> Result { + self.ui("ui.key", json!({"key": p.key})).await + } + + #[tool(description = "Live app: type text into the focused field.")] + async fn ui_type(&self, Parameters(p): Parameters) -> Result { + self.ui("ui.type", json!({"text": p.text})).await + } + + #[tool(description = "Live app: screenshot of the window or one panel (PNG).")] + async fn ui_screenshot(&self, Parameters(p): Parameters) -> Result { + match self.bridge_client() { + Some(b) => self.screenshot(&b, p.panel, p.max_side.unwrap_or(1600)).await, + None => Ok(fail("ui_screenshot needs bridge mode")), + } + } + + #[tool( + description = "Live app: call any control-channel method directly (ui.set, ui.scroll, ui.timeline.hit, ui.timeline.locate, ui.playback, ui.panel.show, ui.menu.list, …)." + )] + async fn ui_control(&self, Parameters(p): Parameters) -> Result { + self.ui(&p.method, p.params.unwrap_or(json!({}))).await + } +} + +impl FilmcraftMcp { + async fn screenshot(&self, b: &BridgeClient, panel: Option, max: u32) -> Result { + let path = std::env::temp_dir().join(format!("filmcraft-mcp-{}.png", std::process::id())); + if let Err(e) = b.call("ui.screenshot", json!({"path": path.to_string_lossy(), "panel": panel})).await { + return Ok(fail(e)); + } + let bytes = match std::fs::read(&path) { + Ok(b) => b, + Err(e) => return Ok(fail(format!("screenshot: {e}"))), + }; + let _ = std::fs::remove_file(&path); + let bytes = match image::load_from_memory(&bytes) { + Ok(img) => { + let rgba = img.to_rgba8(); + let (w, h) = rgba.dimensions(); + png_rgba(w, h, rgba.into_raw(), max).unwrap_or(bytes) + } + Err(_) => bytes, + }; + Ok(png_result(&bytes, "screenshot of the live FilmCraft window")) + } +} + +#[tool_handler( + router = self.tool_router, + name = "filmcraft", + instructions = "FilmCraft video editor (Premiere Pro-class). Every edit is an engine command: `command_list` to discover ids/params, `command_run` to execute (undoable). `project_inspect`/`sequence_inspect` return ids you can pass to commands; `render_frame` shows the result. In bridge mode the `ui_*` tools drive the live app: `ui_elements` lists clickable ids, `ui_click`/`ui_drag`/`ui_key` operate it, `ui_screenshot` shows it. Time is in ticks: 254016000000 per second (commands also accept `seconds`, `frame` or `timecode`)." +)] +impl ServerHandler for FilmcraftMcp {} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn headless_commands_and_render() { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let m = FilmcraftMcp::headless(s); + let v = m.run("sequence.addEditAllTracks", json!({"seconds": 2.0})).await.unwrap(); + assert!(v["cuts"].as_u64().unwrap() >= 2); + let r = m.render_frame(Parameters(RenderParams { seconds: Some(1.0), max_side: Some(320) })).await.unwrap(); + assert_ne!(r.is_error, Some(true)); + } +} diff --git a/docs/control-protocol.md b/docs/control-protocol.md new file mode 100644 index 00000000..47bb92c1 --- /dev/null +++ b/docs/control-protocol.md @@ -0,0 +1,40 @@ +# Control protocol & MCP + +## Desktop control channel +`filmcraft --control 9876` (or `FILMCRAFT_CONTROL_PORT`) listens on `127.0.0.1:` (loopback only). +One JSON request per line → one JSON reply per line: + +```json +{"id": 1, "method": "engine.execute", "params": {"command": "sequence.addEdit", "params": {"seconds": 3}}} +{"id": 1, "ok": true, "result": {"cuts": 2}} +``` + +Methods (handlers in `crates/ui-egui/src/control.rs`): + +| Method | Params | | +|---|---|---| +| `engine.execute` / `ui.menu.invoke` | `{command, params}` | any engine or UI command id | +| `engine.commands` / `ui.menu.list` | – | command registry / menu tree | +| `ui.inspect` | – | UI state (tool, workspace, dock, timeline view, playback, fps…) | +| `ui.elements` | `{prefix?}` | every on-screen interactive element: id, label, rect | +| `ui.set` | `{tool, workspace, mode, theme, focused, playbackRes, timeline:{pps,scroll,fit}}` | | +| `ui.panel.show` / `ui.panel.close` | `{panel}` | | +| `ui.click` / `ui.move` | `{id}` or `{x,y}`, `button`, `count`, `modifiers` | synthetic pointer input | +| `ui.drag` | `{from, to, steps, modifiers}` | press–move–release | +| `ui.scroll` | `{id|x,y, dx, dy, modifiers}` | wheel / trackpad | +| `ui.key` / `ui.type` | `{key}` (`Cmd+K`, `Space`…) / `{text}` | keyboard | +| `ui.timeline.hit` / `ui.timeline.locate` | `{x,y}` / `{clip, edge?}` | timeline hit-testing | +| `ui.playback` | `{action: play|stop|toggle, speed?}` | | +| `ui.screenshot` | `{path?, panel?}` | PNG of the window or one panel | +| `ui.resize`, `ui.focus`, `app.quit` | | | + +Element ids are stable, e.g. `timeline.clip.`, `timeline.track.V1.lock`, `tools.Razor`, +`project.item.`, `effects.item.gaussian_blur`, `panel.tab.Timeline`, `program.transport.playback.toggle`. + +## MCP +`filmcraft-cli mcp` serves MCP on stdio: headless (in-process session; `--demo` / `--project p.fcproj`) +or `--bridge 127.0.0.1:9876` to drive the running app. Tools: `command_list`, `command_run`, +`project_inspect`, `sequence_inspect`, `media_import`, `render_frame`, `ui_inspect`, `ui_elements`, +`ui_click`, `ui_drag`, `ui_key`, `ui_type`, `ui_screenshot`, `ui_control`. `.mcp.json` registers both. + +Time is in ticks (254 016 000 000 per second); commands also accept `seconds`, `frame` or `timecode`.