diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index f89c0e87..d52b19c7 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -36,6 +36,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/basic-shapes.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (basic shapes, groups, transforms) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/paths-gradients.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (path data, gradients, CSS, use) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/quad.gltf` | EffectCraft contributors | Original work: glTF 2.0 test fixture (textured quad, node hierarchy, translation animation; data URIs) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/quad.glb` | EffectCraft contributors | Original work: binary glTF test fixture (the same quad with its texture in the BIN chunk) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/cube.obj` | EffectCraft contributors | Original work: Wavefront OBJ test fixture (unit cube, two materials) | MIT OR Apache-2.0 | diff --git a/Cargo.lock b/Cargo.lock index 0d3ae06b..c7934100 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1214,12 +1214,15 @@ dependencies = [ "effectcraft-model", "effectcraft-path", "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-text", "effectcraft-time", "effectcraft-track", "image", + "kurbo", "log", "rayon", "serde", @@ -1234,14 +1237,18 @@ version = "0.1.0" dependencies = [ "effectcraft-color", "effectcraft-media", + "effectcraft-opusenc", "effectcraft-project", "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "effectcraft-vp9enc", "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", + "filmcraft-opus", "filmcraft-prores", + "filmcraft-vp9", "gif", "image", "log", @@ -1303,7 +1310,11 @@ dependencies = [ "effectcraft-expr", "effectcraft-media", "effectcraft-project", + "effectcraft-psd", + "effectcraft-raster", "effectcraft-script", + "effectcraft-svg", + "effectcraft-time", "serde_json", ] @@ -1342,8 +1353,10 @@ version = "0.1.0" dependencies = [ "effectcraft-model", "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-time", "exr", "filmcraft-codecs", @@ -1369,6 +1382,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-opusenc" +version = "0.1.0" + [[package]] name = "effectcraft-path" version = "0.1.0" @@ -1394,6 +1411,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-psd" +version = "0.1.0" +dependencies = [ + "miniz_oxide 0.8.9", +] + [[package]] name = "effectcraft-raster" version = "0.1.0" @@ -1419,6 +1443,7 @@ dependencies = [ "effectcraft-time", "kurbo", "log", + "lz4_flex", "rayon", "serde", "web-time", @@ -1439,6 +1464,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "effectcraft-svg" +version = "0.1.0" +dependencies = [ + "effectcraft-geom", + "effectcraft-path", + "effectcraft-raster", + "kurbo", + "rayon", + "roxmltree", +] + [[package]] name = "effectcraft-text" version = "0.1.0" @@ -1467,6 +1504,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] @@ -1497,6 +1535,10 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-vp9enc" +version = "0.1.0" + [[package]] name = "effectcraft-web" version = "0.1.0" @@ -1511,10 +1553,12 @@ dependencies = [ "egui", "js-sys", "log", + "serde", "serde_json", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "web-time", ] [[package]] @@ -2807,6 +2851,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + [[package]] name = "mach2" version = "0.4.3" @@ -4047,6 +4097,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 23fb4f29..22709f60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,14 @@ effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } +effectcraft-psd = { path = "crates/psd" } +effectcraft-vp9enc = { path = "crates/vp9enc" } +effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } +effectcraft-opusenc = { path = "crates/opusenc" } effectcraft-script = { path = "crates/script" } # foundation @@ -69,6 +73,9 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "89c filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -132,9 +139,15 @@ opt-level = 3 [profile.dev.package.effectcraft-media] opt-level = 3 +[profile.dev.package.effectcraft-opusenc] +opt-level = 3 + [profile.dev.package.effectcraft-export] opt-level = 3 +[profile.dev.package.effectcraft-vp9enc] +opt-level = 3 + # FilmCraft codec hot loops (same settings as FilmCraft's own workspace). [profile.dev.package.filmcraft-h264] opt-level = 3 diff --git a/ROADMAP.md b/ROADMAP.md index 55b05822..7c7b337b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -20,18 +20,17 @@ once. | M11 | Animation tools: audio playback, meters and waveforms, Lottie import and export (done); presets, Motion Sketch, Wiggler | In progress | | M12 | Performance: layer cache and parallel compositing (done), GPU compositing, disk cache, motion tracking | In progress | | M13+ | Puppet, paint, roto, motion tracking, a plugin API | Planned | -| M15 | The web app (WebAssembly, WebGPU) | Done, single-threaded | +| M15 | The web app (WebAssembly, WebGPU) | Done: browser storage, Web Audio, renders and analyses in Web Workers, offline install ([docs/web.md](docs/web.md)) | | M14 | Built for agents: MCP server, command-line tool, control channel; Settings, keyboard shortcut editor, auto-save and crash recovery ([docs/preferences.md](docs/preferences.md)) | Done | ## How far from full parity -Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): +Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026): -- **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. -- **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in - parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** - by the audit's conservative per-feature figures. -- Hardest remaining pieces: the 3D camera tracker, Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7). +- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of + professional daily-use features, 65% of the long tail); all 298 After Effects effects. +- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude + Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features. +- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API. Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index b8222ddc..4f2192fc 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -9,7 +9,7 @@ //! effectcraft-cli get [--time S] read a property //! effectcraft-cli set [--time S] [--expression E] //! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png] -//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|png|jpeg|tiff|exr|gif] [--start S] [--end S] +//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue diff --git a/apps/effectcraft-web/Cargo.toml b/apps/effectcraft-web/Cargo.toml index d3f8aebb..03663089 100644 --- a/apps/effectcraft-web/Cargo.toml +++ b/apps/effectcraft-web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-web" -description = "EffectCraft in the browser (wasm32): eframe web runner, in-memory files, browser file pickers and downloads, window.effectcraft API" +description = "EffectCraft in the browser (wasm32): eframe web runner, persistent browser storage (OPFS / IndexedDB), Web Audio, Web Worker renders, PWA, window.effectcraft API" version.workspace = true edition.workspace = true license.workspace = true @@ -12,10 +12,14 @@ workspace = true [lib] crate-type = ["cdylib", "rlib"] -# Everything is wasm32-only: on other targets this crate is empty (so `cargo test --workspace` -# and clippy stay cheap). Build with `cargo xtask web` (see docs/web.md). -[target.'cfg(target_arch = "wasm32")'.dependencies] +# The browser glue is wasm32-only; on other targets the crate holds just the portable storage +# model (`store`), unit-tested natively. Build the app with `cargo xtask web` (see docs/web.md). +[dependencies] effectcraft-engine = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] effectcraft-host = { workspace = true } effectcraft-media = { workspace = true } effectcraft-export = { workspace = true } @@ -23,8 +27,8 @@ effectcraft-expr = { workspace = true } effectcraft-ui-egui = { workspace = true } egui = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu"] } -serde_json = { workspace = true } log = { workspace = true } +web-time = { workspace = true } # The wasm-bindgen CLI used by `cargo xtask web` must be exactly this version. wasm-bindgen = "=0.2.129" wasm-bindgen-futures = "0.4" @@ -35,6 +39,7 @@ web-sys = { version = "0.3", features = [ "CssStyleDeclaration", "Document", "Element", + "DedicatedWorkerGlobalScope", "Event", "EventTarget", "File", diff --git a/apps/effectcraft-web/js/host.js b/apps/effectcraft-web/js/host.js new file mode 100644 index 00000000..7b6200eb --- /dev/null +++ b/apps/effectcraft-web/js/host.js @@ -0,0 +1,335 @@ +// Browser services for the EffectCraft web app (imported by the wasm module through +// wasm-bindgen; see src/persist.rs, src/audio.rs and src/worker.rs): +// +// - storage: the Origin Private File System, or IndexedDB where OPFS can't write; +// - audio: a Web Audio output (AudioWorklet, ScriptProcessorNode fallback) fed stereo blocks; +// - workers: a pool of Web Workers, each running its own instance of the engine. +// +// Plain functions only: the Rust side owns the logic. + +// ------------------------------------------------------------------ storage + +const DIR = "effectcraft"; +const DB = "effectcraft"; +const STORE = "entries"; +let backend = null; + +// OPFS file names can't hold "/": keys are percent-encoded. +const enc = (k) => encodeURIComponent(k); +const dec = (n) => decodeURIComponent(n); + +async function opfs() { + if (!navigator.storage || !navigator.storage.getDirectory) throw new Error("no OPFS"); + const root = await navigator.storage.getDirectory(); + const dir = await root.getDirectoryHandle(DIR, { create: true }); + // Probe a write: some browsers expose OPFS but only write it from workers. + const probe = await dir.getFileHandle(".probe", { create: true }); + if (!probe.createWritable) throw new Error("OPFS is read-only on this thread"); + const w = await probe.createWritable(); + await w.write(new Uint8Array([1])); + await w.close(); + await dir.removeEntry(".probe"); + return { + name: "opfs", + async loadAll() { + const out = []; + for await (const [name, h] of dir.entries()) { + if (h.kind !== "file" || name.startsWith(".") || name.endsWith(".crswap")) continue; + const f = await h.getFile(); + out.push({ key: dec(name), data: new Uint8Array(await f.arrayBuffer()), modified: f.lastModified }); + } + return out; + }, + async put(key, data) { + // createWritable writes a swap file and replaces the entry on close: atomic. + const h = await dir.getFileHandle(enc(key), { create: true }); + const w = await h.createWritable(); + await w.write(data); + await w.close(); + }, + async del(key) { + try { await dir.removeEntry(enc(key)); } catch (e) { if (e.name !== "NotFoundError") throw e; } + }, + }; +} + +function req(r) { + return new Promise((res, rej) => { r.onsuccess = () => res(r.result); r.onerror = () => rej(r.error); }); +} + +async function idb() { + if (!self.indexedDB) throw new Error("no IndexedDB"); + const open = indexedDB.open(DB, 1); + open.onupgradeneeded = () => open.result.createObjectStore(STORE, { keyPath: "key" }); + const db = await req(open); + const tx = (mode) => db.transaction(STORE, mode).objectStore(STORE); + return { + name: "indexeddb", + async loadAll() { + const all = await req(tx("readonly").getAll()); + return all.map((r) => ({ key: r.key, data: new Uint8Array(r.data), modified: r.modified })); + }, + async put(key, data, modified) { + await req(tx("readwrite").put({ key, data: data.slice().buffer, modified })); + }, + async del(key) { + await req(tx("readwrite").delete(key)); + }, + }; +} + +function memory() { + const m = new Map(); + return { + name: "memory", + async loadAll() { return [...m.values()]; }, + async put(key, data, modified) { m.set(key, { key, data: data.slice(), modified }); }, + async del(key) { m.delete(key); }, + }; +} + +/// Open the store: OPFS, else IndexedDB, else memory (nothing persists). Resolves with the +/// backend's name. `prefer` ("opfs" | "indexeddb" | "memory") forces one (tests). +export async function storeOpen(prefer) { + const order = { opfs: [opfs, idb, memory], indexeddb: [idb, memory], memory: [memory] }[prefer] ?? [opfs, idb, memory]; + for (const make of order) { + try { backend = await make(); break; } catch (e) { console.info(`storage: ${make.name}: ${e.message ?? e}`); } + } + // Ask the browser not to evict the store under storage pressure (granted for installed apps). + try { if (navigator.storage && navigator.storage.persist) await navigator.storage.persist(); } catch {} + return backend.name; +} + +/// Every stored entry: [{key, data: Uint8Array, modified}]. +export async function storeLoadAll() { return backend.loadAll(); } +export async function storePut(key, data, modified) { return backend.put(key, data, modified); } +export async function storeDelete(key) { return backend.del(key); } + +/// {usage, quota, persisted, backend}. +export async function storeEstimate() { + const out = { backend: backend ? backend.name : null }; + try { + const e = await navigator.storage.estimate(); + out.usage = e.usage; out.quota = e.quota; + out.persisted = await navigator.storage.persisted(); + } catch {} + return out; +} + +// ------------------------------------------------------------------ audio + +// One AudioContext for the page. Browsers start it suspended until the user interacts with the +// page, so the first pointer/key event resumes it ("unlock"). +let ctx = null; +let node = null; +let kind = null; +// Sample frames handed to the output since `audioStart`, and the context time it started at. +let posted = 0; +let startTime = 0; +let underruns = 0; +// ScriptProcessor fallback: the queue lives on this thread. +let spQueue = []; +let spOffset = 0; +// Blocks pushed while the output node is being created. +let starting = false; +let pendingPush = []; +// Bumped by every start/stop: a start superseded while it awaited the worklet gives up. +let generation = 0; + +function context() { + if (!ctx) { + const AC = self.AudioContext || self.webkitAudioContext; + if (!AC) return null; + ctx = new AC({ latencyHint: "interactive" }); + } + return ctx; +} + +function unlock() { + const c = context(); + if (c && c.state !== "running") c.resume().catch(() => {}); +} + +/// Install the user-gesture unlock (called once at startup). +export function audioInstallUnlock() { + for (const ev of ["pointerdown", "keydown", "touchend"]) { + self.addEventListener(ev, unlock, { capture: true, passive: true }); + } +} + +/// {state, sampleRate, backend, posted, played, underruns, baseLatency, outputLatency}. +export function audioState() { + const c = ctx; + return { + state: c ? c.state : "none", + sampleRate: c ? c.sampleRate : 0, + backend: kind, + posted, + played: audioPlayed(), + underruns, + baseLatency: c ? c.baseLatency || 0 : 0, + outputLatency: c ? c.outputLatency || 0 : 0, + }; +} + +/// The output's sample rate (creates the context; 0 without Web Audio). +export function audioSampleRate() { + const c = context(); + return c ? c.sampleRate : 0; +} + +/// Start an output node; resolves when it is ready. Samples pushed before then queue up. +export async function audioStart(workletUrl) { + audioStop(); + const c = context(); + if (!c) throw new Error("Web Audio is not available"); + unlock(); + posted = 0; + underruns = 0; + spQueue = []; + spOffset = 0; + starting = true; + pendingPush = []; + const gen = ++generation; + if (c.audioWorklet && workletUrl) { + try { + if (!c.__ecWorklet) { + await c.audioWorklet.addModule(workletUrl); + c.__ecWorklet = true; + } + if (gen !== generation) return; + node = new AudioWorkletNode(c, "effectcraft-output", { numberOfInputs: 0, outputChannelCount: [2] }); + node.port.onmessage = (e) => { if (e.data && e.data.underrun) underruns += e.data.underrun; }; + kind = "worklet"; + } catch (e) { + console.warn("audio: AudioWorklet unavailable, using ScriptProcessorNode", e); + node = null; + } + } + if (!node) { + node = c.createScriptProcessor(2048, 0, 2); + node.onaudioprocess = (e) => { + const l = e.outputBuffer.getChannelData(0); + const r = e.outputBuffer.getChannelData(1); + let i = 0; + while (i < l.length && spQueue.length) { + const b = spQueue[0]; + const n = Math.min(l.length - i, (b.length - spOffset) >> 1); + for (let k = 0; k < n; k++) { l[i + k] = b[spOffset + 2 * k]; r[i + k] = b[spOffset + 2 * k + 1]; } + i += n; spOffset += 2 * n; + if (spOffset >= b.length) { spQueue.shift(); spOffset = 0; } + } + if (i < l.length) { underruns += l.length - i; l.fill(0, i); r.fill(0, i); } + }; + kind = "scriptProcessor"; + } + starting = false; + if (!node) return; + node.connect(c.destination); + startTime = c.currentTime; + const p = pendingPush; + pendingPush = []; + for (const b of p) send(b); +} + +function send(samples) { + if (kind === "worklet") node.port.postMessage(samples, [samples.buffer]); + else spQueue.push(samples); +} + +/// Queue interleaved stereo samples (Float32Array) for output. +export function audioPush(samples) { + posted += samples.length >> 1; + if (node && !starting) send(samples); + else if (starting) pendingPush.push(samples); +} + +/// Sample frames played since `audioStart`, by the context clock (AudioContext.currentTime). +export function audioPlayed() { + if (!ctx || !node || starting) return 0; + return Math.max(0, Math.round((ctx.currentTime - startTime) * ctx.sampleRate)); +} + +/// Output latency in sample frames (what's handed over but not audible yet comes on top). +export function audioOutputLatency() { + if (!ctx) return 0; + return Math.round(((ctx.outputLatency || 0) + (ctx.baseLatency || 0)) * ctx.sampleRate); +} + +export function audioStop() { + generation++; + starting = false; + pendingPush = []; + if (node) { + try { node.disconnect(); } catch {} + if (kind === "worklet") node.port.postMessage("stop"); + } + node = null; + spQueue = []; +} + +// ------------------------------------------------------------------ workers + +// Idle workers are reused; each keeps the files it was sent (path → size) so a file goes to a +// worker once. +const idle = []; +const running = new Map(); // job id → {worker, done} + +function spawn(base) { + const worker = new Worker(new URL("worker.js", base), { type: "module", name: "effectcraft-worker" }); + const w = { worker, files: new Map(), ready: null }; + w.ready = new Promise((res, rej) => { + worker.onmessage = (e) => { if (e.data && e.data.type === "ready") res(); }; + worker.onerror = (e) => rej(new Error(e.message || "worker failed to start")); + }); + worker.postMessage({ type: "init", module: self.effectcraftModule ?? null, base }); + return w; +} + +/// Run a job in a worker. `files`: [[path, Uint8Array]] the job reads; `onMessage(type, json, +/// path, bytes)` receives "reply" (json), "file" (path, bytes) and "error" (json = message). +export function workerRun(id, json, files, base, onMessage) { + const w = idle.pop() ?? spawn(base); + running.set(id, w); + w.ready.then(() => { + w.worker.onmessage = (e) => { + const m = e.data; + if (m.type === "reply") { + onMessage("reply", m.json, null, null); + if (m.done) { + running.delete(id); + idle.push(w); + } + } else if (m.type === "file") { + onMessage("file", null, m.path, new Uint8Array(m.bytes)); + } + }; + w.worker.onerror = (e) => { + running.delete(id); + onMessage("error", String(e.message || "worker error"), null, null); + w.worker.terminate(); + }; + for (const [path, bytes] of files) { + if (w.files.get(path) === bytes.length) continue; + w.files.set(path, bytes.length); + w.worker.postMessage({ type: "file", path, bytes }); + } + w.worker.postMessage({ type: "job", json }); + }, (e) => { + running.delete(id); + onMessage("error", String(e.message || e), null, null); + }); +} + +/// Stop a job: its worker is terminated (it can't be interrupted mid-job). +export function workerCancel(id) { + const w = running.get(id); + if (!w) return; + running.delete(id); + w.worker.terminate(); +} + +/// {running, idle}. +export function workerStats() { + return { running: running.size, idle: idle.length }; +} diff --git a/apps/effectcraft-web/src/api.rs b/apps/effectcraft-web/src/api.rs index 5579d111..57cfd98d 100644 --- a/apps/effectcraft-web/src/api.rs +++ b/apps/effectcraft-web/src/api.rs @@ -92,9 +92,68 @@ pub fn info() -> JsValue { } v["version"] = json!(env!("CARGO_PKG_VERSION")); v["threads"] = json!(false); + v["workers"] = from_js(&crate::worker::stats()); + v["audio"] = from_js(&crate::audio::state()); + if let Some(e) = crate::persist::last_error() { + v["storage"]["error"] = json!(e); + } to_js(&v) } +/// Save the project to browser storage without downloading it: `path` (default: its current +/// path, or `/.ecproj`). Resolves with `{path, bytes}`. +#[wasm_bindgen(js_name = saveToBrowser)] +pub fn save_to_browser(path: Option) -> js_sys::Promise { + js_sys::Promise::new(&mut |resolve, reject| { + let path = path.clone(); + crate::post(move |app| { + let p = path.or_else(|| app.session.path.clone()).unwrap_or_else(|| format!("/{}.ecproj", app.session.project_name())); + let p = if p.starts_with('/') { p } else { format!("/{p}") }; + crate::files::save_quietly(&p); + match app.session.execute("file.saveAs", json!({"path": p})) { + Ok(v) => { + let _ = resolve.call1(&JsValue::NULL, &to_js(&v)); + } + Err(e) => { + let _ = reject.call1(&JsValue::NULL, &JsValue::from_str(&e.to_string())); + } + } + }); + }) +} + +/// What browser storage holds: `{backend, usage, quota, persisted, files: [{path, size, +/// modified}], config: [name]}` (render outputs, which are not stored, are left out). +#[wasm_bindgen(js_name = listStored)] +pub async fn list_stored() -> JsValue { + let est = wasm_bindgen_futures::JsFuture::from(crate::persist::store_estimate()).await.map(|v| from_js(&v)).unwrap_or(json!({})); + let files: Vec = crate::files::STORE + .files() + .into_iter() + .filter(|f| f.3) + .map(|(path, size, modified, _)| json!({"path": path, "size": size, "modified": modified})) + .collect(); + let config: Vec = + crate::files::STORE.lock().list(crate::store::CONFIG).into_iter().map(|(k, ..)| json!(k.trim_start_matches(crate::store::CONFIG))).collect(); + let mut v = est; + v["files"] = json!(files); + v["config"] = json!(config); + v["pending"] = json!(crate::files::STORE.lock().has_pending()); + to_js(&v) +} + +/// Remove a stored file (media, project, auto-save). Resolves with whether it existed. +#[wasm_bindgen(js_name = removeStored)] +pub fn remove_stored(path: String) -> bool { + crate::files::STORE.remove_file(&path) +} + +/// Resolves once every change is written to browser storage. +#[wasm_bindgen] +pub fn flush() -> js_sys::Promise { + crate::persist::flushed() +} + /// Settle the promises whose replies arrived (called every frame). pub(crate) fn poll_replies() { PENDING.with(|p| { diff --git a/apps/effectcraft-web/src/app.rs b/apps/effectcraft-web/src/app.rs new file mode 100644 index 00000000..0076cd4f --- /dev/null +++ b/apps/effectcraft-web/src/app.rs @@ -0,0 +1,223 @@ +//! The app: startup (storage, session restore, hooks), the eframe app and the page glue. + +use std::cell::RefCell; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use effectcraft_engine::Session; + +use crate::{api, audio, files, persist, worker}; +use effectcraft_ui_egui::EffectcraftApp; +use serde_json::json; +use wasm_bindgen::prelude::*; + +/// Work for the UI thread with access to the app (posted by async tasks: picked files…). +type Action = Box; + +static WORKER: AtomicBool = AtomicBool::new(false); + +/// Running inside a Web Worker (no page, no downloads). +pub fn is_worker() -> bool { + WORKER.load(Ordering::Relaxed) +} + +pub(crate) fn set_worker() { + WORKER.store(true, Ordering::Relaxed); +} + +/// `rel` resolved against the page's URL (`worker.js`, `audio-worklet.js`); `""` = the page's +/// folder. +pub fn page_url(rel: &str) -> String { + let href = web_sys::window().and_then(|w| w.location().href().ok()).unwrap_or_default(); + web_sys::Url::new_with_base(if rel.is_empty() { "./" } else { rel }, &href).map(|u| u.href()).unwrap_or_else(|_| rel.to_string()) +} + +thread_local! { + static ACTIONS: RefCell> = const { RefCell::new(Vec::new()) }; + static CTX: RefCell> = const { RefCell::new(None) }; +} + +/// Run `f` on the UI thread at the next frame. +pub fn post(f: impl FnOnce(&mut EffectcraftApp) + 'static) { + ACTIONS.with(|a| a.borrow_mut().push(Box::new(f))); + repaint(); +} + +/// Ask for a UI frame. +pub fn repaint() { + CTX.with(|c| { + if let Some(c) = c.borrow().as_ref() { + c.request_repaint(); + } + }); +} + +/// Open a project or import media from table paths (picked or dropped files). +pub(crate) fn open_or_import(app: &mut EffectcraftApp, paths: Vec) { + let (projects, media): (Vec, Vec) = paths.into_iter().partition(|p| p.to_ascii_lowercase().ends_with(".ecproj")); + if let Some(p) = projects.last() + && let Err(e) = app.session.execute("file.open", json!({"path": p})) + { + app.ui.status = e.to_string(); + } + if !media.is_empty() { + match app.session.execute("file.import", json!({"paths": media})) { + Ok(r) => { + if let Some(errs) = r["errors"].as_array().filter(|e| !e.is_empty()) { + app.ui.status = errs.iter().filter_map(|e| e.as_str()).collect::>().join("; "); + } + } + Err(e) => app.ui.status = e.to_string(), + } + } +} + +/// The eframe app: the shared EffectCraft UI plus the browser host's per-frame duties. +pub struct WebApp { + pub app: EffectcraftApp, + /// Keep the session snapshot (off with `?empty`). + pub persist: bool, +} + +impl eframe::App for WebApp { + fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { + self.app.raw_input_hook(ctx, raw_input); + } + + fn logic(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) { + let actions = ACTIONS.with(|a| std::mem::take(&mut *a.borrow_mut())); + for f in actions { + f(&mut self.app); + } + // Files dropped on the page: read them, then open/import. + let dropped: Vec = ctx.input(|i| i.raw.dropped_files.iter().filter_map(|f| f.web_file().cloned()).collect()); + if !dropped.is_empty() { + files::read_files(dropped, |paths| post(move |app| open_or_import(app, paths))); + } + self.app.logic(ctx, frame); + api::poll_replies(); + // Render Queue outputs written this frame (here or by a worker) → downloads. + files::flush_downloads(); + // What a reload comes back to. + if self.persist { + persist::snapshot(&self.app.session, ctx.input(|i| i.time), 1.0); + } + api::set_info("gpu", json!({"compositor": self.app.gpu_adapter(), "viewerOnGpu": self.app.viewer_on_gpu()})); + } + + fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { + self.app.ui(ui, frame); + } +} + +fn query() -> Option { + web_sys::window().and_then(|w| w.location().search().ok()).and_then(|s| web_sys::UrlSearchParams::new_with_str(&s).ok()) +} + +fn query_flag(name: &str) -> bool { + query().is_some_and(|p| p.has(name)) +} + +fn query_value(name: &str) -> Option { + query().and_then(|p| p.get(name)) +} + +/// A session wired for the browser: media decoding from memory, expressions, scripting, Render +/// Queue export to downloads, settings in browser storage, renders and analyses in workers. +pub fn session() -> Session { + let pool = Arc::new(effectcraft_media::MediaPool::new()); + persist::register_media(&pool); + Session { + services: Arc::new(files::WebServices), + footage: pool.clone(), + importer: Some(Arc::new(files::WebImporter { pool })), + expr: Some(Arc::new(effectcraft_expr::Expressions)), + expr_check: Some(effectcraft_expr::check_syntax), + script: Some(effectcraft_host::script::runner), + exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })), + config: Some(Arc::new(persist::config())), + offload: (!query_flag("noworkers")).then(|| Arc::new(worker::WorkerOffload) as Arc), + ..Default::default() + } +} + +/// Browser file pickers in place of the desktop's file dialogs. +fn install_hooks(app: &mut EffectcraftApp) { + app.hooks.pick_files = Some(Box::new(|exts: &[&str]| { + let accept = exts.iter().map(|e| format!(".{e}")).collect::>().join(","); + if let Err(e) = files::pick(&accept, true, |paths| post(move |app| open_or_import(app, paths))) { + log::warn!("file picker: {e:?}"); + } + // the import happens when the browser hands over the files + Vec::new() + })); + app.hooks.pick_open_project = Some(Box::new(|| { + if let Err(e) = files::pick(".ecproj", false, |paths| post(move |app| open_or_import(app, paths))) { + log::warn!("file picker: {e:?}"); + } + None + })); + // Saving downloads the file: the chosen name is all a "save dialog" needs. + app.hooks.pick_save = Some(Box::new(|name: &str| Some(files::path_for(if name.is_empty() { "Untitled Project.ecproj" } else { name })))); + app.hooks.audio_device = Some(Box::new(audio::open)); + app.hooks.audio_devices = Some(Box::new(|| vec!["Browser default output".into()])); +} + +/// Entry point (called by the page's bootstrap script once the wasm module is instantiated). +#[wasm_bindgen] +pub async fn start(canvas_id: String) -> Result<(), JsValue> { + eframe::WebLogger::init(log::LevelFilter::Info).ok(); + let doc = web_sys::window().and_then(|w| w.document()).ok_or("no document")?; + let canvas: web_sys::HtmlCanvasElement = doc.get_element_by_id(&canvas_id).ok_or("no canvas")?.dyn_into()?; + let empty = query_flag("empty"); + let home = query_flag("home"); + // Browser storage: settings, recent projects, imported media, saved projects, auto-saves and + // the last session come back. + let t0 = js_sys::Date::now(); + let stored = match persist::load(&query_value("storage").unwrap_or_default()).await { + Ok(n) => n, + Err(e) => { + log::warn!("browser storage unavailable: {e}"); + 0 + } + }; + api::set_info("storage", json!({"backend": persist::backend(), "entries": stored, "loadMs": js_sys::Date::now() - t0})); + audio::install_unlock(); + let runner = eframe::WebRunner::new(); + runner + .start( + canvas, + eframe::WebOptions::default(), + Box::new(move |cc| { + CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone())); + let mut session = session(); + session.load_settings(); + let recovery = session.begin_recovery(); + // `?empty`: a blank project; `?demo`: the demo project; otherwise the last visit's + // project (the demo on a first visit). + let restored = !empty && !query_flag("demo") && persist::restore(&mut session); + if !restored && !empty { + let _ = session.execute("file.openDemoProject", json!({})); + } + api::set_info("restored", json!(restored)); + let mut app = EffectcraftApp::new(session); + app.ui.start_screen = home; + if let Some(r) = recovery.filter(|r| !restored && r.autosave.is_some()) { + app.offer_recovery(r); + } + install_hooks(&mut app); + let (tx, rx) = std::sync::mpsc::channel(); + api::set_sender(tx); + app = app.with_control(rx); + let backend = cc.wgpu_render_state.as_ref().map(|rs| format!("{:?}", rs.adapter.get_info().backend)); + api::set_info("backend", json!(backend)); + Ok(Box::new(WebApp { app, persist: !empty })) + }), + ) + .await?; + // Hide the loading message once the app runs. + if let Some(el) = doc.get_element_by_id("loading") { + el.remove(); + } + Ok(()) +} diff --git a/apps/effectcraft-web/src/audio.rs b/apps/effectcraft-web/src/audio.rs new file mode 100644 index 00000000..c16f3092 --- /dev/null +++ b/apps/effectcraft-web/src/audio.rs @@ -0,0 +1,109 @@ +//! Audio preview through Web Audio (`js/host.js`): an AudioWorklet (ScriptProcessorNode where +//! worklets are unavailable) plays the blocks the UI mixes ahead +//! ([`effectcraft_ui_egui::audio::AudioPlayback::pump`]). +//! +//! Every UI frame [`WebAudioOut::pump`] moves samples from the feed to the output so it stays +//! ~150 ms ahead of what has played. A/V sync follows `AudioContext.currentTime`: frames played +//! = (currentTime − start) × rate, so the device latency the playback clock subtracts is +//! "handed over but not played yet" plus the context's output latency. +//! +//! Browsers keep the AudioContext suspended until the user interacts with the page; the host +//! resumes it on the first pointer or key event. + +use std::sync::Arc; + +use effectcraft_ui_egui::audio::{AudioDevice, AudioFeed}; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen(module = "/js/host.js")] +extern "C" { + #[wasm_bindgen(js_name = audioInstallUnlock)] + pub fn install_unlock(); + #[wasm_bindgen(js_name = audioState)] + pub fn state() -> JsValue; + #[wasm_bindgen(js_name = audioSampleRate)] + fn sample_rate() -> f64; + #[wasm_bindgen(js_name = audioStart)] + fn audio_start(worklet_url: &str) -> js_sys::Promise; + #[wasm_bindgen(js_name = audioPush)] + fn audio_push(samples: js_sys::Float32Array); + #[wasm_bindgen(js_name = audioPlayed)] + fn audio_played() -> f64; + #[wasm_bindgen(js_name = audioOutputLatency)] + fn audio_output_latency() -> f64; + #[wasm_bindgen(js_name = audioStop)] + fn audio_stop(); +} + +/// How far ahead of the playhead the output is kept fed. +const LEAD_SECONDS: f64 = 0.15; + +pub struct WebAudioOut { + rate: u32, + feed: Option>, + /// Sample frames handed to the output since start. + posted: u64, + buf: Vec, +} + +/// The Settings ▸ Audio factory: Web Audio's default output (`None` without Web Audio). +pub fn open(_: &effectcraft_ui_egui::audio::AudioOutput) -> Option> { + let rate = sample_rate(); + (rate > 0.0).then(|| Box::new(WebAudioOut { rate: rate as u32, feed: None, posted: 0, buf: Vec::new() }) as Box) +} + +fn worklet_url() -> String { + crate::page_url("audio-worklet.js") +} + +impl AudioDevice for WebAudioOut { + fn sample_rate(&self) -> u32 { + self.rate + } + + fn start(&mut self, feed: Arc) -> Result<(), String> { + self.feed = Some(feed); + self.posted = 0; + let p = audio_start(&worklet_url()); + wasm_bindgen_futures::spawn_local(async move { + if let Err(e) = wasm_bindgen_futures::JsFuture::from(p).await { + log::warn!("audio output: {e:?}"); + } + }); + Ok(()) + } + + fn stop(&mut self) { + self.feed = None; + audio_stop(); + } + + fn latency_frames(&self) -> u64 { + let played = audio_played() as u64; + self.posted.saturating_sub(played) + audio_output_latency() as u64 + } + + fn pump(&mut self) { + let Some(feed) = &self.feed else { return }; + let played = audio_played() as u64; + let lead = (LEAD_SECONDS * self.rate as f64) as u64; + let want = (played + lead).saturating_sub(self.posted) as usize; + // Only real samples: the output plays silence by itself when it runs dry. + let n = want.min(feed.queued_frames()); + if n == 0 { + return; + } + self.buf.resize(n * 2, 0.0); + feed.pull(&mut self.buf); + audio_push(js_sys::Float32Array::from(&self.buf[..])); + self.posted += n as u64; + } +} + +impl Drop for WebAudioOut { + fn drop(&mut self) { + if self.feed.is_some() { + audio_stop(); + } + } +} diff --git a/apps/effectcraft-web/src/files.rs b/apps/effectcraft-web/src/files.rs index 853a92e6..78700e28 100644 --- a/apps/effectcraft-web/src/files.rs +++ b/apps/effectcraft-web/src/files.rs @@ -1,39 +1,55 @@ -//! Files in the browser: an in-memory file table in place of the file system, `` -//! pickers for Open/Import, and downloads for everything the app writes (projects, renders). +//! Files in the browser: a virtual file table in place of the file system, `` +//! pickers for Open/Import, and downloads for what the user saves and renders. //! //! Picked or dropped files are read into the table under `/`; the engine, the media pool and //! the exporter read and write through it ([`WebServices`], [`WebImporter`], [`export_sink`]). +//! The table is the persistent [`Store`]'s file namespace: imported media, saved projects and +//! auto-saves survive reloads ([`crate::persist`]); render outputs don't (they download). -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, LazyLock, Mutex}; use effectcraft_engine::Services; use effectcraft_engine::project::Footage; use wasm_bindgen::JsCast; use wasm_bindgen::prelude::*; -static FILES: Mutex>>> = Mutex::new(None); +use crate::store::Store; + +/// The page's store (each worker has its own, unpersisted). +pub static STORE: LazyLock> = LazyLock::new(|| { + let s = Arc::new(Store::default()); + let _ = s.clock.set(js_sys::Date::now); + s +}); /// Render outputs written since the last [`flush_downloads`]. static OUTPUTS: Mutex> = Mutex::new(Vec::new()); +/// Saves that write to browser storage only (no download): `api.saveToBrowser`. +static QUIET: Mutex> = Mutex::new(Vec::new()); -fn with_files(f: impl FnOnce(&mut HashMap>) -> R) -> R { - f(FILES.lock().unwrap_or_else(|e| e.into_inner()).get_or_insert_default()) -} - -/// Store a file in the table. +/// Store a file in the table (persisted). pub fn put(path: &str, data: Arc<[u8]>) { - with_files(|m| m.insert(path.to_string(), data)); + STORE.put_file(path, data, true); } pub fn get(path: &str) -> Option> { - with_files(|m| m.get(path).cloned()) + STORE.file(path) } /// `(path, size)` of every file in the table. pub fn list() -> Vec<(String, usize)> { - let mut v: Vec<(String, usize)> = with_files(|m| m.iter().map(|(k, d)| (k.clone(), d.len())).collect()); - v.sort(); - v + STORE.files().into_iter().map(|(p, n, _, _)| (p, n)).collect() +} + +/// The next write of `path` goes to browser storage only (no download). +pub fn save_quietly(path: &str) { + QUIET.lock().unwrap_or_else(|e| e.into_inner()).push(path.to_string()); +} + +/// A rendered file (from the page's exporter or a worker): kept in the table unpersisted and +/// downloaded at the next [`flush_downloads`]. +pub fn add_output(path: &str, data: Arc<[u8]>) { + STORE.put_file(path, data, false); + OUTPUTS.lock().unwrap_or_else(|e| e.into_inner()).push(path.to_string()); } /// The table path for a picked/dropped file name. @@ -85,7 +101,8 @@ pub fn download(name: &str, data: &[u8]) -> Result<(), JsValue> { Ok(()) } -/// Engine file access (projects): reads from the table; writes go to the table and download. +/// Engine file access (projects): reads from the table; writes go to the table (persisted) and +/// download, except [`save_quietly`] saves. pub struct WebServices; impl Services for WebServices { @@ -96,6 +113,15 @@ impl Services for WebServices { } fn write_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> { put(path, data.into()); + let quiet = { + let mut q = QUIET.lock().unwrap_or_else(|e| e.into_inner()); + let n = q.len(); + q.retain(|p| p != path); + q.len() != n + }; + if quiet || crate::is_worker() { + return Ok(()); + } download(file_name(path), data).map_err(|e| std::io::Error::other(format!("download failed: {e:?}"))) } } @@ -116,10 +142,7 @@ impl effectcraft_engine::Importer for WebImporter { /// Render Queue output: files land in the table and are downloaded after the render /// ([`flush_downloads`]; several files, e.g. an image sequence, as one `.zip`). pub fn export_sink() -> Arc { - Arc::new(|path: &str, data: Vec| { - put(path, data.into()); - OUTPUTS.lock().unwrap_or_else(|e| e.into_inner()).push(path.to_string()); - }) + Arc::new(|path: &str, data: Vec| add_output(path, data.into())) } /// Download the render outputs written since the last call. diff --git a/apps/effectcraft-web/src/lib.rs b/apps/effectcraft-web/src/lib.rs index edd90e31..59de4433 100644 --- a/apps/effectcraft-web/src/lib.rs +++ b/apps/effectcraft-web/src/lib.rs @@ -6,175 +6,31 @@ //! //! | Desktop | Web | //! |---|---| -//! | file system (`FsServices`, media reads, export writes) | in-memory file table ([`files`]) | +//! | file system (`FsServices`, media reads, export writes) | a virtual file table ([`files`]) persisted to the Origin Private File System / IndexedDB ([`persist`], [`store`]) | +//! | config directory (settings, shortcuts, recent projects, auto-saves) | the same store ([`store::WebConfig`]) | //! | rfd file dialogs | `` pickers; drop files on the page | //! | written files (Save, Render Queue) | browser downloads (several render files as one `.zip`) | //! | frame render threads | `Frames::pump` on the UI thread between egui frames | -//! | Render Queue thread | the render runs inline (the page waits until it is done) | +//! | Render Queue / analysis threads | Web Workers running their own engine instance ([`worker`]) | +//! | cpal audio output | Web Audio: an AudioWorklet fed from the preview mixdown ([`audio`]) | //! | TCP control channel / MCP | `window.effectcraft` JavaScript API ([`api`]) | //! -//! Everything here is `wasm32`-only; on other targets the crate is empty. -#![cfg(target_arch = "wasm32")] +//! [`store`] is portable (unit-tested natively); everything else is `wasm32`-only. +pub mod store; + +#[cfg(target_arch = "wasm32")] pub mod api; +#[cfg(target_arch = "wasm32")] +mod app; +#[cfg(target_arch = "wasm32")] +pub mod audio; +#[cfg(target_arch = "wasm32")] pub mod files; +#[cfg(target_arch = "wasm32")] +pub mod persist; +#[cfg(target_arch = "wasm32")] +pub mod worker; -use std::cell::RefCell; -use std::sync::Arc; - -use effectcraft_engine::Session; -use effectcraft_ui_egui::EffectcraftApp; -use serde_json::json; -use wasm_bindgen::prelude::*; - -/// Work for the UI thread with access to the app (posted by async tasks: picked files…). -type Action = Box; - -thread_local! { - static ACTIONS: RefCell> = const { RefCell::new(Vec::new()) }; - static CTX: RefCell> = const { RefCell::new(None) }; -} - -/// Run `f` on the UI thread at the next frame. -pub fn post(f: impl FnOnce(&mut EffectcraftApp) + 'static) { - ACTIONS.with(|a| a.borrow_mut().push(Box::new(f))); - repaint(); -} - -/// Ask for a UI frame. -pub fn repaint() { - CTX.with(|c| { - if let Some(c) = c.borrow().as_ref() { - c.request_repaint(); - } - }); -} - -/// Open a project or import media from table paths (picked or dropped files). -fn open_or_import(app: &mut EffectcraftApp, paths: Vec) { - let (projects, media): (Vec, Vec) = paths.into_iter().partition(|p| p.to_ascii_lowercase().ends_with(".ecproj")); - if let Some(p) = projects.last() - && let Err(e) = app.session.execute("file.open", json!({"path": p})) - { - app.ui.status = e.to_string(); - } - if !media.is_empty() { - match app.session.execute("file.import", json!({"paths": media})) { - Ok(r) => { - if let Some(errs) = r["errors"].as_array().filter(|e| !e.is_empty()) { - app.ui.status = errs.iter().filter_map(|e| e.as_str()).collect::>().join("; "); - } - } - Err(e) => app.ui.status = e.to_string(), - } - } -} - -/// The eframe app: the shared EffectCraft UI plus the browser host's per-frame duties. -pub struct WebApp { - pub app: EffectcraftApp, -} - -impl eframe::App for WebApp { - fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { - self.app.raw_input_hook(ctx, raw_input); - } - - fn logic(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) { - let actions = ACTIONS.with(|a| std::mem::take(&mut *a.borrow_mut())); - for f in actions { - f(&mut self.app); - } - // Files dropped on the page: read them, then open/import. - let dropped: Vec = ctx.input(|i| i.raw.dropped_files.iter().filter_map(|f| f.web_file().cloned()).collect()); - if !dropped.is_empty() { - files::read_files(dropped, |paths| post(move |app| open_or_import(app, paths))); - } - self.app.logic(ctx, frame); - api::poll_replies(); - // Render Queue outputs written this frame → downloads. - files::flush_downloads(); - } - - fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { - self.app.ui(ui, frame); - } -} - -fn query_flag(name: &str) -> bool { - web_sys::window().and_then(|w| w.location().search().ok()).and_then(|s| web_sys::UrlSearchParams::new_with_str(&s).ok()).is_some_and(|p| p.has(name)) -} - -/// A session wired for the browser: media decoding from memory, expressions, scripting, Render -/// Queue export to downloads. -pub fn session() -> Session { - let pool = Arc::new(effectcraft_media::MediaPool::new()); - Session { - services: Arc::new(files::WebServices), - footage: pool.clone(), - importer: Some(Arc::new(files::WebImporter { pool })), - expr: Some(Arc::new(effectcraft_expr::Expressions)), - expr_check: Some(effectcraft_expr::check_syntax), - script: Some(effectcraft_host::script::runner), - exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })), - ..Default::default() - } -} - -/// Browser file pickers in place of the desktop's file dialogs. -fn install_hooks(app: &mut EffectcraftApp) { - app.hooks.pick_files = Some(Box::new(|exts: &[&str]| { - let accept = exts.iter().map(|e| format!(".{e}")).collect::>().join(","); - if let Err(e) = files::pick(&accept, true, |paths| post(move |app| open_or_import(app, paths))) { - log::warn!("file picker: {e:?}"); - } - // the import happens when the browser hands over the files - Vec::new() - })); - app.hooks.pick_open_project = Some(Box::new(|| { - if let Err(e) = files::pick(".ecproj", false, |paths| post(move |app| open_or_import(app, paths))) { - log::warn!("file picker: {e:?}"); - } - None - })); - // Saving downloads the file: the chosen name is all a "save dialog" needs. - app.hooks.pick_save = Some(Box::new(|name: &str| Some(files::path_for(if name.is_empty() { "Untitled Project.ecproj" } else { name })))); -} - -/// Entry point (called by the page's bootstrap script once the wasm module is instantiated). -#[wasm_bindgen] -pub async fn start(canvas_id: String) -> Result<(), JsValue> { - eframe::WebLogger::init(log::LevelFilter::Info).ok(); - let doc = web_sys::window().and_then(|w| w.document()).ok_or("no document")?; - let canvas: web_sys::HtmlCanvasElement = doc.get_element_by_id(&canvas_id).ok_or("no canvas")?.dyn_into()?; - let demo = !query_flag("empty"); - let home = query_flag("home"); - let runner = eframe::WebRunner::new(); - runner - .start( - canvas, - eframe::WebOptions::default(), - Box::new(move |cc| { - CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone())); - let mut session = session(); - if demo { - let _ = session.execute("file.openDemoProject", json!({})); - } - let mut app = EffectcraftApp::new(session); - app.ui.start_screen = home; - install_hooks(&mut app); - let (tx, rx) = std::sync::mpsc::channel(); - api::set_sender(tx); - app = app.with_control(rx); - let backend = cc.wgpu_render_state.as_ref().map(|rs| format!("{:?}", rs.adapter.get_info().backend)); - api::set_info("backend", json!(backend)); - Ok(Box::new(WebApp { app })) - }), - ) - .await?; - // Hide the loading message once the app runs. - if let Some(el) = doc.get_element_by_id("loading") { - el.remove(); - } - Ok(()) -} +#[cfg(target_arch = "wasm32")] +pub use app::*; diff --git a/apps/effectcraft-web/src/persist.rs b/apps/effectcraft-web/src/persist.rs new file mode 100644 index 00000000..6735d066 --- /dev/null +++ b/apps/effectcraft-web/src/persist.rs @@ -0,0 +1,190 @@ +//! Persistence: the [`crate::store::Store`] mirrored to the browser's storage (the Origin +//! Private File System, IndexedDB where OPFS can't write, memory as a last resort; `js/host.js`). +//! +//! - [`load`] (before the app starts) opens the storage and reads every entry into the mirror. +//! - Changes flush in the background ([`schedule_flush`], wired as the store's change hook): +//! one write per changed key, in order, with nothing written twice concurrently. +//! - The session snapshot ([`snapshot`]) records the open project and editor state every second +//! or so while it changes, so a reload (or a crash) comes back to where the user was +//! ([`restore`]). + +use std::cell::Cell; + +use effectcraft_engine::Session; +use effectcraft_engine::config::ConfigStore; +use effectcraft_engine::project::Project; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +use crate::files::STORE; +use crate::store::{SNAPSHOT, SNAPSHOT_META, SnapshotMeta, WebConfig}; + +#[wasm_bindgen(module = "/js/host.js")] +extern "C" { + #[wasm_bindgen(js_name = storeOpen)] + fn store_open(prefer: &str) -> js_sys::Promise; + #[wasm_bindgen(js_name = storeLoadAll)] + fn store_load_all() -> js_sys::Promise; + #[wasm_bindgen(js_name = storePut)] + fn store_put(key: &str, data: js_sys::Uint8Array, modified: f64) -> js_sys::Promise; + #[wasm_bindgen(js_name = storeDelete)] + fn store_delete(key: &str) -> js_sys::Promise; + #[wasm_bindgen(js_name = storeEstimate)] + pub(crate) fn store_estimate() -> js_sys::Promise; +} + +thread_local! { + static FLUSHING: Cell = const { Cell::new(false) }; + static BACKEND: std::cell::RefCell = const { std::cell::RefCell::new(String::new()) }; + static LAST_ERROR: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; + static FLUSH_WAITERS: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; + static SNAP: Cell<(u64, f64)> = const { Cell::new((u64::MAX, 0.0)) }; + static LAST_META: std::cell::RefCell = const { std::cell::RefCell::new(String::new()) }; +} + +/// The storage in use: `opfs`, `indexeddb` or `memory`. +pub fn backend() -> String { + BACKEND.with(|b| b.borrow().clone()) +} + +/// The last storage error (quota exceeded…), if any. +pub fn last_error() -> Option { + LAST_ERROR.with(|e| e.borrow().clone()) +} + +fn js_err(e: JsValue) -> String { + e.as_string().or_else(|| js_sys::Reflect::get(&e, &"message".into()).ok().and_then(|m| m.as_string())).unwrap_or_else(|| format!("{e:?}")) +} + +/// Open the browser storage and read everything into the store. `prefer` forces a backend +/// (`?storage=indexeddb` / `memory`; empty = OPFS, then IndexedDB). Returns the entry count. +pub async fn load(prefer: &str) -> Result { + let name = JsFuture::from(store_open(prefer)).await.map_err(js_err)?; + BACKEND.with(|b| *b.borrow_mut() = name.as_string().unwrap_or_default()); + let all = JsFuture::from(store_load_all()).await.map_err(js_err)?; + let all: js_sys::Array = all.dyn_into().map_err(js_err)?; + let n = all.length() as usize; + { + let mut m = STORE.lock(); + for e in all.iter() { + let get = |k: &str| js_sys::Reflect::get(&e, &k.into()).unwrap_or(JsValue::UNDEFINED); + let Some(key) = get("key").as_string() else { continue }; + let data: js_sys::Uint8Array = match get("data").dyn_into() { + Ok(d) => d, + Err(_) => continue, + }; + m.load(&key, data.to_vec().into(), get("modified").as_f64().unwrap_or(0.0)); + } + } + let _ = STORE.on_change.set(schedule_flush); + Ok(n) +} + +/// Write pending changes to the browser storage (in the background; one flush at a time). +pub fn schedule_flush() { + if FLUSHING.with(|f| f.replace(true)) { + return; // the running flush picks the new changes up + } + wasm_bindgen_futures::spawn_local(async { + loop { + let pending = STORE.lock().take_pending(); + if pending.is_empty() { + break; + } + for p in pending { + let r = match &p.data { + Some(d) => JsFuture::from(store_put(&p.key, js_sys::Uint8Array::from(&d[..]), p.modified)).await, + None => JsFuture::from(store_delete(&p.key)).await, + }; + if let Err(e) = r { + let msg = format!("saving {} to browser storage failed: {}", p.key, js_err(e)); + log::warn!("{msg}"); + LAST_ERROR.with(|l| *l.borrow_mut() = Some(msg)); + } + } + } + FLUSHING.with(|f| f.set(false)); + for w in FLUSH_WAITERS.with(|w| std::mem::take(&mut *w.borrow_mut())) { + let _ = w.call0(&JsValue::NULL); + } + }); +} + +/// A promise that resolves once every change so far is in the browser storage. +pub fn flushed() -> js_sys::Promise { + js_sys::Promise::new(&mut |resolve, _| { + if !FLUSHING.with(|f| f.get()) && !STORE.lock().has_pending() { + let _ = resolve.call0(&JsValue::NULL); + } else { + FLUSH_WAITERS.with(|w| w.borrow_mut().push(resolve)); + schedule_flush(); + } + }) +} + +/// The config store for the session. +pub fn config() -> WebConfig { + WebConfig::new(STORE.clone()) +} + +/// Record the open project and editor state when they changed (checked at most every +/// `min_secs`). +pub fn snapshot(s: &Session, now: f64, min_secs: f64) { + let (rev, at) = SNAP.with(|c| c.get()); + if now - at < min_secs { + return; + } + SNAP.with(|c| c.set((rev, now))); + let meta = SnapshotMeta { path: s.path.clone(), dirty: s.is_dirty(), state: serde_json::to_value(&s.state).ok(), saved: 0.0 }; + let key = serde_json::to_string(&meta).unwrap_or_default(); + if rev == s.revision && LAST_META.with(|m| *m.borrow() == key) { + return; + } + let c = config(); + if rev != s.revision { + let _ = c.write(SNAPSHOT, &s.project.to_json()); + } + let _ = c.write(SNAPSHOT_META, &serde_json::to_string(&SnapshotMeta { saved: js_sys::Date::now(), ..meta }).unwrap_or_default()); + SNAP.with(|c| c.set((s.revision, now))); + LAST_META.with(|m| *m.borrow_mut() = key); +} + +/// Reopen the last visit's project and editor state. Returns whether there was one. +pub fn restore(s: &mut Session) -> bool { + let c = config(); + let (Some(json), Some(meta)) = (c.read(SNAPSHOT), c.read(SNAPSHOT_META)) else { return false }; + let meta: SnapshotMeta = serde_json::from_str(&meta).unwrap_or_default(); + let p = match Project::from_json(&json) { + Ok(p) => p, + Err(e) => { + log::warn!("restoring the last session: {e}"); + return false; + } + }; + s.replace_project(p, meta.path.clone()); + if let Some(st) = meta.state.and_then(|v| serde_json::from_value(v).ok()) { + s.state = st; + } + if meta.dirty { + // Unsaved changes stay unsaved. + s.bump(); + } + SNAP.with(|c| c.set((s.revision, 0.0))); + log::info!("restored the last session ({})", meta.path.as_deref().unwrap_or("untitled")); + true +} + +/// Register persisted media with the media pool (so restored projects find their footage). +pub fn register_media(pool: &effectcraft_media::MediaPool) -> usize { + let mut n = 0; + for (path, _, _, persisted) in STORE.files() { + if persisted + && !path.to_ascii_lowercase().ends_with(".ecproj") + && let Some(d) = STORE.file(&path) + { + pool.add_bytes(&path, d); + n += 1; + } + } + n +} diff --git a/apps/effectcraft-web/src/store.rs b/apps/effectcraft-web/src/store.rs new file mode 100644 index 00000000..0e4ea029 --- /dev/null +++ b/apps/effectcraft-web/src/store.rs @@ -0,0 +1,399 @@ +//! What the browser keeps between visits: the in-memory side of the persistent store. +//! +//! The engine's storage traits are synchronous ([`ConfigStore`], [`FileOps`], +//! [`effectcraft_engine::Services`]); the browser's storage (the Origin Private File System, or +//! IndexedDB where OPFS can't write) is asynchronous. So the app works on a [`Mirror`]: every +//! stored entry is loaded into memory before the app starts, reads and writes hit the mirror +//! synchronously, and writes queue up as pending operations that the web host flushes to the +//! browser in the background (coalesced per key, in order). +//! +//! Keys are namespaced: +//! +//! | Key | What | +//! |---|---| +//! | `config/` | settings (`prefs.json`), shortcut presets, the recovery sentinel, the session snapshot | +//! | `files` | the virtual file table: imported media, saved projects, auto-saves (`/EffectCraft Auto-Save/…`) | +//! +//! Render outputs go into the file table unpersisted (they download instead). +//! +//! This module is plain Rust (no browser APIs) so it is unit-tested natively. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_engine::config::{ConfigStore, FileOps}; + +/// Prefix of configuration entries. +pub const CONFIG: &str = "config/"; +/// Prefix of file-table entries (followed by the absolute path, `files/clip.mp4`). +pub const FILES: &str = "files"; +/// The session snapshot (project + editor state), written while the app runs and restored +/// on the next visit. +pub const SNAPSHOT: &str = "session.ecproj"; +/// Its metadata ([`SnapshotMeta`]). +pub const SNAPSHOT_META: &str = "session.json"; + +/// One stored blob. +#[derive(Clone, Debug)] +pub struct Entry { + pub data: Arc<[u8]>, + /// Milliseconds since the Unix epoch. + pub modified: f64, + /// Written to the browser's storage (render outputs are not). + pub persist: bool, +} + +/// A queued write for the browser storage: `data` = `None` deletes the key. +#[derive(Clone, Debug, PartialEq)] +pub struct Pending { + pub key: String, + pub data: Option>, + pub modified: f64, +} + +/// The in-memory copy of the store plus the writes not yet flushed. +#[derive(Default)] +pub struct Mirror { + entries: BTreeMap, + /// Keys changed since the last [`Mirror::take_pending`], in first-change order. + dirty: Vec, +} + +impl Mirror { + /// An entry read from the browser's storage at startup (nothing to flush). + pub fn load(&mut self, key: &str, data: Arc<[u8]>, modified: f64) { + self.entries.insert(key.to_string(), Entry { data, modified, persist: true }); + } + + pub fn put(&mut self, key: &str, data: Arc<[u8]>, modified: f64, persist: bool) { + let was_persisted = self.entries.get(key).is_some_and(|e| e.persist); + self.entries.insert(key.to_string(), Entry { data, modified, persist }); + if persist || was_persisted { + self.touch(key); + } + } + + pub fn remove(&mut self, key: &str) -> bool { + match self.entries.remove(key) { + Some(e) => { + if e.persist { + self.touch(key); + } + true + } + None => false, + } + } + + fn touch(&mut self, key: &str) { + if !self.dirty.iter().any(|k| k == key) { + self.dirty.push(key.to_string()); + } + } + + pub fn get(&self, key: &str) -> Option<&Entry> { + self.entries.get(key) + } + + /// Entries under `prefix`: (key, size, modified, persisted). + pub fn list(&self, prefix: &str) -> Vec<(String, usize, f64, bool)> { + self.entries + .range(prefix.to_string()..) + .take_while(|(k, _)| k.starts_with(prefix)) + .map(|(k, e)| (k.clone(), e.data.len(), e.modified, e.persist)) + .collect() + } + + /// The writes to flush, one per changed key with its latest state (a put, or a delete when + /// the key is gone or no longer persisted). + pub fn take_pending(&mut self) -> Vec { + std::mem::take(&mut self.dirty) + .into_iter() + .map(|key| match self.entries.get(&key) { + Some(e) if e.persist => Pending { data: Some(e.data.clone()), modified: e.modified, key }, + _ => Pending { key, data: None, modified: 0.0 }, + }) + .collect() + } + + pub fn has_pending(&self) -> bool { + !self.dirty.is_empty() + } +} + +/// The key of a file-table path (`/a/b.mov` → `files/a/b.mov`). +pub fn file_key(path: &str) -> String { + if path.starts_with('/') { format!("{FILES}{path}") } else { format!("{FILES}/{path}") } +} + +/// The path of a file-table key. +pub fn key_path(key: &str) -> Option<&str> { + key.strip_prefix(FILES).filter(|p| p.starts_with('/')) +} + +/// The persistent store: a [`Mirror`] and a hook that wakes the flusher after changes. +#[derive(Default)] +pub struct Store { + pub mirror: Mutex, + /// Called after every change that needs flushing (the web host schedules a flush). + pub on_change: OnceLock, + /// Clock in milliseconds since the Unix epoch (tests pin it). + pub clock: OnceLock f64>, +} + +impl Store { + pub fn lock(&self) -> std::sync::MutexGuard<'_, Mirror> { + self.mirror.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn now(&self) -> f64 { + self.clock.get().map_or(0.0, |f| f()) + } + + fn changed(&self) { + if let Some(f) = self.on_change.get() { + f(); + } + } + + // ---- file table + + /// Store a file under `path` (persisted unless `persist` is false: render outputs). + pub fn put_file(&self, path: &str, data: Arc<[u8]>, persist: bool) { + let now = self.now(); + self.lock().put(&file_key(path), data, now, persist); + if persist { + self.changed(); + } + } + + pub fn file(&self, path: &str) -> Option> { + self.lock().get(&file_key(path)).map(|e| e.data.clone()) + } + + pub fn remove_file(&self, path: &str) -> bool { + let r = self.lock().remove(&file_key(path)); + self.changed(); + r + } + + /// Every file: (path, size, modified ms, persisted). + pub fn files(&self) -> Vec<(String, usize, f64, bool)> { + self.lock().list(FILES).into_iter().filter_map(|(k, n, m, p)| Some((key_path(&k)?.to_string(), n, m, p))).collect() + } +} + +/// Settings and friends ([`ConfigStore`]) in the store; auto-saves ([`FileOps`]) in its file +/// table, under `/` (untitled projects auto-save to `/EffectCraft Auto-Save/`). +pub struct WebConfig { + store: Arc, + files: WebFiles, +} + +impl WebConfig { + pub fn new(store: Arc) -> WebConfig { + WebConfig { files: WebFiles(store.clone()), store } + } +} + +/// [`FileOps`] on the store's file table. +pub struct WebFiles(pub Arc); + +impl ConfigStore for WebConfig { + fn read(&self, name: &str) -> Option { + let m = self.store.lock(); + m.get(&format!("{CONFIG}{name}")).map(|e| String::from_utf8_lossy(&e.data).into_owned()) + } + fn write(&self, name: &str, data: &str) -> std::io::Result<()> { + let now = self.store.now(); + self.store.lock().put(&format!("{CONFIG}{name}"), data.as_bytes().into(), now, true); + self.store.changed(); + Ok(()) + } + fn remove(&self, name: &str) -> std::io::Result<()> { + self.store.lock().remove(&format!("{CONFIG}{name}")); + self.store.changed(); + Ok(()) + } + fn dir(&self) -> Option { + Some(PathBuf::from("/")) + } + fn files(&self) -> &dyn FileOps { + &self.files + } +} + +fn path_str(p: &Path) -> String { + p.to_string_lossy().replace('\\', "/") +} + +impl FileOps for WebFiles { + fn list(&self, dir: &Path) -> Vec<(String, std::time::SystemTime)> { + let dir = path_str(dir); + let dir = dir.trim_end_matches('/'); + self.0 + .files() + .into_iter() + .filter_map(|(p, _, m, _)| { + let (parent, name) = p.rsplit_once('/')?; + (parent == dir).then(|| (name.to_string(), std::time::UNIX_EPOCH + std::time::Duration::from_secs_f64(m.max(0.0) / 1000.0))) + }) + .collect() + } + fn write(&self, path: &Path, data: &[u8]) -> std::io::Result<()> { + self.0.put_file(&path_str(path), data.into(), true); + Ok(()) + } + fn remove(&self, path: &Path) -> std::io::Result<()> { + if self.0.remove_file(&path_str(path)) { Ok(()) } else { Err(std::io::Error::new(std::io::ErrorKind::NotFound, path_str(path))) } + } + fn is_file(&self, path: &Path) -> bool { + self.0.file(&path_str(path)).is_some() + } +} + +/// What the session snapshot records besides the project. +#[derive(Clone, Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct SnapshotMeta { + /// The project's file (None = untitled). + pub path: Option, + /// Unsaved changes when the snapshot was taken. + pub dirty: bool, + /// Viewer, timeline and selection state. + pub state: Option, + /// Milliseconds since the Unix epoch. + pub saved: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn store() -> Arc { + let s = Arc::new(Store::default()); + let _ = s.clock.set(|| 1_000.0); + s + } + + #[test] + fn writes_coalesce_per_key_in_order() { + let mut m = Mirror::default(); + m.put("config/a", Arc::from(&b"1"[..]), 1.0, true); + m.put("files/x.png", Arc::from(&b"px"[..]), 2.0, true); + m.put("config/a", Arc::from(&b"2"[..]), 3.0, true); + let p = m.take_pending(); + assert_eq!(p.len(), 2); + assert_eq!((p[0].key.as_str(), p[0].data.as_deref()), ("config/a", Some(&b"2"[..]))); + assert_eq!(p[0].modified, 3.0); + assert_eq!(p[1].key, "files/x.png"); + assert!(!m.has_pending() && m.take_pending().is_empty()); + // Delete after put: one delete. + m.put("files/y", Arc::from(&b"y"[..]), 4.0, true); + m.remove("files/y"); + assert_eq!(m.take_pending(), vec![Pending { key: "files/y".into(), data: None, modified: 0.0 }]); + } + + #[test] + fn unpersisted_entries_never_flush_and_loaded_ones_are_clean() { + let mut m = Mirror::default(); + m.load("files/old.mov", Arc::from(&b"m"[..]), 5.0); + m.put("files/render.gif", Arc::from(&b"g"[..]), 6.0, false); + assert!(!m.has_pending()); + assert!(!m.remove("files/none")); + m.remove("files/render.gif"); + assert!(!m.has_pending()); + // Overwriting a persisted file with an unpersisted one deletes the stored copy. + m.put("files/old.mov", Arc::from(&b"n"[..]), 7.0, false); + assert_eq!(m.take_pending()[0].data, None); + assert_eq!(m.list("files").len(), 1); + } + + #[test] + fn keys_and_paths_round_trip() { + assert_eq!(file_key("/a b/c.mov"), "files/a b/c.mov"); + assert_eq!(file_key("c.mov"), "files/c.mov"); + assert_eq!(key_path("files/a b/c.mov"), Some("/a b/c.mov")); + assert_eq!(key_path("config/prefs.json"), None); + let s = store(); + s.put_file("/clip.mp4", Arc::from(&b"abc"[..]), true); + s.put_file("/out.gif", Arc::from(&b"g"[..]), false); + let f = s.files(); + assert_eq!(f, vec![("/clip.mp4".to_string(), 3, 1000.0, true), ("/out.gif".to_string(), 1, 1000.0, false)]); + assert_eq!(s.file("/clip.mp4").as_deref(), Some(&b"abc"[..])); + } + + #[test] + fn config_store_and_file_ops() { + let s = store(); + let c = WebConfig::new(s.clone()); + assert_eq!(c.read("prefs.json"), None); + c.write("prefs.json", "{\"a\":1}").unwrap(); + assert_eq!(c.read("prefs.json").as_deref(), Some("{\"a\":1}")); + c.remove("prefs.json").unwrap(); + assert_eq!(c.read("prefs.json"), None); + // Config entries are not files. + c.write("session.lock", "{}").unwrap(); + assert!(s.files().is_empty()); + let fs = c.files(); + fs.write(Path::new("/EffectCraft Auto-Save/P auto-save 1.ecproj"), b"{}").unwrap(); + fs.write(Path::new("/EffectCraft Auto-Save/P auto-save 2.ecproj"), b"{}").unwrap(); + fs.write(Path::new("/other.ecproj"), b"{}").unwrap(); + let mut names: Vec = fs.list(Path::new("/EffectCraft Auto-Save")).into_iter().map(|e| e.0).collect(); + names.sort(); + assert_eq!(names, ["P auto-save 1.ecproj", "P auto-save 2.ecproj"]); + assert!(fs.is_file(Path::new("/other.ecproj"))); + fs.remove(Path::new("/other.ecproj")).unwrap(); + assert!(!fs.is_file(Path::new("/other.ecproj"))); + assert!(fs.remove(Path::new("/other.ecproj")).is_err()); + let pending = s.lock().take_pending(); + let keys: Vec<&str> = pending.iter().map(|p| p.key.as_str()).collect(); + assert_eq!( + keys, + [ + "config/prefs.json", + "config/session.lock", + "files/EffectCraft Auto-Save/P auto-save 1.ecproj", + "files/EffectCraft Auto-Save/P auto-save 2.ecproj", + "files/other.ecproj" + ] + ); + assert_eq!(pending[0].data, None, "removed"); + assert_eq!(pending[4].data, None, "removed"); + } + + #[test] + fn autosave_and_recovery_run_in_the_browser_store() { + use effectcraft_engine::Session; + use serde_json::json; + let s = store(); + let mut a = Session { config: Some(Arc::new(WebConfig::new(s.clone()))), ..Default::default() }; + a.load_settings(); + assert!(a.begin_recovery().is_none()); + a.execute("file.newProject", json!({})).unwrap(); + a.execute("comp.new", json!({"name": "C"})).unwrap(); + // Untitled: the auto-save lands in the store's file table. + let p = a.autosave_now().unwrap(); + assert_eq!(p, "/EffectCraft Auto-Save/Untitled Project auto-save 1.ecproj"); + assert!(s.file(&p).is_some()); + // The tab closes without a clean exit: the next visit offers the auto-save. + let mut b = Session { config: Some(Arc::new(WebConfig::new(s.clone()))), ..Default::default() }; + b.load_settings(); + let r = b.begin_recovery().expect("recovery"); + assert!(r.autosave.is_some()); + // Increment and Save sees the store's files. + let fs = WebConfig::new(s.clone()); + fs.files().write(Path::new("/A.ecproj"), b"{}").unwrap(); + let next = effectcraft_engine::autosave::increment_path("/A.ecproj", |p| fs.files().is_file(Path::new(p))); + assert_eq!(next, "/A 2.ecproj"); + } + + #[test] + fn snapshot_meta_round_trips() { + let m = SnapshotMeta { path: Some("/p.ecproj".into()), dirty: true, state: Some(serde_json::json!({"activeComp": 3})), saved: 12.5 }; + let t = serde_json::to_string(&m).unwrap(); + assert_eq!(serde_json::from_str::(&t).unwrap(), m); + assert_eq!(serde_json::from_str::("{}").unwrap(), SnapshotMeta::default()); + } +} diff --git a/apps/effectcraft-web/src/worker.rs b/apps/effectcraft-web/src/worker.rs new file mode 100644 index 00000000..2d96f641 --- /dev/null +++ b/apps/effectcraft-web/src/worker.rs @@ -0,0 +1,186 @@ +//! Renders and analyses in Web Workers ([`effectcraft_engine::offload`]). +//! +//! **Page side** — [`WorkerOffload`], the session's [`Offload`]: serializes the job, ships the +//! footage files it reads and the request to a worker from the host's pool (`js/host.js` +//! `workerRun`), and turns the worker's messages back into [`WorkerReply`]s in the job's +//! [`Inbox`]. Rendered files come back as their bytes and download like any render. +//! +//! **Worker side** — `web/worker.js` instantiates the same wasm module (the page's compiled +//! `WebAssembly.Module`, so nothing is compiled twice) and calls [`worker_init`], then +//! [`worker_file`] per footage file and [`worker_job`] per job: a plain engine session (footage +//! from memory, expressions, the exporter) runs it blocking and posts replies as it goes. Each +//! worker has its own memory: no shared-memory threads, so this works on stable Rust. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::offload::{Inbox, Offload, WorkerReply, WorkerRequest}; +use wasm_bindgen::JsCast; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen(module = "/js/host.js")] +extern "C" { + #[wasm_bindgen(js_name = workerRun)] + fn worker_run(id: f64, json: &str, files: js_sys::Array, base: &str, on_message: &js_sys::Function); + #[wasm_bindgen(js_name = workerCancel)] + fn worker_cancel(id: f64); + #[wasm_bindgen(js_name = workerStats)] + pub fn stats() -> JsValue; +} + +type OnMessage = Closure; + +thread_local! { + /// Message handlers of running jobs (dropped when the job ends). + static HANDLERS: RefCell> = RefCell::new(HashMap::new()); + /// The worker's session and its media pool. + static SESSION: RefCell> = const { RefCell::new(None) }; + static POOL: RefCell>> = const { RefCell::new(None) }; +} + +/// The page's offload: one worker per running job (idle workers are reused). +pub struct WorkerOffload; + +impl Offload for WorkerOffload { + fn start(&self, req: WorkerRequest, inbox: Arc) -> Result<(), String> { + let id = req.id; + let files = js_sys::Array::new(); + for p in &req.files { + if let Some(d) = crate::files::get(p) { + files.push(&js_sys::Array::of2(&JsValue::from_str(p), &js_sys::Uint8Array::from(&d[..]))); + } + } + let json = serde_json::to_string(&req).map_err(|e| e.to_string())?; + let on: OnMessage = Closure::new(move |kind: String, json: JsValue, path: JsValue, bytes: JsValue| { + match kind.as_str() { + "reply" => { + let Some(t) = json.as_string() else { return }; + match serde_json::from_str::(&t) { + Ok(r) => { + let done = matches!(r, WorkerReply::Done); + inbox.push(r); + if done { + end(id); + } + } + Err(e) => log::warn!("worker reply: {e}"), + } + } + "file" => { + if let (Some(p), Ok(b)) = (path.as_string(), bytes.dyn_into::()) { + crate::files::add_output(&p, b.to_vec().into()); + } + } + _ => { + inbox.push(WorkerReply::Failed { error: json.as_string().unwrap_or_else(|| "worker failed".into()) }); + inbox.push(WorkerReply::Done); + end(id); + } + } + crate::repaint(); + }); + worker_run(id as f64, &json, files, &crate::page_url(""), on.as_ref().unchecked_ref()); + HANDLERS.with(|h| h.borrow_mut().insert(id, on)); + Ok(()) + } + + fn cancel(&self, id: u64) { + worker_cancel(id as f64); + end(id); + } +} + +/// Drop a job's handler (after the current call returns). +fn end(id: u64) { + wasm_bindgen_futures::spawn_local(async move { + HANDLERS.with(|h| h.borrow_mut().remove(&id)); + }); +} + +// ---------------------------------------------------------------- worker side + +fn post(msg: &JsValue, transfer: Option<&js_sys::Array>) { + let scope: web_sys::DedicatedWorkerGlobalScope = js_sys::global().unchecked_into(); + let r = match transfer { + Some(t) => scope.post_message_with_transfer(msg, t), + None => scope.post_message(msg), + }; + if let Err(e) = r { + log::warn!("worker: postMessage: {e:?}"); + } +} + +fn obj(pairs: &[(&str, JsValue)]) -> JsValue { + let o = js_sys::Object::new(); + for (k, v) in pairs { + let _ = js_sys::Reflect::set(&o, &JsValue::from_str(k), v); + } + o.into() +} + +/// Set up the worker's engine session (called once by `web/worker.js`). +#[wasm_bindgen(js_name = workerInit)] +pub fn worker_init() { + crate::set_worker(); + eframe::WebLogger::init(log::LevelFilter::Info).ok(); + let pool = Arc::new(effectcraft_media::MediaPool::new()); + POOL.with(|p| *p.borrow_mut() = Some(pool.clone())); + // Rendered files go back to the page (transferred, not copied). + let sink: Arc = Arc::new(|path: &str, data: Vec| { + let bytes = js_sys::Uint8Array::from(&data[..]); + let buf = bytes.buffer(); + post(&obj(&[("type", "file".into()), ("path", path.into()), ("bytes", buf.clone().into())]), Some(&js_sys::Array::of1(&buf))); + }); + let s = Session { + services: Arc::new(crate::files::WebServices), + footage: pool.clone(), + importer: Some(Arc::new(crate::files::WebImporter { pool })), + expr: Some(Arc::new(effectcraft_expr::Expressions)), + expr_check: Some(effectcraft_expr::check_syntax), + exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(sink) })), + ..Default::default() + }; + SESSION.with(|c| *c.borrow_mut() = Some(s)); +} + +/// A footage file for the jobs that follow. +#[wasm_bindgen(js_name = workerFile)] +pub fn worker_file(path: String, bytes: js_sys::Uint8Array) { + let data: Arc<[u8]> = bytes.to_vec().into(); + crate::files::STORE.put_file(&path, data.clone(), false); + POOL.with(|p| { + if let Some(p) = p.borrow().as_ref() { + p.add_bytes(&path, data); + } + }); +} + +/// Run one job (a JSON [`WorkerRequest`]) and post its replies. +#[wasm_bindgen(js_name = workerJob)] +pub fn worker_job(json: String) { + let send: effectcraft_engine::offload::Post = Rc::new(|r: WorkerReply| { + let done = matches!(r, WorkerReply::Done); + let t = serde_json::to_string(&r).unwrap_or_default(); + post(&obj(&[("type", "reply".into()), ("json", t.into()), ("done", done.into())]), None); + }); + let req: WorkerRequest = match serde_json::from_str(&json) { + Ok(r) => r, + Err(e) => { + send(WorkerReply::Failed { error: format!("bad request: {e}") }); + send(WorkerReply::Done); + return; + } + }; + SESSION.with(|c| { + let mut c = c.borrow_mut(); + let Some(s) = c.as_mut() else { + send(WorkerReply::Failed { error: "worker not initialised".into() }); + send(WorkerReply::Done); + return; + }; + effectcraft_engine::offload::run_request(s, req, &send); + }); +} diff --git a/apps/effectcraft-web/tests/smoke.mjs b/apps/effectcraft-web/tests/smoke.mjs index de2f7f53..3ba19343 100644 --- a/apps/effectcraft-web/tests/smoke.mjs +++ b/apps/effectcraft-web/tests/smoke.mjs @@ -4,10 +4,15 @@ // cargo xtask web --serve 8765 & # build + serve /web/dist // node apps/effectcraft-web/tests/smoke.mjs --url http://127.0.0.1:8765/ --out /tmp/ec-web // -// Steps: load (timing), demo comp in the viewer (screenshot, viewer pixels), `render.frame` -// through `window.effectcraft`, Render Queue GIF and PNG-sequence (.zip) exports (downloads), File ▸ Save As (download), -// re-opening the saved project through `effectcraft.addFile`. Prints a JSON report; exits non-zero -// on failure. `--headed` shows the browser. +// Steps: load (timing), the GPU path (WebGPU → GPU compositor, viewer frames on the GPU), demo +// comp in the viewer (screenshot, viewer pixels), `render.frame` through `window.effectcraft`, +// Render Queue GIF and PNG-sequence (.zip) exports (downloads), a background Render Queue job in +// a Web Worker while the page stays responsive (event-loop gaps measured), audio (the +// AudioContext starts on a user gesture; preview playback feeds it), File ▸ Save As (download), +// re-opening the saved project through `effectcraft.addFile`, persistence across a reload (the +// session, imported media, settings, a project saved to browser storage, Open Recent) and an +// offline reload through the service worker. Prints a JSON report; exits non-zero on failure. +// `--headed` shows the browser. import { spawn } from "node:child_process"; import { mkdirSync, writeFileSync, readdirSync, statSync, mkdtempSync } from "node:fs"; import { join } from "node:path"; @@ -85,6 +90,22 @@ async function js(expr, timeout = 300000) { return r.result.value; } +const check = (cond, msg) => { + if (!cond) throw new Error(`check failed: ${msg}`); +}; + +// Wait until the app runs (after a navigation or reload). +async function ready(ms = 180000) { + await until("!!(window.effectcraftLoad && (window.effectcraftLoad.readyMs || window.effectcraftLoad.error))", ms); + const load = await js("window.effectcraftLoad"); + if (load.error) throw new Error(load.error); + return load; +} + +async function idle() { + await until(`effectcraft.execute("renderQueue.list", {}).then(r => !r.rendering)`, 300000, 200); +} + async function shot(name) { const r = await send("Page.captureScreenshot", { format: "png" }); const p = join(out, `${name}.png`); @@ -121,11 +142,11 @@ try { await send("Emulation.setDeviceMetricsOverride", { width: 1600, height: 1000, deviceScaleFactor: 1, mobile: false }); const t0 = Date.now(); await send("Page.navigate", { url }); - await until("!!(window.effectcraftLoad && (window.effectcraftLoad.readyMs || window.effectcraftLoad.error))", 180000); - report.load = await js("window.effectcraftLoad"); + report.load = await ready(); report.load.wallMs = Date.now() - t0; - if (report.load.error) throw new Error(report.load.error); report.info = await js("effectcraft.info()"); + // Storage: OPFS where the browser writes it (Chrome does), else IndexedDB. + check(["opfs", "indexeddb"].includes(report.info.storage?.backend), `storage backend ${JSON.stringify(report.info.storage)}`); // The demo comp in the viewer: wait for the first viewer frame (renderMs > 0). const ms = await until("effectcraft.inspect().then(i => i.renderMs > 0)", 120000, 500); @@ -133,6 +154,15 @@ try { const insp = await js("effectcraft.inspect()"); report.steps.viewer = { waitMs: ms, renderMs: insp.renderMs, activeComp: insp.activeComp, window: insp.window, screenshot: await shot("01-demo") }; + // GPU: with WebGPU, eframe runs on it and the compositor renders viewer frames on the GPU + // (no readback); without it, WebGL2 / CPU. + const info = await js("effectcraft.info()"); + report.steps.gpu = { webgpu: info.webgpu, backend: info.backend, ...info.gpu }; + if (info.webgpu && info.backend === "BrowserWebGpu") { + check(info.gpu && info.gpu.compositor, "GPU compositor on WebGPU"); + check(info.gpu.viewerOnGpu, "viewer frames on the GPU"); + } + // Comp pixels through the API. const f = await js("effectcraft.renderFrame({max_side: 480}).then(r => ({w: r.width, h: r.height, pngBytes: atob(r.png).length}))"); report.steps.renderFrame = f; @@ -145,12 +175,74 @@ try { await js(`effectcraft.execute("renderQueue.render", {})`); const gif = await waitDownload((n) => n.endsWith(".gif")); report.steps.gifExport = { ...gif, ms: Date.now() - rqT }; + await idle(); // An image sequence arrives as one .zip. await js(`effectcraft.execute("renderQueue.setRender", {index: 1, render: false})`); await js(`effectcraft.request("ui.menu.invoke", {id: "renderQueue.add", params: {format: "png", output: "seq_[####].png", resolution: 0.125, timeSpan: "custom", start: 0, end: 0.2}})`); await js(`effectcraft.execute("renderQueue.render", {})`); report.steps.pngSequence = await waitDownload((n) => n.endsWith(".zip")); + await idle(); + + // A background render (`wait: false`, what the Render button does) runs in a Web Worker: the + // page keeps its event loop. Measure the longest gap between 10 ms timers while it renders. + await js(`effectcraft.execute("renderQueue.setRender", {index: 2, render: false})`); + await js(`effectcraft.request("ui.menu.invoke", {id: "renderQueue.add", params: {format: "gif", output: "bg_[compName].gif", resolution: 0.5, timeSpan: "custom", start: 0, end: 4}})`); + report.steps.backgroundRender = await js(`(async () => { + const t0 = performance.now(); + const r = await effectcraft.execute("renderQueue.render", {wait: false}); + const callMs = performance.now() - t0; + let maxGapMs = 0, last = performance.now(), ticks = 0, progress = 0; + for (;;) { + await new Promise((res) => setTimeout(res, 10)); + const now = performance.now(); + maxGapMs = Math.max(maxGapMs, now - last); + last = now; + if (++ticks % 5) continue; + const l = await effectcraft.execute("renderQueue.list", {}); + if (l.progress) progress = Math.max(progress, l.progress.done); + if (!l.rendering) break; + } + return {startedRendering: r.rendering, callMs, renderMs: performance.now() - t0, maxGapMs, ticks, progressFrames: progress, workers: effectcraft.info().workers}; + })()`); + const bg = report.steps.backgroundRender; + bg.download = await waitDownload((n) => n.startsWith("bg_") && n.endsWith(".gif")); + check(bg.startedRendering, "render started in the background"); + check(bg.maxGapMs < 400, `event loop blocked ${bg.maxGapMs} ms during the render`); + check(bg.progressFrames > 0, "progress reported while rendering"); + + // Audio: the AudioContext starts on a user gesture. + report.steps.audio = { before: (await js("effectcraft.info()")).audio }; + check(report.steps.audio.before.state !== "running", "audio context not running before a gesture"); + // (a click in the window's bottom-right corner, the status bar: modifier keys alone don't + // count as user activation) + await send("Input.dispatchMouseEvent", { type: "mousePressed", x: 1590, y: 995, button: "left", clickCount: 1 }); + await send("Input.dispatchMouseEvent", { type: "mouseReleased", x: 1590, y: 995, button: "left", clickCount: 1 }); + await until("effectcraft.info().audio.state === 'running'", 10000); + report.steps.audio.afterGesture = (await js("effectcraft.info()")).audio; + // A 2 s tone in the comp; preview playback feeds the output and the meters. + await js(`(async () => { + const rate = 48000, n = 2 * rate, b = new ArrayBuffer(44 + n * 4), v = new DataView(b); + const w = (o, t) => [...t].forEach((c, i) => v.setUint8(o + i, c.charCodeAt(0))); + w(0, "RIFF"); v.setUint32(4, 36 + n * 4, true); w(8, "WAVE"); w(12, "fmt "); v.setUint32(16, 16, true); + v.setUint16(20, 1, true); v.setUint16(22, 2, true); v.setUint32(24, rate, true); v.setUint32(28, rate * 4, true); + v.setUint16(32, 4, true); v.setUint16(34, 16, true); w(36, "data"); v.setUint32(40, n * 4, true); + for (let i = 0; i < n; i++) { const x = Math.round(Math.sin(2 * Math.PI * 440 * i / rate) * 0.5 * 32767); v.setInt16(44 + 4 * i, x, true); v.setInt16(46 + 4 * i, x, true); } + return effectcraft.addFile(new File([b], "tone.wav")); + })()`); + await until(`effectcraft.execute("layer.addItem", {item: "tone.wav", time: 0}).then(() => true, () => false)`, 20000, 300); + await js(`effectcraft.request("ui.playback", {action: "stop"})`); + await js(`effectcraft.execute("time.set", {time: 0})`); + await js(`effectcraft.request("ui.playback", {action: "play"})`); + await sleep(1500); + const pb = await js(`effectcraft.request("ui.playback", {action: "status"})`); + const au = (await js("effectcraft.info()")).audio; + await js(`effectcraft.request("ui.playback", {action: "stop"})`); + report.steps.audio.playing = { playback: pb, output: au }; + check(pb.audio, "audio preview running"); + check(au.posted > 0.5 * au.sampleRate, `audio fed (${au.posted} frames)`); + check(au.played > 0, "audio clock advancing"); + check(Math.max(...(pb.levelsDb || [-99])) > -20, `meters (${JSON.stringify(pb.levelsDb)})`); // Save As → .ecproj download; reopen it. await js(`effectcraft.execute("file.saveAs", {path: "/smoke.ecproj"})`); @@ -162,6 +254,60 @@ try { if (!comps) throw new Error("reopened project has no active comp"); report.steps.reopen = { ...opened, activeComp: comps }; report.steps.final = { screenshot: await shot("02-after") }; + + // Persistence: the session, imported media, settings and a project saved to browser storage + // survive a reload; File ▸ Open Recent reopens the project. + await js(`effectcraft.execute("layer.newSolid", {name: "Persisted Solid", color: "#22aa66"})`); + await js(`effectcraft.execute("prefs.set", {key: "general.recentItems", value: 7})`); + const saved = await js(`effectcraft.saveToBrowser("/persist-test.ecproj")`); + await js(`effectcraft.execute("layer.newSolid", {name: "Unsaved Solid"})`); + await sleep(1500); // the session snapshot is taken at most every second + await js("effectcraft.flush()"); + const before = await js("effectcraft.listStored()"); + const tr = Date.now(); + await send("Page.reload", {}); + await sleep(500); + const load2 = await ready(); + const after = await js("effectcraft.listStored()"); + const info2 = await js("effectcraft.info()"); + const layers = await js(`effectcraft.execute("comp.info", {}).then(c => c.layers.map(l => l.name))`); + const recentItems = await js(`effectcraft.execute("prefs.get", {key: "general.recentItems"})`); + const rec = await js(`effectcraft.execute("file.recoveryInfo", {})`); + const paths = after.files.map((f) => f.path); + report.steps.persistence = { saved, reloadMs: Date.now() - tr, load: load2, restored: info2.restored, storage: info2.storage, layers, recentItems, recent: rec.recentProjects, stored: paths, usage: after.usage, filesBefore: before.files.length }; + check(info2.restored, "session restored after reload"); + check(layers.includes("Persisted Solid") && layers.includes("Unsaved Solid"), `layers after reload: ${layers}`); + check(paths.includes("/tone.wav") && paths.includes("/persist-test.ecproj"), `stored files: ${paths}`); + check(recentItems === 7, `setting after reload: ${recentItems}`); + check(rec.recentProjects[0] === "/persist-test.ecproj", `recent projects: ${rec.recentProjects}`); + await js(`effectcraft.execute("file.openRecent", {index: 0})`); + const reopened = await js(`effectcraft.execute("comp.info", {}).then(c => c.layers.map(l => l.name))`); + check(reopened.includes("Persisted Solid") && !reopened.includes("Unsaved Solid"), `Open Recent: ${reopened}`); + // The imported tone is still in the project and decodes from storage after the reload. + const tone = await js(`effectcraft.execute("layer.addItem", {item: "tone.wav", time: 0}).then(() => "ok", (e) => String(e))`); + check(tone === "ok", `tone.wav after reload: ${tone}`); + report.steps.persistence.openRecent = reopened; + + // PWA: manifest, service worker in control, and an offline reload. + const pwa = await js(`(async () => { + const m = await (await fetch("manifest.webmanifest")).json(); + const reg = await navigator.serviceWorker.getRegistration(); + const keys = await caches.keys(); + const cached = keys.length ? (await (await caches.open(keys[0])).keys()).length : 0; + return {manifest: m.name, icons: m.icons.length, display: m.display, sw: !!(reg && reg.active), controlled: !!navigator.serviceWorker.controller, caches: keys, cached}; + })()`); + check(pwa.sw && pwa.cached > 4, `service worker: ${JSON.stringify(pwa)}`); + await send("Network.enable", {}); + await send("Network.emulateNetworkConditions", { offline: true, latency: 0, downloadThroughput: -1, uploadThroughput: -1 }); + await send("Page.reload", {}); + await sleep(500); + const offline = await ready(); + const info3 = await js("effectcraft.info()"); + await send("Network.emulateNetworkConditions", { offline: false, latency: 0, downloadThroughput: -1, uploadThroughput: -1 }); + report.steps.pwa = { ...pwa, offlineLoad: offline, offlineControlled: info3.serviceWorker, crossOriginIsolated: info3.crossOriginIsolated }; + check(info3.serviceWorker, "offline reload served by the service worker"); + check(info3.crossOriginIsolated, "cross-origin isolated under the service worker"); + report.steps.final2 = { screenshot: await shot("03-offline") }; } catch (e) { failed = true; report.error = String(e && e.stack || e); diff --git a/apps/effectcraft-web/web/audio-worklet.js b/apps/effectcraft-web/web/audio-worklet.js new file mode 100644 index 00000000..97756c52 --- /dev/null +++ b/apps/effectcraft-web/web/audio-worklet.js @@ -0,0 +1,44 @@ +// EffectCraft audio output (AudioWorklet): plays interleaved stereo blocks posted by the page +// (src/audio.rs feeds them from the preview mixdown). Running dry plays silence, reported back +// as underrun frames. +class EffectcraftOutput extends AudioWorkletProcessor { + constructor() { + super(); + this.queue = []; + this.offset = 0; + this.stopped = false; + this.port.onmessage = (e) => { + if (e.data === "stop") this.stopped = true; + else this.queue.push(e.data); + }; + } + + process(_inputs, outputs) { + const out = outputs[0]; + const l = out[0]; + const r = out[1] ?? out[0]; + let i = 0; + while (i < l.length && this.queue.length) { + const b = this.queue[0]; + const n = Math.min(l.length - i, (b.length - this.offset) >> 1); + for (let k = 0; k < n; k++) { + l[i + k] = b[this.offset + 2 * k]; + r[i + k] = b[this.offset + 2 * k + 1]; + } + i += n; + this.offset += 2 * n; + if (this.offset >= b.length) { + this.queue.shift(); + this.offset = 0; + } + } + if (i < l.length) { + l.fill(0, i); + r.fill(0, i); + if (!this.stopped) this.port.postMessage({ underrun: l.length - i }); + } + return !this.stopped; + } +} + +registerProcessor("effectcraft-output", EffectcraftOutput); diff --git a/apps/effectcraft-web/web/index.html b/apps/effectcraft-web/web/index.html index 3e6cbf06..7c6c6c3c 100644 --- a/apps/effectcraft-web/web/index.html +++ b/apps/effectcraft-web/web/index.html @@ -4,8 +4,12 @@ + + EffectCraft + + + + + + + + + + + + + + + + + + + diff --git a/crates/svg/tests/fixtures/paths-gradients.svg.attribution b/crates/svg/tests/fixtures/paths-gradients.svg.attribution new file mode 100644 index 00000000..d4b8758f --- /dev/null +++ b/crates/svg/tests/fixtures/paths-gradients.svg.attribution @@ -0,0 +1,7 @@ +asset: paths-gradients.svg +title: SVG test fixture: path data, gradients, CSS styles and use +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft contributors (Claude agent, SVG import) +notes: Hand-written for the effectcraft-svg parser and Create Shapes from Vector Layer tests. diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index 70e08ae0..33ffacae 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and Roto Brush segmentation (graph cut, matting)" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization, Roto Brush segmentation (graph cut, matting) and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 00000000..72cb2f17 --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,470 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + tries += 1; + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; + if !o.fix_points { + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } + } + } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-6 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 00000000..30b019e5 --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 00000000..6956ac2b --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 00000000..b719cc8e --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,486 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 00000000..76c0637f --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,380 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 00000000..85edd2bd --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,953 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise() + } + fn huber(&self) -> f64 { + self.noise() + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise() / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + lap("tripod"); + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); + if cancelled() { + return err("cancelled"); + } + lap("keyframe adjustment"); + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + lap("resection"); + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + lap("final adjustment"); + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 00000000..4b5a8346 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,438 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index 629097cf..1c6e6a15 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,6 +18,7 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs index 6659c77d..3d1d627c 100644 --- a/crates/ui-egui/src/audio.rs +++ b/crates/ui-egui/src/audio.rs @@ -1,8 +1,9 @@ //! Real-time audio preview. //! //! The frontend owns no audio API: the native app supplies an [`AudioDevice`] (cpal) through -//! [`crate::Hooks::audio_device`]. During preview playback an [`AudioPlayback`] runs a feeder -//! thread that mixes the composition ahead of the device with the same mixdown export uses +//! [`crate::Hooks::audio_device`], the web app one on Web Audio. During preview playback an +//! [`AudioPlayback`] runs a feeder thread (wasm32: [`AudioPlayback::pump`] every UI frame) that +//! mixes the composition ahead of the device with the same mixdown export uses //! (`effectcraft_render::audio::mix_comp`: Audio switch, solo, Audio Levels, audio effects) and //! pushes it into an [`AudioFeed`]; the device callback pulls from the feed. //! @@ -35,6 +36,9 @@ pub trait AudioDevice: Send { fn latency_frames(&self) -> u64 { 0 } + /// Called every UI frame while playing: devices without an audio thread of their own (Web + /// Audio) move queued samples to the output here. + fn pump(&mut self) {} } /// Opens the default output device (`None` when there is none). @@ -160,10 +164,18 @@ pub struct AudioPlayback { device: Box, stop: Arc, thread: Option>, + /// wasm32 (no threads): what the feeder thread would own, run by [`AudioPlayback::pump`]. + feeder: Option<(RenderSource, ItemId, i64)>, pub rate: u32, pub span: PlaySpan, } +/// Audio kept queued ahead of the device: ~200 ms (a feeder thread tops it up continuously); +/// wasm32 tops it up once per UI frame, so it keeps ~350 ms to ride out slow frames. +fn queue_target(rate: u32) -> usize { + if cfg!(target_arch = "wasm32") { rate as usize * 7 / 20 } else { rate as usize / 5 } +} + impl AudioPlayback { /// Start mixing `comp` over `span` into `device`. pub fn start( @@ -185,13 +197,18 @@ impl AudioPlayback { for _ in 0..prime { cursor = feed_block(&src, comp, &feed, &span, cursor, rate); } + if cfg!(target_arch = "wasm32") { + let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, feeder: Some((src, comp, cursor)), rate, span }; + p.device.start(feed)?; + p.pump(); + return Ok(p); + } let thread = { let (feed, stop) = (feed.clone(), stop.clone()); std::thread::Builder::new() .name("ec-audio-feed".into()) .spawn(move || { - // Keep ~200 ms queued ahead of the device. - let target = rate as usize / 5; + let target = queue_target(rate); while !stop.load(Ordering::Relaxed) { if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { cursor = feed_block(&src, comp, &feed, &span, cursor, rate); @@ -203,7 +220,19 @@ impl AudioPlayback { .map_err(|e| e.to_string())? }; device.start(feed.clone())?; - Ok(AudioPlayback { feed, device, stop, thread: Some(thread), rate, span }) + Ok(AudioPlayback { feed, device, stop, thread: Some(thread), feeder: None, rate, span }) + } + + /// Every UI frame while playing: on wasm32 mix ahead into the feed (what the feeder thread + /// does elsewhere), then let the device take what it needs. + pub fn pump(&mut self) { + if let Some((src, comp, cursor)) = &mut self.feeder { + let target = queue_target(self.rate); + while self.feed.queued_frames() < target && !self.feed.finished.load(Ordering::Relaxed) { + *cursor = feed_block(src, *comp, &self.feed, &self.span, *cursor, self.rate); + } + } + self.device.pump(); } /// The comp sample index audible now. diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index cd052c3e..91d7c335 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -35,10 +35,11 @@ pub enum PanelKind { Markers, MaskInterpolation, ScriptConsole, + EssentialGraphics, } impl PanelKind { - pub const ALL: [PanelKind; 25] = [ + pub const ALL: [PanelKind; 26] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -64,6 +65,7 @@ impl PanelKind { PanelKind::Markers, PanelKind::MaskInterpolation, PanelKind::ScriptConsole, + PanelKind::EssentialGraphics, ]; pub fn title(self) -> &'static str { match self { @@ -92,6 +94,7 @@ impl PanelKind { PanelKind::Markers => "Markers", PanelKind::MaskInterpolation => "Mask Interpolation", PanelKind::ScriptConsole => "Script Console", + PanelKind::EssentialGraphics => "Essential Graphics", } } pub fn id(self) -> String { diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 62239f19..fd628351 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -7,6 +7,11 @@ //! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] renders them //! on the UI thread between egui frames, within a time budget. //! +//! With the disk cache on (Settings ▸ Media & Disk Cache), every CPU-rendered frame is also +//! written to disk under a content key (the comp's content hash, frame, scale, view and render +//! options), and a job first looks there: frames survive restarts and project reopenings. The +//! timeline shows disk-only frames as a blue cache bar. +//! //! With Mercury GPU Acceleration (a [`Gpu`] on egui-wgpu's device and the project's renderer set //! to the GPU) frames stay on the GPU: the compositor writes an RGBA8 texture that the viewer //! registers with egui-wgpu and draws directly; pixels are read back only when something needs @@ -16,6 +21,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; use effectcraft_gpu::{DisplayFrame, Gpu}; @@ -118,10 +124,43 @@ pub struct RenderSource { pub layer_cache: Arc, /// GPU compositor on the viewer's device (Mercury GPU Acceleration), if available. pub gpu: Option, - /// Keep GPU frames on the GPU for the viewer (native egui-wgpu textures). Off by default: - /// Show Channel / exposure / region-of-interest drawing read the CPU frame texture, so GPU - /// frames are read back (the compositing still runs on the GPU). + /// Keep GPU frames on the GPU for the viewer (native egui-wgpu textures). Off on the + /// desktop: Show Channel / exposure / region-of-interest drawing read the CPU frame texture, + /// so GPU frames are read back (the compositing still runs on the GPU). On in the browser + /// while the viewer draws plainly, because there frames can't be read back synchronously + /// (`EffectcraftApp::gpu_display`). pub gpu_display: bool, + /// The persistent disk cache, when enabled. + pub disk: Option>, +} + +/// Content keys of (project revision, comp), computed once per revision. +type ContentKeys = Arc>>; + +fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 { + if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) { + return k; + } + let k = disk_cache::comp_content_key(project, ItemId(comp)); + if let Ok(mut m) = keys.lock() { + if m.len() > 256 { + m.clear(); + } + m.insert((revision, comp), k); + } + k +} + +fn opts_hash(o: &RenderOpts) -> u64 { + let mut h = disk_cache::Hash128::default(); + h.write(format!("{o:?}").as_bytes()); + h.finish() as u64 +} + +/// Disk key of a viewer frame (render options included: region of interest, draft…). +fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 { + let content = content_key(keys, project, key.revision, key.comp); + disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash) } pub struct Frames { @@ -135,6 +174,7 @@ pub struct Frames { ctx: Option, /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, + content_keys: ContentKeys, } impl Default for Frames { @@ -149,6 +189,7 @@ impl Default for Frames { pool: rayon::ThreadPoolBuilder::new().num_threads(threads).thread_name(|i| format!("ec-frame-{i}")).build().expect("frame pool"), ctx: None, last_ms: Arc::new(Mutex::new(0.0)), + content_keys: Arc::default(), } } } @@ -175,6 +216,11 @@ impl Frames { } } + /// The egui context (to request repaints from callbacks). + pub fn context(&self) -> Option { + self.ctx.clone() + } + pub fn get(&self, k: &FrameKey) -> Option { self.cache.lock().ok()?.map.get(k).cloned() } @@ -195,6 +241,21 @@ impl Frames { v } + /// Frames of (revision, comp, scale) in the disk cache but not in RAM — the blue cache bar. + /// `opts` are the viewer's render options (part of the disk key). + pub fn disk_frames(&self, src: &RenderSource, revision: u64, comp: u64, scale: u32, view: u64, frames: i64, opts: &RenderOpts) -> Vec { + let Some(dc) = &src.disk else { return vec![] }; + let ram: HashSet = self.cached_frames(revision, comp, scale).into_iter().collect(); + let oh = opts_hash(opts); + (0..frames.clamp(0, 100_000)) + .filter(|f| !ram.contains(f)) + .filter(|f| { + let key = FrameKey { revision, comp, frame: *f, scale, view }; + dc.contains(disk_cache::Kind::Frame, frame_disk_key(&self.content_keys, &src.project, &key, oh)) + }) + .collect() + } + /// Change the RAM preview cache budget (Settings ▸ Memory & CPU), evicting the oldest /// frames when it shrinks. pub fn set_budget(&self, bytes: usize) { @@ -278,7 +339,14 @@ impl Frames { } fn worker(&self) -> Worker { - Worker { queue: self.queue.clone(), cache: self.cache.clone(), inflight: self.inflight.clone(), ctx: self.ctx.clone(), last: self.last_ms.clone() } + Worker { + queue: self.queue.clone(), + cache: self.cache.clone(), + inflight: self.inflight.clone(), + ctx: self.ctx.clone(), + last: self.last_ms.clone(), + content_keys: self.content_keys.clone(), + } } /// Render queued frames on this thread, most important first, until the queue is empty or @@ -306,12 +374,13 @@ struct Worker { inflight: Arc>>, ctx: Option, last: Arc>, + content_keys: ContentKeys, } impl Worker { /// Render the queued job that matters most right now. `false` when the queue was empty. fn run_next(&self) -> bool { - let Worker { queue, cache, inflight, ctx, last } = self; + let Worker { queue, cache, inflight, ctx, last, content_keys } = self; let job = { let Ok(mut q) = queue.lock() else { return false }; // Urgent first (newest), then prefetch in request order. @@ -325,19 +394,18 @@ impl Worker { job }; let t0 = web_time::Instant::now(); - let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); - r.expr = job.src.expr.as_deref(); - r.cache = Some(&job.src.layer_cache); - r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); - // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no - // adapter, or a frame the GPU cannot finish here) the CPU renders it. - let gpu_frame = match (&job.src.gpu, r.active_accel()) { - (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), - _ => None, - }; - let ci = match gpu_frame { - Some(f) => FrameImage::Gpu(Arc::new(f)), - None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts)))); + let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k)); + let ci = match from_disk { + Some(f) => FrameImage::Cpu(Arc::new(egui::ColorImage::from_rgba_premultiplied([f.width as usize, f.height as usize], &f.rgba))), + None => { + let ci = self.render_job(&job); + if let (Some((dc, k)), FrameImage::Cpu(img)) = (&disk, &ci) { + let rgba: Vec = img.pixels.iter().flat_map(|c| c.to_array()).collect(); + dc.put_frame(*k, img.size[0] as u32, img.size[1] as u32, &rgba); + } + ci + } }; if job.urgent && let Ok(mut l) = last.lock() @@ -361,6 +429,23 @@ impl Worker { } true } + + fn render_job(&self, job: &Job) -> FrameImage { + let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); + r.expr = job.src.expr.as_deref(); + r.cache = Some(&job.src.layer_cache); + r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); + // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no + // adapter, or a frame the GPU cannot finish here) the CPU renders it. + let gpu_frame = match (&job.src.gpu, r.active_accel()) { + (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), + _ => None, + }; + match gpu_frame { + Some(f) => FrameImage::Gpu(Arc::new(f)), + None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + } + } } struct Job { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index a9763d30..912f2e68 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -140,6 +140,8 @@ pub struct EffectcraftApp { /// Last displayed image (Info panel colour, eyedroppers, histograms). GPU frames fill it on /// demand ([`EffectcraftApp::viewer_pixels`]). pub(crate) viewer_image: Option>, + /// wasm32: an asynchronous readback of a GPU viewer frame (key asked for, result). + viewer_readback: (Option, Arc)>>>), /// egui-wgpu's device, once the first frame arrives (desktop and WebGPU). pub(crate) wgpu: Option, /// The GPU compositor on that device (None: no usable adapter → CPU only). @@ -200,6 +202,7 @@ impl EffectcraftApp { viewer_native: None, viewer_shown: None, viewer_image: None, + viewer_readback: Default::default(), wgpu: None, gpu: None, gpu_checked: false, @@ -325,10 +328,23 @@ impl EffectcraftApp { expr: self.session.expr.clone(), layer_cache: self.session.layer_cache.clone(), gpu: self.gpu.clone(), - gpu_display: false, + gpu_display: self.gpu_display(), + disk: self.session.disk_cache.clone(), } } + /// Keep GPU-composited viewer frames on the GPU (drawn from their texture). The browser can't + /// read frames back (its thread can't wait for the GPU), so there the viewer shows GPU frames + /// directly whenever it draws them plainly (RGB, no exposure, no region of interest); the + /// desktop reads them back (see [`frames::RenderSource::gpu_display`]). + fn gpu_display(&self) -> bool { + let v = &self.session.state.viewer; + cfg!(target_arch = "wasm32") + && v.channel == effectcraft_engine::viewer::Channel::Rgb + && v.exposure == 0.0 + && self.session.state.region_of_interest.is_none() + } + /// Set up the GPU compositor on egui-wgpu's device (once). Without an adapter that runs /// compute shaders (WebGL2, software GL) everything stays on the CPU. pub(crate) fn init_gpu(&mut self, frame: &eframe::Frame) { @@ -360,14 +376,38 @@ impl EffectcraftApp { /// The viewer's current pixels as 8-bit premultiplied RGBA, reading a GPU frame back the /// first time something asks for it. + /// + /// The browser can't wait for the GPU: there the first call starts an asynchronous readback + /// and returns `None`; a later call (the next frame) has the pixels. pub fn viewer_pixels(&mut self) -> Option> { - if self.viewer_image.is_none() - && let (Some((_, _, f)), Some(g)) = (&self.viewer_native, &self.gpu) - && self.viewer_shown.as_ref().map(|s| s.1) == self.viewer_native.as_ref().map(|n| n.1) - && let Some(bytes) = g.read_display(f) - { + let to_image = |bytes: Vec, w: u32, h: u32| { let px = bytes.chunks_exact(4).map(|c| egui::Color32::from_rgba_premultiplied(c[0], c[1], c[2], c[3])).collect(); - self.viewer_image = Some(Arc::new(egui::ColorImage::new([f.width as usize, f.height as usize], px))); + Arc::new(egui::ColorImage::new([w as usize, h as usize], px)) + }; + if self.viewer_image.is_none() + && let (Some((_, key, f)), Some(g)) = (&self.viewer_native, &self.gpu) + && self.viewer_shown.as_ref().map(|s| s.1) == Some(*key) + { + if cfg!(target_arch = "wasm32") { + let ready = self.viewer_readback.1.lock().ok().and_then(|mut r| r.take_if(|r| r.0 == *key)); + if let Some((_, img)) = ready { + self.viewer_image = Some(img); + } else if self.viewer_readback.0 != Some(*key) { + self.viewer_readback.0 = Some(*key); + let (slot, k, (w, h)) = (self.viewer_readback.1.clone(), *key, (f.width, f.height)); + let ctx = self.frames.context(); + g.read_display_async(f, move |bytes| { + if let (Some(b), Ok(mut s)) = (bytes, slot.lock()) { + *s = Some((k, to_image(b, w, h))); + } + if let Some(c) = ctx { + c.request_repaint(); + } + }); + } + } else if let Some(bytes) = g.read_display(f) { + self.viewer_image = Some(to_image(bytes, f.width, f.height)); + } } self.viewer_image.clone() } @@ -423,13 +463,11 @@ impl EffectcraftApp { self.request_frame_with(comp, frame, scale, true); } - fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { - let Some(c) = self.session.project.comp(comp) else { return }; - let key = self.frame_key(comp, frame, scale); - let t = c.frame_rate.tick_of(frame); + /// Render options of viewer frames of `comp` at `scale`. + pub fn frame_opts(&self, comp: ItemId, scale: f64) -> RenderOpts { let (draft, _) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting); let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp)); - let opts = RenderOpts { + RenderOpts { scale, motion_blur: true, guides: true, @@ -437,7 +475,15 @@ impl EffectcraftApp { view: self.session.view_camera(comp), roi, backend: effectcraft_engine::render::Backend::Auto, - }; + proxy: Default::default(), + } + } + + fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { + let Some(c) = self.session.project.comp(comp) else { return }; + let key = self.frame_key(comp, frame, scale); + let t = c.frame_rate.tick_of(frame); + let opts = self.frame_opts(comp, scale); if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); } else { @@ -534,7 +580,8 @@ impl EffectcraftApp { queued += 1; } } - if let Some(a) = &self.audio { + if let Some(a) = &mut self.audio { + a.pump(); self.meter.update(a.feed.take_peaks(), now); } else if self.meter.active() { self.meter.update([0.0; 2], now); @@ -740,12 +787,22 @@ impl EffectcraftApp { self.session.poll_mask_track(); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } + // Analyses running in a worker (the web app). + if !self.session.offloaded.is_empty() { + self.session.poll_offload(); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // Warp Stabilizer: finish analyses and start queued (re-)analyses in the background. if self.session.warp_job.is_some() || !self.session.warp_pending.is_empty() { self.session.poll_warp(true); self.session.poll_roto(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 4b7366cb..eed7e639 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -794,6 +794,9 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), + "essential.exportTemplate" => ("path", Ask::Save("Template.ectemplate")), + "essential.importTemplate" => ("path", Ask::Open(&["ectemplate"])), + "file.setProxy" => ("path", Ask::Open(&["mp4", "mov", "m4v", "mkv", "webm", "png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"])), _ => return None, }; if params.get(key).is_some() @@ -810,7 +813,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, @@ -833,6 +836,10 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 03afd94c..20142225 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -574,6 +574,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, effectcraft_engine::effects::roto::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, @@ -823,7 +824,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let comp_name = app.session.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time: app.session.time(), expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time: app.session.time(), expr: snap_expr.as_deref(), footage: None }; let hdr = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); p.text(pos2(hdr.min.x + 10.0, hdr.center().y), Align2::LEFT_CENTER, format!("{} • {}", comp_name, layer.name), Tokens::ui(11.5), t.text_dim); p.line_segment([hdr.left_bottom(), hdr.right_bottom()], Stroke::new(1.0, t.separator)); @@ -950,6 +951,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), diff --git a/crates/ui-egui/src/panels/essential.rs b/crates/ui-egui/src/panels/essential.rs new file mode 100644 index 00000000..f4c565c5 --- /dev/null +++ b/crates/ui-egui/src/panels/essential.rs @@ -0,0 +1,427 @@ +//! Essential Graphics panel (Window ▸ Essential Graphics). +//! +//! - **Primary** composition, **Solo Supported Properties** (the timeline lists only what can +//! be exposed) and the template name. +//! - The controls: drag a property name from the timeline onto the list (or Animation ▸ Add +//! Property to Essential Graphics); each shows as a text / colour / slider / checkbox / point / +//! angle / dropdown control editing the source property; rename by double-clicking the name; +//! reorder with ▲▼, remove with ✕; groups and comments from Add Formatting; Media +//! Replacement for the selected footage layer. +//! - **Export Template…** writes an open `.ectemplate`. +//! - With a precomp layer that has Essential Properties selected, its instance values: edit them +//! (overrides), Revert or Push to Comp. +//! +//! Every control runs an engine `essential.*` / `prop.set` command and registers an automation +//! id (`essential.primary`, `essential.solo`, `essential.name`, `essential.control..value`, +//! `essential.instance..value`, `essential.export`…). + +use effectcraft_engine::keyframe::Value as KV; +use effectcraft_engine::project::essential::{self, ControlType, EgControl, EgKind}; +use effectcraft_engine::project::{ItemId, ItemKind, LayerSource, ParamUi, Property}; +use egui::{Rect, RichText}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::panels::DragPayload; + +type Actions = Vec<(String, Value)>; + +/// The comp the panel edits: the Primary comp, else the active one. +fn primary(app: &EffectcraftApp) -> Option { + let s = &app.session; + s.state.essential_primary.filter(|c| s.project.comp(*c).is_some()).or(s.active_comp_id()) +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let ctx = ui.ctx().clone(); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink(8.0)).layout(egui::Layout::top_down(egui::Align::Min))); + let mut out = None; + egui::ScrollArea::vertical().id_salt("eg-panel").auto_shrink([false, false]).show(&mut child, |ui| out = Some(body(app, ui))); + let Some((actions, invoke)) = out else { return }; + for (id, p) in actions { + if let Err(e) = app.session.execute(&id, p) { + app.ui.status = e.to_string(); + } + } + for (id, p) in invoke { + if let Err(e) = crate::menus::invoke(app, &ctx, &id, p) { + app.ui.status = e; + } + } +} + +fn body(app: &mut EffectcraftApp, ui: &mut egui::Ui) -> (Actions, Actions) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let mut actions: Actions = vec![]; + let mut invoke: Vec<(String, Value)> = vec![]; + let Some(cid) = primary(app) else { + ui.label(RichText::new("Open a composition to build Essential Graphics").color(t.text_faint)); + return (actions, invoke); + }; + let project = app.session.project.clone(); + let comp_name = project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + + // ---- Primary + Solo. + ui.horizontal(|ui| { + ui.label(RichText::new("Primary:").color(t.text_dim)); + let r = egui::ComboBox::from_id_salt("eg-primary").selected_text(&comp_name).width(150.0).show_ui(ui, |ui| { + for (id, _) in project.comps() { + let name = project.item(*id).map(|i| i.name.clone()).unwrap_or_default(); + if ui.selectable_label(*id == cid, name).clicked() { + actions.push(("essential.setPrimary".into(), json!({"comp": id.0}))); + } + } + }); + app.auto.add("essential.primary", r.response.rect, &comp_name); + }); + let solo = app.session.state.essential_solo; + let r = ui.add(egui::Button::new("Solo Supported Properties").selected(solo)); + app.auto.add("essential.solo", r.rect, "Solo Supported Properties"); + if r.clicked() { + actions.push(("essential.soloSupported".into(), json!({"on": !solo}))); + } + // ---- Name. + let eg = project.comp(cid).and_then(|c| c.essential.clone()).unwrap_or_default(); + let shown_name = if eg.name.is_empty() { comp_name.clone() } else { eg.name.clone() }; + let name_id = egui::Id::new("eg-name-buf"); + let mut name: String = ctx.data(|d| d.get_temp(name_id)).unwrap_or_else(|| shown_name.clone()); + let r = ui.add(egui::TextEdit::singleline(&mut name).hint_text("Template name").desired_width(f32::INFINITY)); + app.auto.add("essential.name", r.rect, &shown_name); + if r.has_focus() { + ctx.data_mut(|d| d.insert_temp(name_id, name.clone())); + } else { + if r.lost_focus() && name.trim() != shown_name { + actions.push(("essential.setName".into(), json!({"comp": cid.0, "name": name.trim()}))); + } + ctx.data_mut(|d| d.remove::(name_id)); + } + ui.separator(); + + // ---- Controls (drop target for timeline properties). + let list = ui.scope(|ui| { + ui.set_min_height(60.0); + if eg.controls.is_empty() { + ui.label(RichText::new("Drag properties here from the timeline,\nor use Animation > Add Property to Essential Graphics.").color(t.text_faint)); + } + controls(app, ui, cid, &eg.controls, None, &mut actions); + }); + let drop_rect = list.response.rect.expand2(egui::vec2(0.0, 4.0)); + app.auto.add("essential.dropZone", drop_rect, "Drag properties here"); + if let Some(payload) = egui::DragAndDrop::payload::(&ctx) + && let DragPayload::Property { layer, prop } = *payload + && ui.rect_contains_pointer(drop_rect) + { + ui.painter().rect_stroke(drop_rect, 2.0, egui::Stroke::new(2.0, t.accent), egui::StrokeKind::Inside); + if ctx.input(|i| i.pointer.any_released()) { + let comp = app.session.active_comp_id().map(|c| c.0); + actions.push(("essential.addProperty".into(), json!({"comp": comp, "layer": layer, "prop": prop}))); + egui::DragAndDrop::clear_payload(&ctx); + } + } + ui.separator(); + + // ---- Bottom buttons. + ui.horizontal_wrapped(|ui| { + let r = ui.menu_button("Add Formatting", |ui| { + if ui.button("Add Group").clicked() { + actions.push(("essential.addGroup".into(), json!({"comp": cid.0}))); + ui.close(); + } + if ui.button("Add Comment").clicked() { + actions.push(("essential.addComment".into(), json!({"comp": cid.0}))); + ui.close(); + } + }); + app.auto.add("essential.addFormatting", r.response.rect, "Add Formatting"); + let footage_layer = app.session.active_comp().and_then(|c| { + app.session.state.selected_layers.iter().filter_map(|l| c.layer(*l)).find(|l| matches!(l.source, LayerSource::Footage { .. })).map(|l| l.id.0) + }); + let r = ui.add_enabled(footage_layer.is_some(), egui::Button::new("Media Replacement")); + app.auto.add("essential.addMedia", r.rect, "Add Media Replacement"); + if r.clicked() + && let Some(l) = footage_layer + { + actions.push(("essential.addMedia".into(), json!({"layer": l}))); + } + let r = ui.add_enabled(!eg.controls.is_empty(), egui::Button::new("Export Template…")); + app.auto.add("essential.export", r.rect, "Export Template"); + if r.clicked() { + invoke.push(("essential.exportTemplate".into(), json!({"comp": cid.0}))); + } + }); + + // ---- The selected instance's Essential Properties. + instance(app, ui, &mut actions); + (actions, invoke) +} + +/// One list of controls (top level or a group's children). +fn controls(app: &mut EffectcraftApp, ui: &mut egui::Ui, cid: ItemId, list: &[EgControl], group: Option, actions: &mut Actions) { + let t = app.tokens; + let n = list.len(); + for (i, c) in list.iter().enumerate() { + let row = ui.horizontal(|ui| { + // Reorder / remove. + for (label, auto, enabled, to) in [("▲", "up", i > 0, i.saturating_sub(1)), ("▼", "down", i + 1 < n, i + 1)] { + let r = ui.add_enabled(enabled, egui::Button::new(RichText::new(label).size(9.0)).small().frame(false)); + app.auto.add(&format!("essential.control.{}.{auto}", c.id), r.rect, label); + if r.clicked() { + actions.push(("essential.move".into(), json!({"comp": cid.0, "control": c.id, "group": group, "index": to}))); + } + } + match &c.kind { + EgKind::Group { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name).strong(), actions); + } + EgKind::Comment { text } => { + let shown = EgControl { name: text.clone(), ..c.clone() }; + name_label(app, ui, cid, &shown, RichText::new(text).italics().color(t.text_dim), actions); + } + EgKind::Property { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name), actions); + let src = essential::source_prop(&app.session.project, cid, c).map(|(l, p)| (l.id.0, l.layer_time(app.session.time()), p.clone())); + match src { + Some((layer, lt, p)) => { + let v = p.value_at(lt); + if let Some(nv) = value_widget(app, ui, &format!("essential.control.{}.value", c.id), &p, &v) { + actions.push(( + "prop.set".into(), + json!({"comp": cid.0, "layer": layer, "prop": p.uid, "value": nv, "merge": format!("eg-{}", c.id)}), + )); + } + } + None => { + ui.label(RichText::new("(missing)").color(t.danger)); + } + } + } + EgKind::Media { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name), actions); + let item = essential::source_media(&app.session.project, cid, c); + let name = item.and_then(|i| app.session.project.item(i)).map(|i| i.name.clone()).unwrap_or_default(); + ui.label(RichText::new(format!("▣ {name}")).color(t.text_dim)); + } + } + let r = ui.add(egui::Button::new(RichText::new("×").size(10.0)).small().frame(false)).on_hover_text("Remove"); + app.auto.add(&format!("essential.control.{}.remove", c.id), r.rect, "Remove"); + if r.clicked() { + actions.push(("essential.remove".into(), json!({"comp": cid.0, "control": c.id}))); + } + }); + app.auto.add(&format!("essential.control.{}.row", c.id), row.response.rect, &c.name); + if let EgKind::Group { children } = &c.kind { + ui.indent(("eg-group", c.id), |ui| controls(app, ui, cid, children, Some(c.id), actions)); + } + } +} + +/// A control's name; double-click to rename (comments: edit the text). +fn name_label(app: &mut EffectcraftApp, ui: &mut egui::Ui, cid: ItemId, c: &EgControl, text: RichText, actions: &mut Actions) { + let edit_id = egui::Id::new(("eg-rename", c.id)); + let editing: Option = ui.ctx().data(|d| d.get_temp(edit_id)); + match editing { + Some(mut buf) => { + let r = ui.add(egui::TextEdit::singleline(&mut buf).desired_width(120.0)); + app.auto.add(&format!("essential.control.{}.rename", c.id), r.rect, &c.name); + r.request_focus(); + if r.lost_focus() || ui.input(|i| i.key_pressed(egui::Key::Enter)) { + if !buf.trim().is_empty() && buf.trim() != c.name { + actions.push(("essential.rename".into(), json!({"comp": cid.0, "control": c.id, "name": buf.trim()}))); + } + ui.ctx().data_mut(|d| d.remove::(edit_id)); + } else { + ui.ctx().data_mut(|d| d.insert_temp(edit_id, buf)); + } + } + None => { + let r = ui.add(egui::Label::new(text).sense(egui::Sense::click()).truncate()); + app.auto.add(&format!("essential.control.{}.name", c.id), r.rect, &c.name); + if r.double_clicked() { + ui.ctx().data_mut(|d| d.insert_temp(edit_id, c.name.clone())); + } + } + } +} + +/// The editor for a control's value; returns the new value (JSON for `prop.set`). +fn value_widget(app: &mut EffectcraftApp, ui: &mut egui::Ui, auto: &str, p: &Property, v: &KV) -> Option { + let ty = essential::control_type(p)?; + let mut out = None; + let rect; + match ty { + ControlType::Text => { + let cur = match v { + KV::Text(d) => d.text.clone(), + KV::Str(s) => s.clone(), + _ => String::new(), + }; + let id = egui::Id::new(("eg-text", auto)); + let mut buf: String = ui.ctx().data(|d| d.get_temp(id)).unwrap_or_else(|| cur.clone()); + let r = ui.add(egui::TextEdit::singleline(&mut buf).desired_width(140.0)); + if r.has_focus() { + ui.ctx().data_mut(|d| d.insert_temp(id, buf)); + } else { + if r.lost_focus() && buf != cur { + out = Some(json!(buf)); + } + ui.ctx().data_mut(|d| d.remove::(id)); + } + rect = r.rect; + } + ControlType::Color => { + let c = v.as_color(); + let mut rgba = egui::Rgba::from_rgba_unmultiplied(c[0], c[1], c[2], c[3]); + let r = egui::widgets::color_picker::color_edit_button_rgba(ui, &mut rgba, egui::widgets::color_picker::Alpha::Opaque); + if r.changed() { + let u = rgba.to_rgba_unmultiplied(); + out = Some(json!([u[0], u[1], u[2], u[3]])); + } + rect = r.rect; + } + ControlType::Checkbox => { + let mut on = v.as_bool(); + let r = ui.checkbox(&mut on, ""); + if r.changed() { + out = Some(json!(on)); + } + rect = r.rect; + } + ControlType::Dropdown => { + let opts = match &p.ui { + ParamUi::Popup { options } => options.clone(), + _ => vec![], + }; + let cur = v.as_enum() as usize; + let r = egui::ComboBox::from_id_salt(("eg-dd", auto)).selected_text(opts.get(cur).cloned().unwrap_or_default()).show_ui(ui, |ui| { + for (i, o) in opts.iter().enumerate() { + if ui.selectable_label(i == cur, o).clicked() { + out = Some(json!(i)); + } + } + }); + rect = r.response.rect; + } + ControlType::Point => { + let mut a = v.as_vec2(); + let r1 = ui.add(egui::DragValue::new(&mut a[0]).speed(1.0).max_decimals(1)); + let r2 = ui.add(egui::DragValue::new(&mut a[1]).speed(1.0).max_decimals(1)); + if r1.changed() || r2.changed() { + out = Some(json!(a)); + } + rect = r1.rect.union(r2.rect); + } + ControlType::Angle => { + let mut a = v.as_f64(); + let r = ui.add(egui::DragValue::new(&mut a).speed(1.0).suffix("°").max_decimals(1)); + if r.changed() { + out = Some(json!(a)); + } + rect = r.rect; + } + ControlType::Slider | ControlType::Media => { + let mut x = v.as_f64(); + let r = match p.ui { + ParamUi::Slider { slider_min, slider_max, .. } => ui.add(egui::Slider::new(&mut x, slider_min..=slider_max).max_decimals(2)), + _ => ui.add(egui::DragValue::new(&mut x).speed(0.5).max_decimals(2)), + }; + if r.changed() { + out = Some(json!(x)); + } + rect = r.rect; + } + } + app.auto.add(auto, rect, &p.name); + out +} + +/// Essential Properties of the selected precomp layer: per-instance values, overrides marked ●, +/// Revert and Push to Comp. +fn instance(app: &mut EffectcraftApp, ui: &mut egui::Ui, actions: &mut Actions) { + let t = app.tokens; + let Some(comp) = app.session.active_comp().cloned() else { return }; + let Some(layer) = app.session.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| essential::group(l).is_some()).cloned() else { return }; + let LayerSource::Comp { item: src } = layer.source else { return }; + let Some(eg) = app.session.project.comp(src).and_then(|c| c.essential.clone()) else { return }; + let over = essential::overridden(&layer); + let lt = layer.layer_time(app.session.time()); + ui.separator(); + ui.label(RichText::new(format!("Essential Properties — {}", layer.name)).strong()); + let group = essential::group(&layer).cloned().unwrap_or_else(|| effectcraft_engine::project::PropGroup::new(0, "", "")); + for c in eg.flat() { + let m = essential::match_id(c.id); + let mut pr = None; + group.walk("", &mut |_, x| { + if x.match_id == m { + pr = Some(x.clone()); + } + }); + let Some(p) = pr else { continue }; + let is_over = over.contains(&p.uid); + ui.horizontal(|ui| { + ui.label(RichText::new(if is_over { "●" } else { "○" }).color(if is_over { t.accent } else { t.text_faint })).on_hover_text(if is_over { + "Overridden in this instance" + } else { + "Follows the source composition" + }); + ui.label(&c.name); + let auto = format!("essential.instance.{}.value", c.id); + match c.kind { + EgKind::Media { .. } => { + let cur = ItemId(p.value.as_f64() as u64); + let cur_name = app.session.project.item(cur).map(|i| i.name.clone()).unwrap_or_default(); + let items: Vec<(ItemId, String)> = app + .session + .project + .items + .values() + .filter(|i| matches!(&i.kind, ItemKind::Footage(f) if f.has_video)) + .map(|i| (i.id, i.name.clone())) + .collect(); + let r = egui::ComboBox::from_id_salt(("eg-media", layer.id.0, c.id)).selected_text(cur_name).show_ui(ui, |ui| { + for (id, name) in &items { + if ui.selectable_label(*id == cur, name).clicked() { + actions.push(("essential.set".into(), json!({"layer": layer.id.0, "control": c.id, "item": id.0}))); + } + } + }); + app.auto.add(&auto, r.response.rect, &c.name); + } + _ => { + let v = p.value_at(lt); + if let Some(nv) = value_widget(app, ui, &auto, &p, &v) { + actions.push(( + "essential.set".into(), + json!({"layer": layer.id.0, "control": c.id, "value": nv, "merge": format!("egi-{}-{}", layer.id.0, c.id)}), + )); + } + } + } + if is_over { + let r = ui.small_button("Revert"); + app.auto.add(&format!("essential.instance.{}.revert", c.id), r.rect, "Revert"); + if r.clicked() { + actions.push(("essential.revert".into(), json!({"layer": layer.id.0, "control": c.id}))); + } + let r = ui.small_button("Push"); + app.auto.add(&format!("essential.instance.{}.push", c.id), r.rect, "Push to Comp"); + if r.clicked() { + actions.push(("essential.pushToComp".into(), json!({"layer": layer.id.0, "control": c.id}))); + } + } + }); + } + ui.horizontal(|ui| { + let any = !over.is_empty(); + let r = ui.add_enabled(any, egui::Button::new("Push All to Comp")); + app.auto.add("essential.instance.pushAll", r.rect, "Push Override Values to Source"); + if r.clicked() { + actions.push(("essential.pushToComp".into(), json!({"layer": layer.id.0}))); + } + let r = ui.add_enabled(any, egui::Button::new("Revert All")); + app.auto.add("essential.instance.revertAll", r.rect, "Revert All"); + if r.clicked() { + actions.push(("essential.revert".into(), json!({"layer": layer.id.0}))); + } + }); +} diff --git a/crates/ui-egui/src/panels/expr_bar.rs b/crates/ui-egui/src/panels/expr_bar.rs new file mode 100644 index 00000000..09d4a608 --- /dev/null +++ b/crates/ui-egui/src/panels/expr_bar.rs @@ -0,0 +1,95 @@ +//! The expression error bar: a strip at the bottom of the Composition viewer listing failing +//! expressions of the comp at the current time (`expr.errors`), with ◀ ▶ to step through them. +//! Clicking the message selects the layer and reveals the property in the timeline. +//! Automation ids: `viewer.exprErrors`, `viewer.exprErrors.prev`, `viewer.exprErrors.next`. + +use effectcraft_engine::commands::expr_tools::{self, ExprError}; +use effectcraft_engine::project::ItemId; +use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; +use serde_json::json; + +use crate::EffectcraftApp; +use crate::theme::Tokens; + +/// Height of the bar. +pub const HEIGHT: f32 = 22.0; + +#[derive(Clone, Default)] +struct Cache { + key: (u64, u64, i64), + errors: Vec, + /// Shown error (index). + current: usize, +} + +fn cache_id() -> egui::Id { + egui::Id::new("expr-error-bar") +} + +/// The comp's failing expressions (recomputed when the project or the time changes; kept while +/// playing). +fn errors(app: &EffectcraftApp, ctx: &egui::Context, cid: ItemId) -> Cache { + let t = app.session.time_of(cid); + let key = (app.session.revision, cid.0, t.0); + let mut c: Cache = ctx.data(|d| d.get_temp(cache_id())).unwrap_or_default(); + let stale = c.key != key && !(app.playback.playing && c.key.1 == cid.0); + if stale { + c.errors = expr_tools::errors(&app.session, cid, t); + c.key = key; + c.current = c.current.min(c.errors.len().saturating_sub(1)); + ctx.data_mut(|d| d.insert_temp(cache_id(), c.clone())); + } + c +} + +/// Height the bar needs for `cid` (0 when every expression evaluates). +pub fn height(app: &EffectcraftApp, ctx: &egui::Context, cid: ItemId) -> f32 { + if app.session.expr.is_none() || errors(app, ctx, cid).errors.is_empty() { 0.0 } else { HEIGHT } +} + +pub fn draw(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, cid: ItemId) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let mut c = errors(app, &ctx, cid); + if c.errors.is_empty() { + return; + } + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, Color32::from_rgb(0x4a, 0x3a, 0x12)); + let warn = Color32::from_rgb(0xf5, 0xb0, 0x30); + p.text(pos2(rect.min.x + 10.0, rect.center().y), Align2::LEFT_CENTER, "⚠", Tokens::ui(13.0), warn); + let n = c.errors.len(); + let i = c.current.min(n - 1); + // ◀ i/n ▶ on the right. + let next = Rect::from_center_size(pos2(rect.max.x - 14.0, rect.center().y), vec2(16.0, 16.0)); + let count = Rect::from_center_size(pos2(rect.max.x - 44.0, rect.center().y), vec2(40.0, 16.0)); + let prev = Rect::from_center_size(pos2(rect.max.x - 74.0, rect.center().y), vec2(16.0, 16.0)); + for (r, label, auto, d) in [(prev, "◀", "viewer.exprErrors.prev", n - 1), (next, "▶", "viewer.exprErrors.next", 1)] { + let resp = ui.interact(r, egui::Id::new(("expr-bar", auto)), Sense::click()); + p.text(r.center(), Align2::CENTER_CENTER, label, Tokens::ui(11.0), if resp.hovered() { Color32::WHITE } else { t.text }); + app.auto.add(auto, r, label); + if resp.clicked() { + c.current = (i + d) % n; + ctx.data_mut(|dd| dd.insert_temp(cache_id(), c.clone())); + } + } + p.text(count.center(), Align2::CENTER_CENTER, format!("{}/{n}", i + 1), Tokens::ui(11.0), t.text); + let comp_name = app.session.project.item(cid).map(|it| it.name.clone()).unwrap_or_default(); + let e = c.errors[i].clone(); + let msg_rect = Rect::from_min_max(pos2(rect.min.x + 26.0, rect.min.y), pos2(prev.min.x - 6.0, rect.max.y)); + let resp = ui.interact(msg_rect, egui::Id::new("expr-bar-msg"), Sense::click()).on_hover_text("Click to reveal the expression in the timeline"); + p.with_clip_rect(msg_rect).text( + pos2(msg_rect.min.x, rect.center().y), + Align2::LEFT_CENTER, + e.describe(&comp_name), + Tokens::ui(11.5), + if resp.hovered() { Color32::WHITE } else { t.text }, + ); + app.auto.add("viewer.exprErrors", msg_rect, &e.describe(&comp_name)); + if resp.clicked() { + let _ = app.session.execute("layer.select", json!({"comp": cid.0, "layers": [e.layer]})); + let _ = app.session.execute("prop.select", json!({"comp": cid.0, "layer": e.layer, "prop": e.prop, "selectKeys": false})); + let _ = crate::menus::frontend(app, &ctx, "timeline.revealProps", json!({"props": [{"layer": e.layer, "prop": e.prop}]})); + app.ui.timeline.expr_closed.remove(&e.prop); + } +} diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 4b731cca..3e051948 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -108,6 +108,16 @@ fn has(p: &Value, keys: &[&str]) -> bool { } /// If `id` is a dialog command invoked without its parameters, open its form and return true. +/// The import includes a Photoshop document. +fn is_layered(p: &Value) -> bool { + let psd = |v: &Value| v.as_str().is_some_and(|s| matches!(s.rsplit('.').next().map(str::to_ascii_lowercase).as_deref(), Some("psd" | "psb"))); + match p.get("paths").or(p.get("path")) { + Some(Value::Array(a)) => a.iter().any(psd), + Some(v) => psd(v), + None => false, + } +} + pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let s = &app.session; let comp = s.active_comp(); @@ -116,6 +126,16 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let lt = layer.map(|l| l.layer_time(t)).unwrap_or(t); let base = p.clone(); let (title, fields): (String, Vec) = match id { + // Photoshop files: Import Kind (Footage / Composition / – Retain Layer Sizes). + "file.import" if !has(p, &["importAs"]) && is_layered(p) => ( + "Import Photoshop File".into(), + vec![Field::choice( + "importAs", + "Import Kind", + &[("Footage", json!("footage")), ("Composition", json!("composition")), ("Composition - Retain Layer Sizes", json!("compositionLayerSizes"))], + 1, + )], + ), "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); @@ -218,27 +238,48 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ), ], ), - "file.interpretFootage" if !has(p, &["frameRate", "alpha", "loop", "pixelAspect", "colorProfile"]) => { - let f = s.state.project_selection.first().and_then(|i| s.project.item(*i)).and_then(|it| match &it.kind { - effectcraft_engine::project::ItemKind::Footage(f) => Some(f.clone()), + "file.interpretFootage" | "file.interpretProxy" + if !has(p, &["frameRate", "alpha", "loop", "pixelAspect", "colorProfile", "fields", "invertAlpha", "matteColor", "linearLight"]) => + { + let proxy = id == "file.interpretProxy"; + let f = s.state.project_selection.first().and_then(|i| s.project.item(*i)).and_then(|it| match (&it.kind, &it.proxy) { + (_, Some(px)) if proxy => Some(px.footage.clone()), + (effectcraft_engine::project::ItemKind::Footage(f), _) if !proxy => Some(f.clone()), _ => None, }); let Some(f) = f else { return false }; + let hex = |c: [f32; 3]| format!("#{:02x}{:02x}{:02x}", (c[0] * 255.0).round() as u8, (c[1] * 255.0).round() as u8, (c[2] * 255.0).round() as u8); ( - "Interpret Footage".into(), + if proxy { "Interpret Footage: Proxy".into() } else { "Interpret Footage".into() }, vec![ - Field::num("frameRate", "Frame rate", f.frame_rate.as_f64()), + // Main Options ▸ Alpha. Field::choice( "alpha", "Alpha", - &[("Straight", json!("straight")), ("Premultiplied", json!("premultiplied")), ("Ignore", json!("ignore"))], + &[ + ("Interpret Straight - Unmatted", json!("straight")), + ("Interpret Premultiplied - Matted With Color", json!("premultiplied")), + ("Ignore", json!("ignore")), + ("Guess", json!("guess")), + ], f.alpha as usize, ), + Field::text("matteColor", "Matte color (premultiplied)", &hex(f.premul_color)), + Field::bool("invertAlpha", "Invert Alpha", f.invert_alpha), + // Main Options ▸ Frame Rate, Fields and Pulldown, Other Options. + Field::num("frameRate", "Assume this frame rate", f.frame_rate.as_f64()), + Field::choice( + "fields", + "Separate Fields", + &[("Off", json!("off")), ("Upper Field First", json!("upper")), ("Lower Field First", json!("lower"))], + f.fields as usize, + ), + Field::num("pixelAspect", "Pixel Aspect Ratio", f.pixel_aspect), Field::num("loop", "Loop (times)", f.loop_count as f64), - Field::num("pixelAspect", "Pixel aspect ratio", f.pixel_aspect), + // Color. Field::choice( "colorProfile", - "Color profile", + "Assign Profile", &[ ("Embedded / sRGB", json!("auto")), ("sRGB IEC61966-2.1", json!("srgb")), @@ -248,6 +289,7 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ], f.color_profile.map_or(0, |c| 1 + effectcraft_engine::project::ColorSpace::ALL.iter().position(|x| *x == c).unwrap_or(0)), ), + Field::bool("linearLight", "Interpret As Linear Light", f.linear_light), ], ) } diff --git a/crates/ui-egui/src/panels/layer_panel.rs b/crates/ui-egui/src/panels/layer_panel.rs index c20a0fba..20ccf358 100644 --- a/crates/ui-egui/src/panels/layer_panel.rs +++ b/crates/ui-egui/src/panels/layer_panel.rs @@ -138,7 +138,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let tex = match cached { Some(c) if c.key == key => Some(c), _ => { - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref(), footage: None }; let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() }); r.expr = app.session.expr.as_deref(); r.cache = Some(&app.session.layer_cache); diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 037d6b4c..2e6d03ae 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,11 +1,14 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; pub mod effect_controls; pub mod effects_presets; +pub mod essential; +pub mod expr_bar; pub mod flowchart; pub mod forms; pub mod fx_widgets; @@ -52,6 +55,8 @@ pub enum DragPayload { Item(u64), /// An effect id from Effects & Presets. Effect(String), + /// A property dragged from the timeline (to the Essential Graphics panel). + Property { layer: u64, prop: u64 }, } /// The current time formatted per project settings (timecode with `;` for drop-frame, or frames). @@ -96,6 +101,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Layer => layer_panel::show(app, ui, rect), PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), + PanelKind::EssentialGraphics => essential::show(app, ui, rect), PanelKind::Flowchart => flowchart::show(app, ui, rect), } } diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index ed185087..ce750fac 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -357,6 +357,24 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.auto.add(&format!("project.item.{}.twirl", id.0), tw, &it.name); } icons::paint(&lp, Rect::from_center_size(pos2(x0 + 18.0, r.center().y), vec2(14.0, 14.0)), item_icon(&it), t.text_dim); + // Proxy indicator: filled = the proxy is used, hollow = set but off. Click to switch. + if let Some(px) = it.proxy.as_ref() { + let pr = Rect::from_center_size(pos2(x0 + 4.0, r.center().y), vec2(9.0, 9.0)); + if px.enabled { + lp.rect_filled(pr, 1.0, t.text); + } else { + lp.rect_stroke(pr, 1.0, Stroke::new(1.0, t.text), egui::StrokeKind::Inside); + } + let presp = ui.interact(pr.expand(2.0).intersect(list), egui::Id::new(("pproxy", id.0)), Sense::click()).on_hover_text(format!( + "Proxy: {} ({})", + px.footage.path, + if px.enabled { "in use" } else { "off" } + )); + app.auto.add(&format!("project.item.{}.proxy", id.0), pr, if px.enabled { "Proxy in use" } else { "Proxy off" }); + if presp.clicked() { + actions.push(("file.useProxy".into(), json!({"item": id.0}))); + } + } let name_clip = Rect::from_min_max(pos2(x0 + 30.0, r.min.y), pos2(r.min.x + name_w + 4.0, r.max.y)).intersect(list); let name_rect = Rect::from_min_max(pos2(x0 + 28.0, r.min.y + 2.0), pos2(r.min.x + name_w + 2.0, r.max.y - 2.0)); // Inline edits (rename / comment). diff --git a/crates/ui-egui/src/panels/properties.rs b/crates/ui-egui/src/panels/properties.rs index fbd4e5e9..c5795685 100644 --- a/crates/ui-egui/src/panels/properties.rs +++ b/crates/ui-egui/src/panels/properties.rs @@ -69,7 +69,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let project = app.session.project.clone(); let expr = app.session.expr.clone(); let now = app.session.time(); - let ectx = EvalCtx { project: &project, comp_id: cid, comp: &comp, time: now, expr: expr.as_deref() }; + let ectx = EvalCtx { project: &project, comp_id: cid, comp: &comp, time: now, expr: expr.as_deref(), footage: None }; let x0 = rect.min.x + PAD; let w = rect.width() - 2.0 * PAD; // Vertical scrolling: content height is known after drawing, so clamp with last frame's. diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index 154dd4c2..c02090bc 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -77,6 +77,10 @@ fn settings_menu(it: &RenderQueueItem) -> Vec<(String, Value)> { v.push(("-".into(), Value::Null)); v.push((mark(s.motion_blur, "Motion Blur: On for Checked Layers"), json!({"motionBlur": !s.motion_blur}))); v.push((mark(s.skip_existing, "Skip Existing Files"), json!({"skipExisting": !s.skip_existing}))); + v.push(("-".into(), Value::Null)); + for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { + v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); + } v } @@ -108,7 +112,7 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } - OutputFormat::JpegSequence => { + OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); diff --git a/crates/ui-egui/src/panels/roto_tool.rs b/crates/ui-egui/src/panels/roto_tool.rs index 25354ee3..7047f238 100644 --- a/crates/ui-egui/src/panels/roto_tool.rs +++ b/crates/ui-egui/src/panels/roto_tool.rs @@ -68,7 +68,7 @@ pub fn draw_overlay( let ov = match cached { Some(c) if c.key == key => Some(c), _ => { - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time, expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time, expr: app.session.expr.as_deref(), footage: None }; let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() }); r.expr = app.session.expr.as_deref(); r.cache = Some(&app.session.layer_cache); diff --git a/crates/ui-egui/src/panels/text_panels.rs b/crates/ui-egui/src/panels/text_panels.rs index 48c762cb..fef958a7 100644 --- a/crates/ui-egui/src/panels/text_panels.rs +++ b/crates/ui-egui/src/panels/text_panels.rs @@ -56,7 +56,7 @@ pub fn text_target(app: &EffectcraftApp) -> Option { .iter() .filter_map(|id| comp.layer(*id)) .find(|l| matches!(l.source, effectcraft_engine::project::LayerSource::Text))?; - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time: app.session.time(), expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time: app.session.time(), expr: app.session.expr.as_deref(), footage: None }; effectcraft_engine::render::text::source_text(&ectx, layer).map(|d| TextTarget { layer: layer.id.0, doc: d, range: None }) } @@ -469,7 +469,7 @@ pub fn align_layers(app: &mut EffectcraftApp, op: &str) { let time = app.session.time(); let mut moves = vec![]; { - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref(), footage: None }; for l in comp.layers.iter().filter(|l| app.session.state.selected_layers.contains(&l.id)) { let Some(b) = effectcraft_engine::render::content_bounds(&ectx, l) else { continue }; let (m, _) = ectx.layer_to_comp(l); diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 46f0097a..10ad62e6 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -494,6 +494,19 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { continue; } rows.push(Row { layer: l.id, depth: 0, kind: RowKind::Layer }); + // Essential Graphics ▸ Solo Supported Properties: every property it can expose. + if app.session.state.essential_solo { + let mut v = vec![]; + for (i, c) in l.props.children.iter().enumerate() { + if c.match_id() == effectcraft_engine::project::essential::GROUP { + continue; + } + let g = PropGroup { children: vec![l.props.children[i].clone()], ..PropGroup::new(0, "", "") }; + collect_props(&g, &mut v, &|p| prop_visible(p, l) && effectcraft_engine::project::essential::control_type(p).is_some()); + } + rows.extend(v.into_iter().map(|uid| Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } })); + continue; + } if !tl.open_layers.contains(&l.id.0) { continue; } @@ -855,6 +868,31 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if let Some((a, b)) = run { draw_run(a, b); } + // Disk-cached frames not in RAM (blue), like After Effects. + if app.session.disk_cache.is_some() { + let frames = comp.frame_rate.frame_at(comp.duration); + let opts = app.frame_opts(cid, scale_key as f64 / 1000.0); + let disk = app.frames.disk_frames(&app.render_source(), app.session.revision, cid.0, scale_key, app.view_hash(cid), frames, &opts); + let draw_blue = |a: i64, b: i64| { + let x0 = tm.x(a as f64 * fd); + let x1 = tm.x((b + 1) as f64 * fd); + p.rect_filled(Rect::from_min_max(pos2(x0, cy0), pos2(x1, cy0 + 2.5)), 0.0, t.cache_blue); + }; + let mut run: Option<(i64, i64)> = None; + for f in disk { + run = match run { + Some((a, b)) if f == b + 1 => Some((a, f)), + Some((a, b)) => { + draw_blue(a, b); + Some((f, f)) + } + None => Some((f, f)), + }; + } + if let Some((a, b)) = run { + draw_blue(a, b); + } + } // Ticks + labels. // Label spacing in whole frames (AE: `00:15f`-style seconds:frames labels). let fps_i = fr.as_f64().round().max(1.0) as i64; @@ -1030,7 +1068,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.ui.timeline.scroll_y = app.ui.timeline.scroll_y.min(max_scroll); let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref(), footage: None }; let lp = p.with_clip_rect(rows_rect.intersect(Rect::from_min_max(rows_rect.min, pos2(graph_x0 - 1.0, rows_rect.max.y)))); let gp = p.with_clip_rect(rows_rect.intersect(Rect::from_min_max(pos2(graph_x0, rows_rect.min.y), rows_rect.max))); let mut y = rows_rect.min.y - app.ui.timeline.scroll_y; @@ -1495,6 +1533,40 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { lp.text(wr.center(), Align2::CENTER_CENTER, "⚠", Tokens::ui(12.0), Color32::from_rgb(0xf0, 0xa0, 0x30)); let _ = ui.interact(wr, egui::Id::new(("expr-err", uid)), Sense::hover()).on_hover_text(e); } + // ⓕ: the Expression Language menu inserts at the cursor. + let lang_r = Rect::from_center_size(pos2(cw.switches + 70.0, cy), vec2(16.0, 16.0)); + let lang = ui.interact(lang_r, egui::Id::new(("expr-lang", uid)), Sense::click()).on_hover_text("Expression Language Menu"); + lp.circle_stroke(lang_r.center(), 6.5, Stroke::new(1.0, if lang.hovered() { t.text } else { t.text_dim })); + lp.text(lang_r.center(), Align2::CENTER_CENTER, "f", Tokens::semibold(10.0), if lang.hovered() { t.text } else { t.text_dim }); + app.auto.add(&format!("timeline.prop.{uid}.exprLanguage"), lang_r, "Expression Language Menu"); + let mut picked: Option<&'static str> = None; + egui::Popup::menu(&lang).show(|ui| { + for (cat, items) in effectcraft_engine::commands::expr_tools::language_menu() { + ui.menu_button(cat, |ui| { + for (label, text) in items { + if ui.button(label).clicked() { + picked = Some(text); + ui.close(); + } + } + }); + } + }); + if let Some(text) = picked { + let edit_id = egui::Id::new(("expr-edit", uid)); + let buf_id = egui::Id::new(("expr-buf", uid)); + let cur: String = ctx.data(|d| d.get_temp(buf_id)).unwrap_or_else(|| ex.text.clone()); + let at = egui::text_edit::TextEditState::load(&ctx, edit_id) + .and_then(|st| st.cursor.char_range()) + .map(|r| r.primary.index.0) + .unwrap_or(cur.chars().count()) + .min(cur.chars().count()); + let byte = cur.char_indices().nth(at).map(|(b, _)| b).unwrap_or(cur.len()); + let mut next = cur.clone(); + next.insert_str(byte, text); + actions.push(("prop.setExpression".into(), json!({"layer": layer.id.0, "prop": uid, "expression": next}))); + ctx.data_mut(|d| d.remove::(buf_id)); + } // The editor in the time-graph area. ui.set_clip_rect(right_clip); gp.rect_filled(Rect::from_min_max(pos2(graph_x0, r.min.y), r.max), 0.0, Color32::from_rgb(0x1a, 0x1a, 0x1a)); @@ -1606,8 +1678,19 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { Tokens::ui(12.0), if prop.has_expression() { Color32::from_rgb(0xe8, 0x7c, 0x5c) } else { t.text }, ); - let name_resp = - ui.interact(Rect::from_min_max(pos2(name_x, r.min.y), pos2(cw.switches - 2.0, r.max.y)), egui::Id::new(("pname", uid)), Sense::click()); + let name_resp = ui.interact( + Rect::from_min_max(pos2(name_x, r.min.y), pos2(cw.switches - 2.0, r.max.y)), + egui::Id::new(("pname", uid)), + Sense::click_and_drag(), + ); + if name_resp.drag_started() { + // Drop on the Essential Graphics panel to expose the property. + egui::DragAndDrop::set_payload(&ctx, crate::panels::DragPayload::Property { layer: layer.id.0, prop: *uid }); + } + if name_resp.dragged() { + egui::Tooltip::always_open(ctx.clone(), ui.layer_id(), egui::Id::new("prop-drag-tip"), egui::PopupAnchor::Pointer) + .show(|ui| ui.label(&prop.name)); + } if name_resp.clicked() { actions.push(("prop.select".into(), json!({"layer": layer.id.0, "prop": uid, "add": ui.input(|i| i.modifiers.shift)}))); } @@ -1889,6 +1972,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Drop targets: footage/comps from the Project panel, effects from Effects & Presets. if let Some(payload) = egui::DragAndDrop::payload::(&ctx) + && !matches!(*payload, crate::panels::DragPayload::Property { .. }) && ui.rect_contains_pointer(rows_rect) { p.rect_stroke(rows_rect, 0.0, Stroke::new(2.0, t.accent), StrokeKind::Inside); @@ -1902,6 +1986,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("effect.apply".into(), json!({"effect": e, "layers": [l.0]}))); } } + crate::panels::DragPayload::Property { .. } => {} } egui::DragAndDrop::clear_payload(&ctx); } diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b2..364e114e 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index b0382d07..cdbd47b2 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -373,7 +373,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Bottom control bar. let bar_h = 30.0; let bar = Rect::from_min_max(pos2(rect.min.x, rect.max.y - bar_h), rect.max); - let full = Rect::from_min_max(pos2(rect.min.x, nav.max.y), pos2(rect.max.x, bar.min.y)); + // The expression error bar sits above the control bar while expressions fail. + let err_h = super::expr_bar::height(app, &ctx, cid); + let err_bar = Rect::from_min_max(pos2(rect.min.x, bar.min.y - err_h), pos2(rect.max.x, bar.min.y)); + let full = Rect::from_min_max(pos2(rect.min.x, nav.max.y), pos2(rect.max.x, err_bar.min.y)); let pasteboard = app.ui.viewer.pasteboard.map(|[r, g, b]| Color32::from_rgb(r, g, b)).unwrap_or(t.pasteboard); // View ▸ Switch View Layout: extra views (Top / Front / Right) beside the main view, which // keeps the overlays and the interaction. @@ -420,7 +423,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref(), footage: None }; // The frame (or snapshot) through Show Channel and exposure; ROI frames cover the region. vt::draw_frame(app, &ctx, &painter, comp_rect, cid, &ectx); painter.rect_stroke(comp_rect, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); @@ -938,6 +941,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { comp: app.session.project.comp(cid).unwrap_or(&comp), time, expr: app.session.expr.as_deref(), + footage: None, }; if let Some(m) = l2c(&e2, &l).0.inverse() { *i2 = m; @@ -1198,6 +1202,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); @@ -1211,6 +1218,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { vt::draw_snap(&ctx, &painter, &map, &ectx); vt::rulers(app, ui, &map, outer, area); vt::bottom_bar(app, ui, bar, zoom, fit, time, &comp); + if err_h > 0.0 { + super::expr_bar::draw(app, ui, err_bar, cid); + } } /// Parent-space position change per comp pixel of drag (x and y) for a layer: 2D layers map @@ -1450,6 +1460,7 @@ fn create_shape(app: &mut EffectcraftApp, tool: Tool, a: [f64; 2], b: [f64; 2], comp: &comp, time: app.session.time(), expr: None, + footage: None, }; let inv = l2c(&ectx, &l).0.inverse().unwrap_or(Mat3::IDENTITY); let p0 = inv.apply(gv2(cx - w / 2.0, cy - h / 2.0)); diff --git a/crates/ui-egui/src/panels/viewer_tools.rs b/crates/ui-egui/src/panels/viewer_tools.rs index cea240f8..36178036 100644 --- a/crates/ui-egui/src/panels/viewer_tools.rs +++ b/crates/ui-egui/src/panels/viewer_tools.rs @@ -361,19 +361,35 @@ pub(crate) fn draw_frame(app: &mut EffectcraftApp, ctx: &egui::Context, painter: painter.text(comp_rect.left_top() + vec2(6.0, 6.0), Align2::LEFT_TOP, "Snapshot", Tokens::ui(11.0), SNAP_COLOR); return; } - let Some((tex, k)) = app.viewer_tex.clone() else { return }; - if k.comp != cid.0 { - return; - } - let zoom = comp_rect.width() as f64 / ectx.comp.width.max(1) as f64; - let rect = match texture_roi(ctx) { - Some([x, y, w, h]) => Rect::from_min_size(comp_rect.min + vec2((x * zoom) as f32, (y * zoom) as f32), vec2((w * zoom) as f32, (h * zoom) as f32)), - None => comp_rect, + // A GPU frame (the web viewer): drawn straight from its texture; Show Channel / exposure + // read it back first (asynchronously in the browser: from the next frame on). + let gpu = matches!((&app.viewer_shown, &app.viewer_native), (Some(s), Some(n)) if s.0 == n.0 && s.1 == n.1 && n.1.comp == cid.0); + let rect = if gpu { + let id = app.viewer_shown.map(|s| s.0).expect("shown"); + if plain || app.viewer_pixels().is_none() { + if !plain { + ctx.request_repaint(); + } + painter.image(id, comp_rect, uv, Color32::WHITE); + return; + } + comp_rect + } else { + let Some((tex, k)) = app.viewer_tex.clone() else { return }; + if k.comp != cid.0 { + return; + } + let zoom = comp_rect.width() as f64 / ectx.comp.width.max(1) as f64; + let rect = match texture_roi(ctx) { + Some([x, y, w, h]) => Rect::from_min_size(comp_rect.min + vec2((x * zoom) as f32, (y * zoom) as f32), vec2((w * zoom) as f32, (h * zoom) as f32)), + None => comp_rect, + }; + if plain { + painter.image(tex.id(), rect, uv, Color32::WHITE); + return; + } + rect }; - if plain { - painter.image(tex.id(), rect, uv, Color32::WHITE); - return; - } let Some(src) = app.viewer_image.clone() else { return }; let key = { use std::hash::{Hash, Hasher}; diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index 5c82e916..c32ecb24 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -60,6 +60,8 @@ pub struct Tokens { pub cti: Color32, pub work_area: Color32, pub cache_green: Color32, + /// Timeline cache bar for frames held in the disk cache (not in RAM). + pub cache_blue: Color32, pub keyframe: Color32, pub keyframe_selected: Color32, pub pasteboard: Color32, @@ -108,6 +110,7 @@ impl Tokens { cti: Color32::from_rgb(0x2d, 0x8c, 0xeb), work_area: Color32::from_rgb(0x4a, 0x4a, 0x4a), cache_green: Color32::from_rgb(0x3c, 0xa6, 0x4c), + cache_blue: Color32::from_rgb(0x3a, 0x6f, 0xd8), keyframe: Color32::from_rgb(0xa8, 0xa8, 0xa8), keyframe_selected: Color32::from_rgb(0x3d, 0x8f, 0xf5), pasteboard: Color32::from_rgb(0x1a, 0x1a, 0x1a), diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 00000000..45374fba --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml new file mode 100644 index 00000000..62144b25 --- /dev/null +++ b/crates/vp9enc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-vp9enc" +description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md new file mode 100644 index 00000000..c8f781b8 --- /dev/null +++ b/crates/vp9enc/README.md @@ -0,0 +1,15 @@ +# effectcraft-vp9enc + +A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, +BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 +DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 +(mathematically lossless). + +Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March +2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode +info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) +and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The +boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec +tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR +Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an +external test oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs new file mode 100644 index 00000000..45db92dd --- /dev/null +++ b/crates/vp9enc/src/bool.rs @@ -0,0 +1,193 @@ +//! Boolean (arithmetic) encoder: the inverse of the VP9 boolean decoder (spec §9.2), written as +//! the classic carry-propagating range coder described in RFC 6386 §7.3 (VP8 and VP9 share the +//! bool coder). + +pub struct BoolEncoder { + out: Vec, + range: u32, + bottom: u32, + bit_count: i32, +} + +impl Default for BoolEncoder { + fn default() -> Self { + BoolEncoder::new() + } +} + +impl BoolEncoder { + pub fn new() -> Self { + BoolEncoder { out: Vec::new(), range: 255, bottom: 0, bit_count: 24 } + } + + fn add_one(&mut self) { + let mut i = self.out.len(); + while i > 0 { + i -= 1; + if self.out[i] == 255 { + self.out[i] = 0; + } else { + self.out[i] += 1; + return; + } + } + } + + /// Write `bit` with probability `prob`/256 of it being 0. + #[inline] + pub fn write(&mut self, bit: bool, prob: u8) { + let split = 1 + (((self.range - 1) * prob as u32) >> 8); + if bit { + self.bottom = self.bottom.wrapping_add(split); + self.range -= split; + } else { + self.range = split; + } + while self.range < 128 { + self.range <<= 1; + if self.bottom & (1 << 31) != 0 { + self.add_one(); + } + self.bottom <<= 1; + self.bit_count -= 1; + if self.bit_count == 0 { + self.out.push((self.bottom >> 24) as u8); + self.bottom &= (1 << 24) - 1; + self.bit_count = 8; + } + } + } + + pub fn literal(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.write((v >> i) & 1 != 0, 128); + } + } + + /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). + pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { + // Find the path to the leaf. + fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.push((node >> 1, b == 1)); + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if path(tree, t as usize, value, out) { + return true; + } + out.pop(); + } + false + } + let mut p = Vec::with_capacity(8); + path(tree, 0, value, &mut p); + for (i, bit) in p { + self.write(bit, probs[i]); + } + } + + /// Flush (RFC 6386 §7.3) and return the bytes. + pub fn finish(mut self) -> Vec { + let mut c = self.bit_count; + let mut v = self.bottom; + if v & (1u32 << (32 - c)) != 0 { + self.add_one(); + } + v <<= c & 7; + c >>= 3; + while { + c -= 1; + c >= 0 + } { + v <<= 8; + } + for _ in 0..4 { + self.out.push((v >> 24) as u8); + v <<= 8; + } + self.out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A straightforward decoder per VP9 spec §9.2. + struct Dec<'a> { + d: &'a [u8], + pos: usize, + value: u64, + bits: i32, + range: u32, + } + impl<'a> Dec<'a> { + fn new(d: &'a [u8]) -> Self { + let mut x = Dec { d, pos: 0, value: 0, bits: 0, range: 255 }; + x.fill(); + x + } + fn fill(&mut self) { + while self.bits <= 48 { + let b = if self.pos < self.d.len() { self.d[self.pos] } else { 0 }; + self.pos += 1; + self.value |= (b as u64) << (56 - self.bits); + self.bits += 8; + } + } + fn read(&mut self, p: u8) -> bool { + let split = 1 + (((self.range - 1) * p as u32) >> 8); + let big = (split as u64) << 56; + let bit = if self.value >= big { + self.range -= split; + self.value -= big; + true + } else { + self.range = split; + false + }; + while self.range < 128 { + self.range <<= 1; + self.value <<= 1; + self.bits -= 1; + } + self.fill(); + bit + } + } + + #[test] + fn round_trip() { + let mut e = BoolEncoder::new(); + let mut seq = vec![]; + let mut s = 12345u32; + for i in 0..20000 { + s = s.wrapping_mul(1103515245).wrapping_add(12345); + let p = ((s >> 8) % 255 + 1) as u8; + let bit = (s >> 20) % 256 >= p as u32 || i % 97 == 0; + seq.push((bit, p)); + e.write(bit, p); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for (i, (b, p)) in seq.into_iter().enumerate() { + assert_eq!(dec.read(p), b, "bit {i}"); + } + } + + #[test] + fn all_ones_carry() { + let mut e = BoolEncoder::new(); + for _ in 0..5000 { + e.write(true, 1); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for _ in 0..5000 { + assert!(dec.read(1)); + } + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs new file mode 100644 index 00000000..02cbb0e2 --- /dev/null +++ b/crates/vp9enc/src/lib.rs @@ -0,0 +1,857 @@ +//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & +//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see +//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width +//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard +//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by +//! prediction error, default probabilities (no probability updates). The encoder reconstructs +//! exactly what a decoder reconstructs (the specification's integer inverse transforms and +//! intra edge rules), so prediction never drifts. + +mod bool; +mod tables; + +use bool::BoolEncoder; +use tables::*; + +const DC_PRED: u8 = 0; +const V_PRED: u8 = 1; +const H_PRED: u8 = 2; +const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; + +const DCT_DCT: u8 = 0; +const ADST_DCT: u8 = 1; +const DCT_ADST: u8 = 2; +const ADST_ADST: u8 = 3; + +/// mode2txfm_map for the modes used here. +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// Encoder settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + /// 1..=100; 100 is lossless. + pub quality: u8, + /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. + pub full_range: bool, +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +// ---------------------------------------------------------------- transforms (8.7) + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms, so `inverse4(forward(res)) ≈ res`. +fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +// ---------------------------------------------------------------- planes and prediction + +struct Plane { + w: usize, + /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). + max_x: usize, + max_y: usize, + src: Vec, + rec: Vec, +} + +impl Plane { + fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { + let mut src = vec![0u8; w * h]; + for y in 0..h { + let sy = y.min(vh.saturating_sub(1)); + for x in 0..w { + let sx = x.min(vw.saturating_sub(1)); + src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); + } + } + Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + } +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- the encoder + +pub struct Vp9Encoder { + cfg: EncoderConfig, + qidx: u8, + lossless: bool, +} + +/// Per-tile coding state. +struct Tile<'a> { + enc: &'a Vp9Encoder, + bc: BoolEncoder, + planes: &'a mut [Plane; 3], + mi_cols: usize, + mi_rows: usize, + col_start: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// y mode of every 8×8 block (for mode contexts). + modes: &'a mut [u8], +} + +impl Vp9Encoder { + pub fn new(cfg: EncoderConfig) -> Self { + let q = cfg.quality.clamp(1, 100); + let lossless = q >= 100; + // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; + Vp9Encoder { cfg, qidx, lossless } + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// ((w+1)/2)×((h+1)/2), tightly packed). + pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let mi_cols = w.div_ceil(8); + let mi_rows = h.div_ceil(8); + let sb_cols = mi_cols.div_ceil(8); + let sb_rows = mi_rows.div_ceil(8); + let (pw, ph) = (sb_cols * 64, sb_rows * 64); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut planes = [ + Plane::new(y, w, h, pw, ph, mi_cols * 8 - 1, mi_rows * 8 - 1), + Plane::new(u, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + Plane::new(v, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + ]; + // Tile columns: as few as the width allows (tile_info, 6.2.13). + let mut min_log2 = 0; + while (64usize << min_log2) < sb_cols { + min_log2 += 1; + } + let mut max_log2 = 1; + while (sb_cols >> max_log2) >= 4 { + max_log2 += 1; + } + max_log2 -= 1; + let tile_cols = 1usize << min_log2; + let mut modes = vec![0u8; mi_cols * mi_rows]; + let mut tiles = vec![]; + for t in 0..tile_cols { + let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); + let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); + let mut tile = Tile { + enc: self, + bc: BoolEncoder::new(), + planes: &mut planes, + mi_cols, + mi_rows, + col_start: start, + above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; mi_cols + 8], + left_part: vec![0; 8], + modes: &mut modes, + }; + tile.bc.write(false, 128); + for r in (0..mi_rows).step_by(8) { + for p in 0..3 { + tile.left_nz[p].fill(0); + } + tile.left_part.fill(0); + for c in (start..end).step_by(8) { + tile.partition(r, c, 12); + } + } + tiles.push(tile.bc.finish()); + } + // Compressed header. + let mut ch_enc = BoolEncoder::new(); + ch_enc.write(false, 128); + if !self.lossless { + ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + } + ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) + for _ in 0..3 { + ch_enc.write(false, 252); // skip probabilities unchanged + } + let compressed = ch_enc.finish(); + // Uncompressed header. + let mut bw = BitWriter::default(); + bw.put(2, 2); // frame_marker + bw.put(0, 1); // profile low bit + bw.put(0, 1); // profile high bit + bw.put(0, 1); // show_existing_frame + bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(1, 1); // show_frame + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + bw.put(0, 2); // frame_context_idx + bw.put(0, 6); // loop_filter_level + bw.put(0, 3); // sharpness + bw.put(0, 1); // mode_ref_delta_enabled + bw.put(self.qidx as u32, 8); + bw.put(0, 1); // delta_q_y_dc + bw.put(0, 1); // delta_q_uv_dc + bw.put(0, 1); // delta_q_uv_ac + bw.put(0, 1); // segmentation_enabled + if min_log2 < max_log2 { + bw.put(0, 1); // no extra tile columns + } + bw.put(0, 1); // tile_rows_log2 = 0 + bw.put(compressed.len() as u32, 16); + let mut out = bw.finish(); + out.extend_from_slice(&compressed); + let n = tiles.len(); + for (i, t) in tiles.into_iter().enumerate() { + if i + 1 < n { + out.extend_from_slice(&(t.len() as u32).to_be_bytes()); + } + out.extend_from_slice(&t); + } + out + } +} + +#[derive(Default)] +struct BitWriter { + out: Vec, + acc: u32, + n: u32, +} + +impl BitWriter { + fn put(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1); + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out + } +} + +const BLOCK_8X8: u8 = 3; +const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; +const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; + +impl Tile<'_> { + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + if r >= self.mi_rows || c >= self.mi_cols { + return; + } + let num8 = NUM_8X8[bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < self.mi_rows; + let has_cols = (c + half) < self.mi_cols; + let bsl = MI_WIDTH_LOG2[bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let above = (above >> boffset) & 1; + let left = (left >> boffset) & 1; + let ctx = bsl * 4 + left as usize * 2 + above as usize; + let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; + if bsize == BLOCK_8X8 { + // PARTITION_NONE. + self.bc.write(false, probs[0]); + self.block(r, c); + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.bc.tree(&PARTITION_TREE, probs, 3); + } else if has_cols { + self.bc.write(true, probs[1]); + } else if has_rows { + self.bc.write(true, probs[2]); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + fn block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. + let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). + self.bc.write(false, DEFAULT_SKIP_PROB[0]); + let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; + let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); + let o = y_mode as usize * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); + self.modes[r * self.mi_cols + c] = y_mode; + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.mi_cols * 8) >> ss; + let max_y = (self.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); + // Quantised levels (raster order). + let levels: [i32; 16] = if self.enc.lossless { + forward_wht(&res) + } else { + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let qs = if k == 0 { dcq } else { acq } as f64; + let a = coef.abs() / qs; + // Rounding with a small dead zone for AC coefficients. + let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; + lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; + } + lv + }; + let sc = scan(tx_type); + let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); + // Tokens. + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, tx_type, &levels, eob, ctx0); + // Reconstruct exactly as the decoder does. + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { dcq } else { acq }; + } + let resid = if eob == 0 { + [0i32; 16] + } else if self.enc.lossless { + inverse_wht(&deq) + } else { + inverse4(&deq, tx_type) + }; + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. + let base = (ptype * 2) * 6 * 6 * 3; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = base + (band * 6 + ctx) * 3; + let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; + if check_eob { + let more = c < eob; + self.bc.write(more, p[0]); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.bc.write(false, p[1]); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.bc.write(true, p[1]); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.bc.write(false, p[2]); + } else { + self.bc.write(true, p[2]); + let pp = pareto(p[2]); + let path: &[(usize, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.bc.write(b, pp[n]); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.bc.write(levels[pos] < 0, 128); + c += 1; + } + } +} + +// ---------------------------------------------------------------- colour conversion + +/// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). +pub fn rgba_to_yuv420(rgba: &[u8], w: u32, h: u32, full_range: bool) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut y = vec![0u8; w * h]; + let mut u = vec![0u8; cw * ch]; + let mut v = vec![0u8; cw * ch]; + let (ys, yo, cs) = if full_range { (255.0, 0.0, 255.0) } else { (219.0, 16.0, 224.0) }; + let px = |x: usize, yy: usize| -> [f32; 3] { + let i = (yy.min(h - 1) * w + x.min(w - 1)) * 4; + [rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0] + }; + let luma = |c: [f32; 3]| 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; + for yy in 0..h { + for x in 0..w { + y[yy * w + x] = (luma(px(x, yy)) * ys + yo).round().clamp(0.0, 255.0) as u8; + } + } + for cy in 0..ch { + for cx in 0..cw { + let mut acc = [0.0f32; 3]; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let c = px(cx * 2 + dx, cy * 2 + dy); + for k in 0..3 { + acc[k] += c[k] / 4.0; + } + } + let l = luma(acc); + let cb = (acc[2] - l) / 1.8556; + let cr = (acc[0] - l) / 1.5748; + u[cy * cw + cx] = (cb * cs + 128.0).round().clamp(0.0, 255.0) as u8; + v[cy * cw + cx] = (cr * cs + 128.0).round().clamp(0.0, 255.0) as u8; + } + } + (y, u, v) +} + +/// The alpha channel as a luma plane (full range) with neutral chroma — the second VP9 stream of +/// WebM alpha. +pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let y: Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); + (y, vec![128; cw * ch], vec![128; cw * ch]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs new file mode 100644 index 00000000..dc8027a6 --- /dev/null +++ b/crates/vp9enc/src/tables.rs @@ -0,0 +1,130 @@ +//! Tables from the VP9 Bitstream & Decoding Process Specification v0.6 (section 10), as +//! transcribed in FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0, the same +//! licence as this crate). + +#![allow(clippy::unreadable_literal)] + +pub const DC_QLOOKUP: [i32; 256] = [ + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, + 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 70, 70, 71, 72, + 73, 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, 111, 113, 114, + 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, 177, 180, + 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, + 284, 288, 292, 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, + 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, + 775, 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, +]; + +pub const AC_QLOOKUP: [i32; 256] = [ + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, + 136, 138, 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, 373, 380, 387, + 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, + 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, + 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, +]; + +pub const DEFAULT_SCAN_4X4: [u16; 16] = [0, 4, 1, 5, 8, 2, 12, 9, 3, 6, 13, 10, 7, 14, 11, 15]; + +pub const COL_SCAN_4X4: [u16; 16] = [0, 4, 8, 1, 12, 5, 9, 2, 13, 6, 10, 3, 7, 14, 11, 15]; + +pub const ROW_SCAN_4X4: [u16; 16] = [0, 1, 4, 2, 5, 3, 6, 8, 9, 7, 12, 10, 13, 11, 14, 15]; + +pub const COEFBAND_4X4: [u8; 16] = [0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5]; + +pub const ENERGY_CLASS: [u8; 12] = [0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5]; + +pub const PARETO_TABLE: [u8; 1024] = [ + 3, 86, 128, 6, 86, 23, 88, 29, 9, 86, 129, 17, 88, 61, 94, 76, 15, 87, 129, 28, 89, 93, 100, 110, 20, 88, 130, 38, 91, 118, 106, 136, 26, 89, 131, 48, 92, + 139, 111, 156, 31, 90, 131, 58, 94, 156, 117, 171, 37, 90, 132, 66, 95, 171, 122, 184, 42, 91, 132, 75, 97, 183, 127, 194, 47, 92, 133, 83, 98, 193, 132, + 202, 52, 93, 133, 90, 100, 201, 137, 208, 57, 94, 134, 98, 101, 208, 142, 214, 62, 94, 135, 105, 103, 214, 146, 218, 66, 95, 135, 111, 104, 219, 151, 222, + 71, 96, 136, 117, 106, 224, 155, 225, 76, 97, 136, 123, 107, 227, 159, 228, 80, 98, 137, 129, 109, 231, 162, 231, 84, 98, 138, 134, 110, 234, 166, 233, 89, + 99, 138, 140, 112, 236, 170, 235, 93, 100, 139, 145, 113, 238, 173, 236, 97, 101, 140, 149, 115, 240, 176, 238, 101, 102, 140, 154, 116, 242, 179, 239, + 105, 103, 141, 158, 118, 243, 182, 240, 109, 104, 141, 162, 119, 244, 185, 241, 113, 104, 142, 166, 120, 245, 187, 242, 116, 105, 143, 170, 122, 246, 190, + 243, 120, 106, 143, 173, 123, 247, 192, 244, 123, 107, 144, 177, 125, 248, 195, 244, 127, 108, 145, 180, 126, 249, 197, 245, 130, 109, 145, 183, 128, 249, + 199, 245, 134, 110, 146, 186, 129, 250, 201, 246, 137, 111, 147, 189, 131, 251, 203, 246, 140, 112, 147, 192, 132, 251, 205, 247, 143, 113, 148, 194, 133, + 251, 207, 247, 146, 114, 149, 197, 135, 252, 208, 248, 149, 115, 149, 199, 136, 252, 210, 248, 152, 115, 150, 201, 138, 252, 211, 248, 155, 116, 151, 204, + 139, 253, 213, 249, 158, 117, 151, 206, 140, 253, 214, 249, 161, 118, 152, 208, 142, 253, 216, 249, 163, 119, 153, 210, 143, 253, 217, 249, 166, 120, 153, + 212, 144, 254, 218, 250, 168, 121, 154, 213, 146, 254, 220, 250, 171, 122, 155, 215, 147, 254, 221, 250, 173, 123, 155, 217, 148, 254, 222, 250, 176, 124, + 156, 218, 150, 254, 223, 250, 178, 125, 157, 220, 151, 254, 224, 251, 180, 126, 157, 221, 152, 254, 225, 251, 183, 127, 158, 222, 153, 254, 226, 251, 185, + 128, 159, 224, 155, 255, 227, 251, 187, 129, 160, 225, 156, 255, 228, 251, 189, 131, 160, 226, 157, 255, 228, 251, 191, 132, 161, 227, 159, 255, 229, 251, + 193, 133, 162, 228, 160, 255, 230, 252, 195, 134, 163, 230, 161, 255, 231, 252, 197, 135, 163, 231, 162, 255, 231, 252, 199, 136, 164, 232, 163, 255, 232, + 252, 201, 137, 165, 233, 165, 255, 233, 252, 202, 138, 166, 233, 166, 255, 233, 252, 204, 139, 166, 234, 167, 255, 234, 252, 206, 140, 167, 235, 168, 255, + 235, 252, 207, 141, 168, 236, 169, 255, 235, 252, 209, 142, 169, 237, 171, 255, 236, 252, 210, 144, 169, 237, 172, 255, 236, 252, 212, 145, 170, 238, 173, + 255, 237, 252, 214, 146, 171, 239, 174, 255, 237, 253, 215, 147, 172, 240, 175, 255, 238, 253, 216, 148, 173, 240, 176, 255, 238, 253, 218, 149, 173, 241, + 177, 255, 239, 253, 219, 150, 174, 241, 179, 255, 239, 253, 220, 152, 175, 242, 180, 255, 240, 253, 222, 153, 176, 242, 181, 255, 240, 253, 223, 154, 177, + 243, 182, 255, 240, 253, 224, 155, 178, 244, 183, 255, 241, 253, 225, 156, 178, 244, 184, 255, 241, 253, 226, 158, 179, 244, 185, 255, 242, 253, 228, 159, + 180, 245, 186, 255, 242, 253, 229, 160, 181, 245, 187, 255, 242, 253, 230, 161, 182, 246, 188, 255, 243, 253, 231, 163, 183, 246, 189, 255, 243, 253, 232, + 164, 184, 247, 190, 255, 243, 253, 233, 165, 185, 247, 191, 255, 244, 253, 234, 166, 185, 247, 192, 255, 244, 253, 235, 168, 186, 248, 193, 255, 244, 253, + 236, 169, 187, 248, 194, 255, 244, 253, 236, 170, 188, 248, 195, 255, 245, 253, 237, 171, 189, 249, 196, 255, 245, 254, 238, 173, 190, 249, 197, 255, 245, + 254, 239, 174, 191, 249, 198, 255, 245, 254, 240, 175, 192, 249, 199, 255, 246, 254, 240, 177, 193, 250, 200, 255, 246, 254, 241, 178, 194, 250, 201, 255, + 246, 254, 242, 179, 195, 250, 202, 255, 246, 254, 242, 181, 196, 250, 203, 255, 247, 254, 243, 182, 197, 251, 204, 255, 247, 254, 244, 184, 198, 251, 205, + 255, 247, 254, 244, 185, 199, 251, 206, 255, 247, 254, 245, 186, 200, 251, 207, 255, 247, 254, 246, 188, 201, 252, 207, 255, 248, 254, 246, 189, 202, 252, + 208, 255, 248, 254, 247, 191, 203, 252, 209, 255, 248, 254, 247, 192, 204, 252, 210, 255, 248, 254, 248, 194, 205, 252, 211, 255, 248, 254, 248, 195, 206, + 252, 212, 255, 249, 254, 249, 197, 207, 253, 213, 255, 249, 254, 249, 198, 208, 253, 214, 255, 249, 254, 250, 200, 210, 253, 215, 255, 249, 254, 250, 201, + 211, 253, 215, 255, 249, 254, 250, 203, 212, 253, 216, 255, 249, 254, 251, 204, 213, 253, 217, 255, 250, 254, 251, 206, 214, 254, 218, 255, 250, 254, 252, + 207, 216, 254, 219, 255, 250, 254, 252, 209, 217, 254, 220, 255, 250, 254, 252, 211, 218, 254, 221, 255, 250, 254, 253, 213, 219, 254, 222, 255, 250, 254, + 253, 214, 221, 254, 223, 255, 250, 254, 253, 216, 222, 254, 224, 255, 251, 254, 253, 218, 224, 254, 225, 255, 251, 254, 254, 220, 225, 254, 225, 255, 251, + 254, 254, 222, 227, 255, 226, 255, 251, 254, 254, 224, 228, 255, 227, 255, 251, 254, 254, 226, 230, 255, 228, 255, 251, 254, 255, 228, 231, 255, 230, 255, + 251, 254, 255, 230, 233, 255, 231, 255, 252, 254, 255, 232, 235, 255, 232, 255, 252, 254, 255, 235, 237, 255, 233, 255, 252, 254, 255, 238, 240, 255, 235, + 255, 252, 255, 255, 241, 243, 255, 236, 255, 252, 254, 255, 246, 247, 255, 239, 255, 253, 255, +]; + +pub const KF_PARTITION_PROBS: [u8; 48] = [ + 158, 97, 94, 93, 24, 99, 85, 119, 44, 62, 59, 67, 149, 53, 53, 94, 20, 48, 83, 53, 24, 52, 18, 18, 150, 40, 39, 78, 12, 26, 67, 33, 11, 24, 7, 5, 174, 35, + 49, 68, 11, 27, 57, 15, 9, 12, 3, 3, +]; + +pub const KF_Y_MODE_PROBS: [u8; 900] = [ + 137, 30, 42, 148, 151, 207, 70, 52, 91, 92, 45, 102, 136, 116, 180, 74, 90, 100, 73, 32, 19, 187, 222, 215, 46, 34, 100, 91, 30, 32, 116, 121, 186, 93, 86, + 94, 72, 35, 36, 149, 68, 206, 68, 63, 105, 73, 31, 28, 138, 57, 124, 55, 122, 151, 67, 23, 21, 140, 126, 197, 40, 37, 171, 86, 27, 28, 128, 154, 212, 45, + 43, 53, 74, 32, 27, 107, 86, 160, 63, 134, 102, 59, 67, 44, 140, 161, 202, 78, 67, 119, 63, 36, 126, 146, 123, 158, 60, 90, 96, 43, 46, 168, 134, 107, 128, + 69, 142, 92, 44, 29, 68, 159, 201, 177, 50, 57, 77, 58, 38, 76, 114, 97, 172, 78, 133, 92, 46, 41, 76, 140, 63, 184, 69, 112, 57, 38, 32, 85, 140, 46, 112, + 54, 151, 133, 39, 27, 61, 131, 110, 175, 44, 75, 136, 52, 30, 74, 113, 130, 175, 51, 64, 58, 47, 35, 80, 100, 74, 143, 64, 163, 74, 36, 61, 116, 114, 128, + 162, 80, 125, 82, 82, 26, 26, 171, 208, 204, 44, 32, 105, 55, 44, 68, 166, 179, 192, 57, 57, 108, 42, 26, 11, 199, 241, 228, 23, 15, 85, 68, 42, 19, 131, + 160, 199, 55, 52, 83, 58, 50, 25, 139, 115, 232, 39, 52, 118, 50, 35, 33, 153, 104, 162, 64, 59, 131, 44, 24, 16, 150, 177, 202, 33, 19, 156, 55, 27, 12, + 153, 203, 218, 26, 27, 49, 53, 49, 21, 110, 116, 168, 59, 80, 76, 38, 72, 19, 168, 203, 212, 50, 50, 107, 103, 26, 36, 129, 132, 201, 83, 80, 93, 59, 38, + 83, 112, 103, 162, 98, 136, 90, 62, 30, 23, 158, 200, 207, 59, 57, 50, 67, 30, 29, 84, 86, 191, 102, 91, 59, 60, 32, 33, 112, 71, 220, 64, 89, 104, 53, 26, + 34, 130, 56, 149, 84, 120, 103, 53, 21, 23, 133, 109, 210, 56, 77, 172, 77, 19, 29, 112, 142, 228, 55, 66, 36, 61, 29, 29, 93, 97, 165, 83, 175, 162, 47, + 47, 43, 114, 137, 181, 100, 99, 95, 69, 23, 29, 128, 83, 199, 46, 44, 101, 53, 40, 55, 139, 69, 183, 61, 80, 110, 40, 29, 19, 161, 180, 207, 43, 24, 91, + 60, 34, 19, 105, 61, 198, 53, 64, 89, 52, 31, 22, 158, 40, 209, 58, 62, 89, 44, 31, 29, 147, 46, 158, 56, 102, 198, 35, 19, 12, 135, 87, 209, 41, 45, 167, + 55, 25, 21, 118, 95, 215, 38, 39, 66, 51, 38, 25, 113, 58, 164, 70, 93, 97, 47, 54, 34, 146, 108, 203, 72, 103, 151, 64, 19, 37, 156, 66, 138, 49, 95, 133, + 46, 27, 80, 150, 55, 124, 55, 121, 135, 36, 23, 27, 165, 149, 166, 54, 64, 118, 53, 21, 36, 131, 63, 163, 60, 109, 81, 40, 26, 35, 154, 40, 185, 51, 97, + 123, 35, 19, 34, 179, 19, 97, 48, 129, 124, 36, 20, 26, 136, 62, 164, 33, 77, 154, 45, 18, 32, 130, 90, 157, 40, 79, 91, 45, 26, 28, 129, 45, 129, 49, 147, + 123, 38, 44, 51, 136, 74, 162, 57, 97, 121, 75, 17, 22, 136, 138, 185, 32, 34, 166, 56, 39, 58, 133, 117, 173, 48, 53, 187, 35, 21, 12, 161, 212, 207, 20, + 23, 145, 56, 29, 19, 117, 109, 181, 55, 68, 112, 47, 29, 17, 153, 64, 220, 59, 51, 114, 46, 16, 24, 136, 76, 147, 41, 64, 172, 34, 17, 11, 108, 152, 187, + 13, 15, 209, 51, 24, 14, 115, 133, 209, 32, 26, 104, 55, 30, 18, 122, 79, 179, 44, 88, 116, 37, 49, 25, 129, 168, 164, 41, 54, 148, 82, 22, 32, 127, 143, + 213, 39, 41, 70, 62, 44, 61, 123, 105, 189, 48, 57, 64, 47, 25, 17, 175, 222, 220, 24, 30, 86, 68, 36, 17, 106, 102, 206, 59, 74, 74, 57, 39, 23, 151, 68, + 216, 55, 63, 58, 49, 30, 35, 141, 70, 168, 82, 40, 115, 51, 25, 15, 136, 129, 202, 38, 35, 139, 68, 26, 16, 111, 141, 215, 29, 28, 28, 59, 39, 19, 114, 75, + 180, 77, 104, 42, 40, 61, 26, 126, 152, 206, 61, 59, 93, 78, 23, 39, 111, 117, 170, 74, 124, 94, 48, 34, 86, 101, 92, 146, 78, 179, 134, 47, 22, 24, 138, + 187, 178, 68, 69, 59, 56, 25, 33, 105, 112, 187, 95, 177, 129, 48, 31, 27, 114, 63, 183, 82, 116, 56, 43, 28, 37, 121, 63, 123, 61, 192, 169, 42, 17, 24, + 109, 97, 177, 56, 76, 122, 58, 18, 28, 105, 139, 182, 70, 92, 63, 46, 23, 32, 74, 86, 150, 67, 183, 88, 36, 38, 48, 92, 122, 165, 88, 137, 91, 65, 70, 60, + 155, 159, 199, 61, 60, 81, 44, 78, 115, 132, 119, 173, 71, 112, 93, 39, 38, 21, 184, 227, 206, 42, 32, 64, 58, 47, 36, 124, 137, 193, 80, 82, 78, 49, 50, + 35, 144, 95, 205, 63, 78, 59, 41, 53, 52, 148, 71, 142, 65, 128, 51, 40, 36, 28, 143, 143, 202, 40, 55, 137, 52, 34, 29, 129, 183, 227, 42, 35, 43, 42, 44, + 44, 104, 105, 164, 64, 130, 80, 43, 81, 53, 140, 169, 204, 68, 84, 72, +]; + +pub const KF_UV_MODE_PROBS: [u8; 90] = [ + 144, 11, 54, 157, 195, 130, 46, 58, 108, 118, 15, 123, 148, 131, 101, 44, 93, 131, 113, 12, 23, 188, 226, 142, 26, 32, 125, 120, 11, 50, 123, 163, 135, 64, + 77, 103, 113, 9, 36, 155, 111, 157, 32, 44, 161, 116, 9, 55, 176, 76, 96, 37, 61, 149, 115, 9, 28, 141, 161, 167, 21, 25, 193, 120, 12, 32, 145, 195, 142, + 32, 38, 86, 116, 12, 64, 120, 140, 125, 49, 115, 121, 102, 19, 66, 162, 182, 122, 35, 59, 128, +]; + +pub const DEFAULT_SKIP_PROB: [u8; 3] = [192, 128, 64]; + +pub const DEFAULT_COEF_PROBS: [u8; 432] = [ + 195, 29, 183, 84, 49, 136, 8, 42, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 107, 169, 35, 99, 159, 17, 82, 140, 8, 66, 114, 2, 44, 76, 1, 19, 32, 40, 132, 201, + 29, 114, 187, 13, 91, 157, 7, 75, 127, 3, 58, 95, 1, 28, 47, 69, 142, 221, 42, 122, 201, 15, 91, 159, 6, 67, 121, 1, 42, 77, 1, 17, 31, 102, 148, 228, 67, + 117, 204, 17, 82, 154, 6, 59, 114, 2, 39, 75, 1, 15, 29, 156, 57, 233, 119, 57, 212, 58, 48, 163, 29, 40, 124, 12, 30, 81, 3, 12, 31, 191, 107, 226, 124, + 117, 204, 25, 99, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 148, 210, 37, 126, 194, 8, 93, 157, 2, 68, 118, 1, 39, 69, 1, 17, 33, 41, 151, 213, 27, 123, 193, 3, + 82, 144, 1, 58, 105, 1, 32, 60, 1, 13, 26, 59, 159, 220, 23, 126, 198, 4, 88, 151, 1, 66, 114, 1, 38, 71, 1, 18, 34, 114, 136, 232, 51, 114, 207, 11, 83, + 155, 3, 56, 105, 1, 33, 65, 1, 17, 34, 149, 65, 234, 121, 57, 215, 61, 49, 166, 28, 36, 114, 12, 25, 76, 3, 16, 42, 214, 49, 220, 132, 63, 188, 42, 65, + 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 137, 221, 104, 131, 216, 49, 111, 192, 21, 87, 155, 2, 49, 87, 1, 16, 28, 89, 163, 230, 90, 137, 220, 29, 100, 183, 10, + 70, 135, 2, 42, 81, 1, 17, 33, 108, 167, 237, 55, 133, 222, 15, 97, 179, 4, 72, 135, 1, 45, 85, 1, 19, 38, 124, 146, 240, 66, 124, 224, 17, 88, 175, 4, 58, + 122, 1, 36, 75, 1, 18, 37, 141, 79, 241, 126, 70, 227, 66, 58, 182, 30, 44, 136, 12, 34, 96, 2, 20, 47, 229, 99, 249, 143, 111, 235, 46, 109, 192, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 82, 158, 236, 94, 146, 224, 25, 117, 191, 9, 87, 149, 3, 56, 99, 1, 33, 57, 83, 167, 237, 68, 145, 222, 10, 103, 177, 2, 72, 131, 1, 41, + 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, + 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, +]; diff --git a/docs/agents.md b/docs/agents.md index f49ddc22..8689b9da 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -147,6 +147,34 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + ### Roto Brush & Refine Edge The Roto Brush tool (Alt+W cycles Roto Brush / Refine Edge) paints in the Layer panel; agents use diff --git a/docs/architecture.md b/docs/architecture.md index 67d0b973..39acd396 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,18 +16,22 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `vp9enc` | VP9 intra-frame encoder (profile 0, 8-bit 4:2:0; lossless at quality 100) for WebM export, from the VP9 bitstream specification | +| L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | +| L1 | `psd` | Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit; RGB/CMYK/Gray/Lab) and a minimal writer, from Adobe's published format specification | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `svg` | SVG import (W3C SVG 1.1/2 static subset: shapes, paths, transforms, `use`, CSS, gradients) and rasterisation at any scale | | L2 | `model` | 3D models for Advanced 3D: glTF 2.0 (`.gltf`/`.glb`) and OBJ/MTL import (meshes, PBR metallic-roughness materials and textures, node hierarchy, skins, animations), parametric primitives, extruded/bevelled outline meshes and polygon triangulation | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | -| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | +| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, Photoshop and SVG stills, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | -| L3 | `export` | Render queue encoding: H.264, ProRes, PNG/JPEG/TIFF/EXR sequences, GIF, audio | +| L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | @@ -106,6 +110,14 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +A persistent **disk cache** (`render::disk_cache`, Settings ▸ Media & Disk Cache) backs both +the layer cache and the viewer's RAM preview: layer buffers that were slow to render and every +preview frame are written (LZ4, atomic rename, checksummed) under 128-bit content keys — the +comp's content hash (the comp, what it uses, footage file size and time) plus frame, scale and +view for frames; the layer key salted with the footage fingerprint for layers — so entries +survive restarts and never go stale. LRU eviction keeps it under the size limit; Edit ▸ Purge +empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. + **Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through `EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* @@ -154,7 +166,8 @@ chain with one upload and one readback. Tests render scenes on both paths and co (≤ 1/255 at 8 bpc, ≤ 1e-3 at 32 bpc); they skip without an adapter. The desktop viewer builds the `Gpu` on egui-wgpu's device and shows frames from GPU textures without reading them back (`ui-egui::frames`); headless renders, the CLI (unless `--gpu`) and CI use the CPU. On the web -(WebGPU) the GPU composites viewer frames; steps that need a readback fall back to the CPU. +(WebGPU) the GPU composites viewer frames; steps that need a readback fall back to the CPU (the +Info panel's pixel readout reads GPU frames back asynchronously). **Colour and bit depth** (`crates/render/src/color.rs`, `crates/color/src/space.rs`). Pixels are `f32`, but 8 and 16 bpc projects clamp and quantise each layer after its source and masks and @@ -207,6 +220,40 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**Essential Graphics** (`project::essential`, `engine::commands::essential`): a comp's +`essential` field lists its exposed controls (properties, Media Replacement, groups, comments). +Every precomp layer of such a comp gets an Essential Properties group (`GroupKind::Essential`, +children `eg`), kept in step by `Session::edit`; editing a child overrides it for +that instance. The renderer draws an instance with overrides from a copy of the project with the +overridden source properties set (`essential::with_overrides`). Templates are `.ectemplate` +ZIPs: `manifest.json` (controls), `project.ecproj` (the comp and its dependencies), `media/…` and +`poster.png`; importing offsets every id past the project's (`essential::offset_ids`). +**Responsive Design — Time**: a time-stretched precomp maps time piecewise so its protected +marker regions play at 100 % (`eval::responsive_source_time`). + +**Proxies and interpretation**: an item's `proxy` (footage or comp) is decoded in its place at +the source's nominal size when `RenderOpts::proxy` (Render Settings ▸ Proxy Use) allows it. +Interpret Footage's Separate Fields turns each field into a frame at twice the rate (the other +lines interpolated); pixel aspect stretches footage, solids and precomps in the comp +(`EvalCtx::par_ratio`, not inherited by children); Invert Alpha and Interpret As Linear Light +apply after decoding. + +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + **Roto Brush & Refine Edge** (`effectcraft_track::roto`, `effects::roto`, `engine::roto`): strokes (foreground, background, Refine Edge) are stored as JSON in the effect's hidden Strokes parameter with the base frame and segmentation span. Each frame is segmented by graph cut @@ -224,7 +271,11 @@ in the effect. Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, `time`, `value`, `wiggle`, `loopOut`, vector maths on arrays, and so on). A syntax error keeps the -text, disables the expression and shows a warning, like After Effects. +text, disables the expression and shows a warning, like After Effects. `sampleImage` renders the +sampled layer through the evaluating renderer's footage source (`EvalCtx::footage`), cached per +thread and frame; `footage(name)` reads data footage (JSON, CSV, TSV imported with File ▸ Import, +text kept in the project) through `sourceData`, `sourceText` and `dataValue`. `expr.errors` +lists failing expressions for the viewer's error bar. ## 5a. Scripting @@ -259,6 +310,14 @@ inspect and click the interface. See [agents.md](agents.md) and Settings (`engine::prefs`), keyboard shortcut presets (`engine::shortcuts`) and auto-save / crash recovery (`engine::autosave`) live in the engine too; they persist through a -`ConfigStore` the frontend provides (a directory on the desktop, `localStorage` on the web). +`ConfigStore` the frontend provides (a directory on the desktop; the Origin Private File System, +or IndexedDB, on the web). Auto-saves go through the store's `FileOps` (the file system by +default, the browser storage on the web). + +Renders and analyses run on background threads on the desktop. Where there are no threads (the +browser), `engine::offload` sends them to a second engine instance instead: the session's +`Offload` ships the serialized project, the footage it reads and the job (`WorkerRequest`) to a +Web Worker, and the worker's replies (progress, item status, the analysed property group) are +applied on the UI thread. The shortcut dispatcher and the menus read the active preset. See [preferences.md](preferences.md). diff --git a/docs/parity.md b/docs/parity.md index 8f04428d..021960ec 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -1,9 +1,35 @@ # Parity with After Effects How close EffectCraft is to After Effects 2026, feature by feature, and how much work is left. -Audited against the code at commit `5273f1b` (2 October 2026). Every feature in our catalog was -graded from the code itself: a disabled menu entry, or a setting that is stored but never -rendered, does not count as done. + +## Current status (audit at commit `fa26ad9`, 2 October 2026, late) + +| Measure | Value | +|---|---| +| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 93%** | +| Unweighted | ≈ 91% | +| P0 / P1 / P2 | ≈ 96% / 90% / 65% | +| Features done / partial / missing | 41 / 50 / 1 of 92 (missing: a plug-in API) | +| **Effects** | **298 of 298** After Effects 2026 effects exist (some still simplified) | +| Disabled menu entries left | 26 commands (31 menu items) | +| Remaining work | ≈ 50–55 agent-hours at the pace measured so far (≈ 150 by the original conservative audit scale) | +| **Wall-clock estimate** | **≈ 11–14 hours** with five agents in parallel; ≈ 6–8 hours for 100% of P0 + P1 | + +By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, +Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, +Tracking 88%, Effects 85%, Web 65%. + +What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking +renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; GPU +rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; +approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur +and blend modes; a few viewer and colour-management menu items; OpenType features and variable +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and +the "better than After Effects" items (a plug-in API, branching history, GPU particles). + +The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for +history. ## Summary @@ -15,7 +41,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **297 of 298** After Effects 2026 effects by name (99.7%, after M9.11 and Warp Stabilizer; was 257); only the 3D Camera Tracker is left | +| **Effects** | **298 of 298** After Effects 2026 effects by name (100%, after M9.11, Warp Stabilizer and the 3D Camera Tracker; was 257) | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,39 +99,39 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: the 3D camera tracker, -the Warp Stabilizer and mask-tracking UI on top of -the new tracking library, SVG/PSD import, WebM, a disk cache, and wasm threads. (The missing -effect categories, 3D Channel, Immersive Video and OCIO, landed in M9.11; the scripting object -model in M14.4; Roto Brush and Refine Edge in M6.7; native macOS menus, the Composition Flowchart, -Timeline column/search depth, the Home screen and every After Effects workspace in the UI-polish wave.) +What is left is concentrated in large systems: the Warp Stabilizer and mask-tracking UI on top of +the new tracking library, and wasm threads. (The missing effect categories, 3D Channel, Immersive +Video and OCIO, landed in M9.11; the scripting object model in M14.4; Roto Brush and Refine Edge in +M6.7; native macOS menus, the Composition Flowchart, Timeline column/search depth, the Home screen +and every After Effects workspace in the UI-polish wave; PSD/SVG import, WebM/WAV/AIFF output and +the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) ## By area | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 85% | 4.2 | multiple output modules, pre-render, WebM, audio-only | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | -| Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | +| Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | | Shapes | ≈ 80% | 1.5 | Lottie can't carry stroke taper/wave (stroke Taper and Wave, Dash 2/Gap 2/Dash 3/Gap 3 and radial-gradient Highlight Length/Angle landed in M13.5; pen tool for shape paths and vertex editing in M6.5) | -| Compositions | 72% | 6.0 | Essential Graphics panel (the marker dialog and Composition Flowchart landed) | +| Compositions | ≈ 80% | 3.5 | Mocha-style planar tracks for templates, Essential Graphics' rare controls (font menus, mirrored properties) (the marker dialog, Composition Flowchart, Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-6, CMP-7) | | Animation | 67% | 10.0 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | | Text | ≈ 91% | 0.8 | no OpenType feature panel, no extruded strokes, variable-axis animation changes outlines but not advances (vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | -| Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 74% | 11.3 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes (and iris bokeh in Advanced 3D's depth of field), cameras/lights from models; Classic 3D iris-shaped bokeh with highlights, progressive DOF on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | -| Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) | +| Web | 85% | 1.0 | viewer frames and Roto Brush propagation still on the page's thread; GPU effects in the browser (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2) | +| 3D | 78% | 9.3 | multi-view layouts, Advanced 3D motion blur and blend modes (and iris shapes in Advanced 3D's depth of field), cameras/lights from models; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | +| Effects | ≈ 75% | 5.0 | GPU versions of more effects (every After Effects effect exists since M9.11, M12.5 and M12.6) | | Interface | 74% | 5.0 | a richer Learn area (Timeline outline and Project panel columns scroll horizontally, the Layer Style dialog, ROI resize handles, Pan Behind snapping and 3D Reference Axes landed in M13.5; native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | -| Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | +| Project | ≈ 60% | 6.5 | 8/16/32-bit pipeline and colour management, auto-save, folder moves (proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | -| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing -The 3D Camera Tracker (built on the tracking library by other work). Boris FX +None: the 3D Camera Tracker landed in M12.6. Boris FX Mocha and Cineware are third-party and not counted. ### Added in M9.11 (39 effects) @@ -152,8 +178,8 @@ taper/wave, Advanced 3D's DOF has no iris shapes, and variable axes don't change 9. A real 8/16/32-bit pipeline with linear blending and colour management. 10. ~~Drag-to-dock and floating panels, saved workspaces, native macOS menus.~~ 11. Auto-save, crash recovery, recent projects (in progress). -12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). -13. Lottie (in progress), WebM, SVG and PSD import. +12. ~~Expression gaps: `sampleImage`, `footage()`~~ (landed with data footage, the error bar and the Expression Language menu; the `sourceText` style API landed in M9.9). +13. ~~Lottie, WebM, SVG and PSD import~~ (landed; a disk cache too). 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. diff --git a/docs/web.md b/docs/web.md index 7c4ec31d..041ef76c 100644 --- a/docs/web.md +++ b/docs/web.md @@ -2,9 +2,10 @@ `apps/effectcraft-web` runs the same engine and egui UI as the desktop app in the browser: the workspace compiled to `wasm32-unknown-unknown`, started by eframe's web runner on WebGPU (WebGL2 -fallback). It is a static site: one `.wasm`, its `wasm-bindgen` JavaScript glue, `index.html` and an -icon. Nothing is uploaded anywhere; projects, media and renders stay in the page and on the user's -machine. +fallback). It is a static site: one `.wasm`, its `wasm-bindgen` JavaScript glue, `index.html`, a +few small scripts (worker, audio worklet, service worker), a web manifest and icons. Nothing is +uploaded anywhere: projects, media, settings and renders stay in the browser (its private storage) +and on the user's machine. ## Build and run @@ -15,48 +16,136 @@ machine. | wasm-opt (optional) | any recent binaryen | `brew install binaryen`; used by `cargo xtask web` when found | ```sh -cargo xtask web # release build → /web/dist (index.html, effectcraft_web.js, effectcraft_web_bg.wasm, favicon.svg) +cargo xtask web # release build → /web/dist cargo xtask web --dev # unoptimised build (faster to compile, slow to run) cargo xtask web --serve 8765 # build, then serve dist on http://127.0.0.1:8765/ ``` -`` is `$CARGO_TARGET_DIR` or `target`. `--serve` is a tiny localhost-only static server that -sends `Cross-Origin-Opener-Policy` / `Cross-Origin-Embedder-Policy`, so the page is cross-origin +`` is `$CARGO_TARGET_DIR` or `target`. `dist` holds `index.html`, `effectcraft_web.js` + +`effectcraft_web_bg.wasm` (+ `snippets/`, the browser glue `js/host.js`), `worker.js`, +`audio-worklet.js`, `sw.js` (its precache list and version filled in by the build), +`manifest.webmanifest`, `favicon.svg` and `icon-256.png` / `icon-512.png` (copied from +`assets/app-icon`). `--serve` is a tiny localhost-only static server that sends +`Cross-Origin-Opener-Policy` / `Cross-Origin-Embedder-Policy`, so the page is cross-origin isolated like a production deployment should be. Any static server works (`python3 -m http.server -d target/web/dist 8765`); it must serve `.wasm` as `application/wasm`. +Service workers need a secure context: `https://`, or `http://localhost` / `127.0.0.1`. -The release `.wasm` is about 31 MB before `wasm-opt` (20 MB code: the effects, codecs, the -expression engine and the UI; 8.5 MB data, mostly the bundled fonts; 2 MB function names for -readable panics). Served compressed it is a fraction of that. +The release `.wasm` is about 38 MB before `wasm-opt` (code: the effects, codecs, the expression +engine and the UI; data, mostly the bundled fonts; function names for readable panics). Served +compressed it is a fraction of that, and the service worker caches it after the first visit. -The web crate is empty on non-wasm targets, so `cargo test --workspace` and clippy are unaffected; -`cargo xtask wasm` (part of `cargo xtask ci`) checks it, with every L0–L4 crate and -`effectcraft-ui-egui`, for `wasm32-unknown-unknown`. +On other targets the web crate holds only its portable storage model (`store.rs`, unit-tested by +`cargo test --workspace`); `cargo xtask wasm` (part of `cargo xtask ci`) checks the whole crate, +with every L0–L4 crate and `effectcraft-ui-egui`, for `wasm32-unknown-unknown`. -URL flags: `?empty` (start without the demo project), `?home` (show the start screen). +URL flags: `?empty` (a blank project; the session is not restored or recorded), `?demo` (the demo +project instead of the last session), `?home` (show the start screen), `?storage=indexeddb` / +`?storage=memory` (force a storage backend), `?noworkers` (render and analyse on the page's +thread), `?nosw` (don't register the service worker). ## How the desktop pieces map to the browser | Desktop | Web | |---|---| | frame render thread pool (`ui-egui/src/frames.rs`) | `Frames::pump`: queued frames render on the UI thread after each egui frame, most important first (the viewer's frame, then prefetch), within 40 ms (24 ms while playing) | -| Render Queue on a background thread | `Session::start_render` always runs inline on wasm32: the page waits until the render is done | +| Render Queue on a background thread | a Web Worker running a second engine instance (below); progress streams back, the page never waits | +| tracker / mask tracker / Warp Stabilizer / 3D Camera Tracker / Roto Brush Freeze threads | the same workers; the analysed property group comes back as one undo step | | `rayon` parallel loops (raster, effects, export batches) | rayon's global pool falls back to the calling thread when threads are unavailable: same code, serial | | `std::time::Instant` / `SystemTime` (panic on wasm32) | `web-time` (re-exports `std::time` on native) | -| `std::fs` project reads/writes (`FsServices`) | `files::WebServices`: an in-memory file table; saving downloads the `.ecproj` | +| `std::fs` project reads/writes (`FsServices`) | `files::WebServices`: the virtual file table, persisted (below); saving also downloads the `.ecproj` | +| config directory: settings, shortcut presets, recent projects, the recovery sentinel | `store::WebConfig` in browser storage | +| auto-save folder | `/EffectCraft Auto-Save/` in browser storage (`ConfigStore::files`) | | media reads (`MediaPool`, `probe`) | `files::WebImporter`: bytes from the file table, handed to `MediaPool::add_bytes` / `probe_bytes` | | export writes (`effectcraft-export`) | the job's `sink` (`effectcraft_host::FileExporter { sink }`): files land in the table and download after the render; several files (an image sequence) download as one stored `.zip` | | rfd file dialogs | `` for File ▸ Open / Import; drop files anywhere on the page (`.ecproj` opens, everything else imports); "Save As" / "Output To" pick a download name | | system fonts | not scanned; the bundled fonts (Inter, Noto Serif, JetBrains Mono) are always there | | TCP control channel / MCP | `window.effectcraft` (below) | -| audio output | none yet (if `cpal` lands for desktop playback it must stay behind a cfg/feature off wasm32; a WebAudio backend would replace it) | +| cpal audio output | Web Audio (below) | +| GPU compositor on the desktop's wgpu device | the same compositor on eframe's WebGPU device (below) | -### Threads +### Persistence -wasm threads need a cross-origin isolated page **and** a wasm build with atomics (a nightly -`build-std` toolchain), so this build is single-threaded: frames, effects and exports run on the UI -thread. A long Render Queue job freezes the page until it finishes; keep web renders short or -small (resolution, time span). +Everything that lives in files on the desktop lives in the browser's **Origin Private File +System** (OPFS); where OPFS can't be written from the page (older Safari), **IndexedDB**; with +neither (some private windows), memory only. `effectcraft.info().storage.backend` says which. +The page asks for persistent storage (`navigator.storage.persist()`), which browsers grant to +installed apps, so the store isn't evicted under storage pressure. + +The engine's storage traits are synchronous and the browser's are not, so the app works on an +in-memory mirror (`store.rs`): every stored entry is read before the app starts, reads and writes +hit the mirror, and changes flush in the background (one write per changed key, in order; OPFS +writes are atomic: a swap file replaced on close). Keys: + +| Key | What | +|---|---| +| `config/prefs.json`, `config/shortcuts.json` | Settings (with File ▸ Open Recent's list) and keyboard shortcut presets | +| `config/session.lock` | the crash-recovery sentinel | +| `config/session.ecproj`, `config/session.json` | the session snapshot: the open project and editor state (active comp, time, selection), written at most once a second while they change | +| `files/` | the file table: imported media, projects saved with File ▸ Save / Save As or `saveToBrowser`, auto-saves | + +On the next visit the app reopens the snapshot (unsaved changes stay unsaved: the project is +dirty), registers every stored media file with the media pool so its footage decodes, and File ▸ +Open Recent reopens saved projects from storage. Without a snapshot (first visit) it opens the +demo project. Render outputs are not stored: they download. + +### Background jobs: Web Workers + +wasm threads need shared memory, which needs a nightly `build-std` toolchain, so this build is +single-threaded per instance. Long jobs run in **Web Workers** instead, each with its own instance +of the same module (the page compiles it once and posts the `WebAssembly.Module`, so nothing is +compiled twice) and its own memory (`crates/engine/src/offload.rs`, `src/worker.rs`): + +1. The session's `Offload` serializes a `WorkerRequest`: the project JSON, the footage paths it + reads, and the job (`render` with the resolved queue items and output paths; `warp`, `camera`, + `track`, `maskTrack`, `rotoFreeze` with the target and analysis parameters). +2. `js/host.js` takes an idle worker from its pool (or starts one), sends it the footage bytes it + doesn't have yet, then the request. +3. The worker runs the job on a plain engine session, blocking, and posts `WorkerReply`s: render + progress (at most every 100 ms), item started/finished, rendered files (transferred), analysis + progress, the analysed property group, `failed`, `done`. +4. The page applies them every frame: `Session::poll_render` updates the queue items and the + progress bar, `Session::poll_offload` writes an analysis result as one undo step (the effect / + tracker / mask group is replaced; other edits made meanwhile are kept) and the downloads start. + +Stopping a job terminates its worker (a worker can't be interrupted mid-frame): the item being +rendered becomes "User Stopped", an analysis writes nothing. `renderQueue.render {wait: true}` (the +default for agents) and analysis commands with `wait: true` still run on the page's thread and +return when done. Roto Brush propagation fills the session's segmentation cache rather than the +project, so it runs on the page's thread too, as do viewer frames (`Frames::pump`). + +### Audio + +Audio preview goes through **Web Audio**: an AudioWorklet (`audio-worklet.js`; a +ScriptProcessorNode where worklets are unavailable) plays interleaved stereo blocks. The mixdown +is the desktop's (`AudioPlayback`, `mix_comp`): without a feeder thread, `AudioPlayback::pump` +mixes ahead on every UI frame (≈ 350 ms queued) and the output device takes what keeps it +≈ 150 ms ahead of the playhead. A/V sync follows `AudioContext.currentTime`: the playback clock +is the frames played by the context clock minus its output latency, and the viewer shows the +frame under the audible sample, as on the desktop. The Audio panel's meters read the same feed. + +Browsers start an AudioContext suspended until the user interacts with the page; the app resumes +it on the first pointer or key event, so audio plays from the first preview after any click or +key press. + +### GPU + +eframe starts on WebGPU where the browser has it (WebGL2 otherwise), and the GPU compositor +(`effectcraft-gpu`) runs on the same device: viewer frames are composited by compute shaders and +drawn straight from their texture, with no readback. Steps that need the CPU mid-render (3D runs, +adjustment layers, CPU-only effects) fall back to the CPU renderer for that frame, because the +page's thread can't wait for a GPU readback. The Info panel's pixel readout and the eyedroppers +read GPU frames back asynchronously (`Gpu::read_display_async`: the pixels arrive a frame +later). On WebGL2, or without a usable adapter, everything renders on the CPU. +`effectcraft.info().gpu` reports `{compositor, viewerOnGpu}`. + +### Offline and install (PWA) + +`manifest.webmanifest` (name, icons, standalone display) makes the app installable; `sw.js` +precaches every file of the build into a cache named after the build's hash, serves them +cache-first (the cached responses keep the server's COOP/COEP headers, so the page stays +cross-origin isolated offline), and drops older caches when a new build activates. After the +first visit the app loads without a network. ## `window.effectcraft`: the agent / test API @@ -70,34 +159,53 @@ with the method's `result`, errors reject with the message. | `effectcraft.commands()` / `effectcraft.inspect()` | `engine.commands` / `ui.inspect` | | `effectcraft.renderFrame(params)` | `render.frame` with `base64: true`: `{comp, time, width, height, png}` | | `effectcraft.screenshot(params)` | `ui.screenshot`; without `path` the PNG comes back inline as `png` (base64) | -| `effectcraft.addFile(fileOrUrl, name?)` | put a `File`/`Blob` (or fetched URL) into the file table and open (`.ecproj`) or import it; resolves with its path | +| `effectcraft.addFile(fileOrUrl, name?)` | put a `File`/`Blob` (or fetched URL) into the file table (stored) and open (`.ecproj`) or import it; resolves with its path | | `effectcraft.files()` / `effectcraft.readFile(path)` | the file table `[{path, size}]` / a file's bytes (`Uint8Array`), e.g. a render | -| `effectcraft.info()` | graphics backend, version, `crossOriginIsolated`, load timings | +| `effectcraft.saveToBrowser(path?)` | save the project to browser storage without downloading it (default: its path, or `/.ecproj`); resolves with `{path, bytes}` | +| `effectcraft.listStored()` | `{backend, usage, quota, persisted, pending, files: [{path, size, modified}], config: [name]}` | +| `effectcraft.removeStored(path)` | delete a stored file | +| `effectcraft.flush()` | resolves once every change is written to browser storage | +| `effectcraft.info()` | graphics backend, `gpu`, `storage`, `audio` (`{state, sampleRate, backend, posted, played, underruns}`), `workers`, `restored`, `webgpu`, `serviceWorker`, version, `crossOriginIsolated`, load timings | `window.effectcraftLoad` holds `{wasmMs, readyMs}` (module fetch + compile, and until the app runs). ```js await effectcraft.execute("layer.newSolid", {color: "#ff8800"}); await effectcraft.request("ui.menu.invoke", {id: "renderQueue.add", params: {format: "gif", resolution: 0.25}}); -await effectcraft.execute("renderQueue.render", {}); // the GIF downloads +await effectcraft.execute("renderQueue.render", {wait: false}); // renders in a worker; the GIF downloads +await effectcraft.saveToBrowser("/my-project.ecproj"); // File ▸ Open Recent has it after a reload ``` ## Browser test `apps/effectcraft-web/tests/smoke.mjs` drives headless Chrome over the DevTools protocol (Node ≥ 22, -no npm packages): load, the demo comp in the viewer, `render.frame`, Render Queue GIF and PNG -sequence (`.zip`) downloads, Save As and re-opening the saved project. It writes screenshots and -`report.json`: +no npm packages): load, the GPU path (WebGPU → GPU compositor, viewer on the GPU), the demo comp +in the viewer, `render.frame`, Render Queue GIF and PNG sequence (`.zip`) downloads, a background +render in a worker while measuring the page's event-loop gaps (must stay under 400 ms) and its +progress, the AudioContext starting on a (simulated) key press and a tone playing through it +with the meters moving, Save As and re-opening the saved project, persistence across a reload +(session snapshot, a stored project, imported media, a setting, Open Recent) and an offline +reload through the service worker. It writes screenshots and `report.json`: ```sh cargo xtask web --serve 8765 & node apps/effectcraft-web/tests/smoke.mjs --url http://127.0.0.1:8765/ --out target/web/smoke ``` +Native unit tests cover the storage model (`apps/effectcraft-web/src/store.rs`: write +coalescing, file table, `ConfigStore` / `FileOps`, auto-save and crash recovery through the +browser store) and the worker protocol (`crates/engine/src/offload.rs`: serde round trips of +every request and reply, a render through an in-process offload). + ## Gaps -- Single-threaded (above); Render Queue renders block the page. -- No audio playback. -- Imported media live in memory for the session only (nothing is persisted; reload = fresh start). +- Viewer frames, RAM preview and Roto Brush propagation still render on the page's thread (one + frame at a time between UI frames). +- GPU effects run only in the desktop GPU path: in the browser, effects run on the CPU (inside + workers for renders) because the page's thread can't wait for mid-render readbacks. +- No disk cache (Settings ▸ Media & Disk Cache) in the browser: the layer cache stays in memory. +- Cancelling an analysis drops its partial result (the worker is terminated). +- Stored media count against the origin's storage quota; `effectcraft.listStored()` shows usage, + `removeStored` frees it (there is no storage manager in the UI yet). - `engine.execute` rejects the spread Render Settings / Output Module keys of `renderQueue.add` (`resolution`, `timeSpan`…) through its strict parameter check; use `ui.menu.invoke` as above. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2e2172e1..21928b90 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,14 +23,18 @@ const LAYERS: &[(&str, u8)] = &[ ("time", 0), ("geom", 0), ("color", 0), + ("vp9enc", 0), ("testkit", 0), + ("opusenc", 0), ("raster", 1), ("keyframe", 1), ("path", 1), + ("psd", 1), ("project", 2), ("text", 2), ("effects", 2), ("track", 2), + ("svg", 2), ("model", 2), ("render", 3), ("media", 3), @@ -285,8 +289,14 @@ fn web(args: &[String]) -> Result<(), String> { std::fs::copy(e.path(), dist.join(&name)).map_err(|e| format!("{name}: {e}"))?; } } + // PWA icons: the app icon (assets/app-icon, attributed there). + for n in [256, 512] { + let src = format!("assets/app-icon/hicolor/{n}x{n}/apps/ai.storyteller.effectcraft.png"); + std::fs::copy(&src, dist.join(format!("icon-{n}.png"))).map_err(|e| format!("{src}: {e}"))?; + } + service_worker(&dist)?; let size = |p: &std::path::Path| std::fs::metadata(p).map(|m| m.len()).unwrap_or(0); - let total: u64 = std::fs::read_dir(&dist).map_err(|e| e.to_string())?.flatten().map(|e| size(&e.path())).sum(); + let total: u64 = dist_files(&dist).iter().map(|f| size(&dist.join(f))).sum(); println!("web: {} ({:.1} MB wasm, {:.1} MB total)", dist.display(), size(&bg) as f64 / 1e6, total as f64 / 1e6); if let Some(port) = serve { serve_dir(&dist, port)?; @@ -294,6 +304,42 @@ fn web(args: &[String]) -> Result<(), String> { Ok(()) } +/// Every file under `dist`, as `/`-separated relative paths, sorted. +fn dist_files(dist: &std::path::Path) -> Vec { + fn walk(dir: &std::path::Path, rel: &str, out: &mut Vec) { + for e in std::fs::read_dir(dir).into_iter().flatten().flatten() { + let name = e.file_name().to_string_lossy().to_string(); + let r = if rel.is_empty() { name } else { format!("{rel}/{name}") }; + if e.path().is_dir() { + walk(&e.path(), &r, out); + } else { + out.push(r); + } + } + } + let mut v = vec![]; + walk(dist, "", &mut v); + v.sort(); + v +} + +/// Fill the service worker's precache list and version (a hash of the build's files). +fn service_worker(dist: &std::path::Path) -> Result<(), String> { + let sw = dist.join("sw.js"); + let text = std::fs::read_to_string(&sw).map_err(|e| format!("sw.js: {e}"))?; + let files: Vec = dist_files(dist).into_iter().filter(|f| f != "sw.js").collect(); + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for f in &files { + for b in f.bytes().chain(std::fs::read(dist.join(f)).map_err(|e| format!("{f}: {e}"))?) { + h ^= b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + } + let list: Vec = std::iter::once("./".to_string()).chain(files.iter().map(|f| format!("./{f}"))).map(|f| format!("{f:?}")).collect(); + let text = text.replace("__EC_VERSION__", &format!("{h:016x}")).replace("__EC_FILES__", &format!("[{}]", list.join(", "))); + std::fs::write(&sw, text).map_err(|e| format!("sw.js: {e}")) +} + /// A tiny static file server for the web build (localhost only). Sends the cross-origin /// isolation headers so `crossOriginIsolated` is true, like a production deployment would. fn serve_dir(dir: &std::path::Path, port: u16) -> Result<(), String> { @@ -330,7 +376,7 @@ fn serve_dir(dir: &std::path::Path, port: u16) -> Result<(), String> { "js" => "text/javascript", "wasm" => "application/wasm", "svg" => "image/svg+xml", - "json" => "application/json", + "json" | "webmanifest" => "application/json", "png" => "image/png", "gif" => "image/gif", "ecproj" => "application/json",