From a1cc0a6838e098f99f45f356c4d1beb3374820f6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 09:06:56 -0400 Subject: [PATCH] M11.9: proxies (create / attach / toggle / reconnect) and ingest settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - media.createProxies: background job (jobs.list progress) transcoding with our own encoders — ProRes 422 Proxy ¼/½, ProRes LT ½, H.264 ¼/½ (audio included) — into /Proxies or a chosen folder; attached when the job finishes (Session::poll_persistence applies finished media jobs). - media.attachProxies (duration ±1 frame and frame rate must match; size/aspect may differ), media.detachProxies, media.reconnectFullRes, media.toggleProxies (Preferences ▸ Media ▸ enableProxies; bumps the view revision without dirtying the project), media.proxyPresets. - Proxy frames are smaller; ProxySource reports the original size and the compositor derives its pixel scale from the frame, so Motion and effects render the same picture (44 dB PSNR vs full res with Motion + Gaussian Blur, 48 dB at ½). Export always renders full resolution. - ExportSettings::prores_profile (proxy / lt / standard / hq). - project.ingestSettings + file.import: copy (fingerprint-verified), transcode, create proxies, copy and create proxies. - UI: Toggle Proxies button on both monitors (lit when on), Create Proxies dialog, Project panel Proxy submenu and P badges. - Measured (one core, 4K H.264 testsrc2): CPU composite 201 → 35 ms/frame at ¼ with ProRes Proxy ¼, 277 → 131 ms at ½ with H.264 ¼; decode 9.2 → 0.7 ms with H.264 ¼. - Tests: create+attach+toggle same composite modulo scale, attach checks, export ignores proxies, ingest copy + proxies, ignored 4K benchmark, headless UI test. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + crates/engine/Cargo.toml | 4 + crates/engine/src/autosave.rs | 10 + crates/engine/src/commands.rs | 14 + crates/engine/src/lib.rs | 25 +- crates/engine/src/previews.rs | 1 + crates/engine/src/proxies.rs | 587 +++++++++++++++++++++ crates/engine/src/proxies_tests.rs | 206 ++++++++ crates/export/src/lib.rs | 19 +- crates/project/src/lib.rs | 31 ++ crates/ui-egui/src/icons.rs | 8 + crates/ui-egui/src/panels/media_dialogs.rs | 74 ++- crates/ui-egui/src/panels/monitor.rs | 2 + crates/ui-egui/src/panels/project.rs | 14 + crates/ui-egui/src/state.rs | 14 + crates/ui-egui/tests/media_ui.rs | 31 +- docs/architecture.md | 2 +- docs/control-protocol.md | 8 +- docs/project-files.md | 69 ++- 19 files changed, 1104 insertions(+), 17 deletions(-) create mode 100644 crates/engine/src/proxies.rs create mode 100644 crates/engine/src/proxies_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 72dbdba5..cd6f7751 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1441,8 +1441,10 @@ dependencies = [ "filmcraft-media", "filmcraft-project", "filmcraft-render", + "filmcraft-testkit", "filmcraft-text", "filmcraft-time", + "image", "log", "serde", "serde_json", diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 287c1a32..1305de24 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -28,3 +28,7 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } log = { workspace = true } + +[dev-dependencies] +filmcraft-testkit = { workspace = true } +image = { version = "0.25", default-features = false, features = ["png"] } diff --git a/crates/engine/src/autosave.rs b/crates/engine/src/autosave.rs index e66b7c2d..05c2355e 100644 --- a/crates/engine/src/autosave.rs +++ b/crates/engine/src/autosave.rs @@ -104,6 +104,16 @@ pub struct Preferences { pub audio: AudioPrefs, pub playback: PlaybackPrefs, pub trim: TrimPrefs, + pub media: MediaPrefs, +} + +/// Preferences ▸ Media. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct MediaPrefs { + /// "Enable proxies": playback and monitors read attached proxies (the monitors' Toggle + /// Proxies button). Export always uses full-resolution media. + pub enable_proxies: bool, } /// Preferences ▸ Audio (the mixer-automation part) and the Track Mixer panel-menu toggle. diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index abb85973..bca4e4a1 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -543,6 +543,18 @@ fn build() -> Vec { if ids.is_empty() && sequences.is_empty() && reports.is_empty() && !errors.is_empty() { return Err(EngineError::Other(errors.join("; "))); } + // Project Settings ▸ Ingest: copy / transcode / create proxies + let item_ids: Vec = ids.iter().map(|i| ItemId(*i)).collect(); + let ingest = match crate::proxies::ingest(s, &item_ids) { + Ok(v) => v, + Err(e) => { + errors.push(format!("ingest: {e}")); + Value::Null + } + }; + if !ingest.is_null() { + return Ok(json!({"items": ids, "sequences": sequences, "documents": reports, "errors": errors, "ingest": ingest})); + } if reports.is_empty() { Ok(json!({"items": ids, "errors": errors})) } else { @@ -1854,6 +1866,7 @@ fn build() -> Vec { v.extend(crate::graphics::commands()); v.extend(crate::shortcuts::commands()); v.extend(crate::relink::commands()); + v.extend(crate::proxies::commands()); // Labels as individual commands (Edit ▸ Label ▸ ) for l in Label::ALL { let _ = l; @@ -1879,6 +1892,7 @@ fn export_media(s: &mut Session, p: &Value) -> Result { bitrate_kbps: u64_p(p, "bitrateKbps").unwrap_or(20_000) as u32, burn_captions: bool_p(p, "burnCaptions").unwrap_or(false), part_of_batch: false, + prores_profile: str_p(p, "proresProfile").unwrap_or_default().to_string(), }; let id = s.jobs.len() as u64 + 1; let job = crate::Job { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3b183117..ee7b4b8c 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -17,6 +17,7 @@ pub mod interchange; pub mod media_pool; pub mod mixer; pub mod previews; +pub mod proxies; pub mod relink; pub mod shortcut_presets; pub mod shortcuts; @@ -207,6 +208,8 @@ pub struct Session { pub shortcuts: shortcuts::Shortcuts, /// Missing / offline media found by the last scan (Link Media dialog). pub offline: relink::OfflineState, + /// Proxy / ingest / project-manager jobs whose results still have to be applied to the project. + pub media_jobs: Vec, /// Nesting depth of [`Session::execute`] (commands that run other commands). exec_depth: u32, } @@ -272,6 +275,7 @@ impl Session { trim_play: Default::default(), shortcuts: shortcuts::Shortcuts::new(), offline: Default::default(), + media_jobs: Vec::new(), exec_depth: 0, } } @@ -282,6 +286,7 @@ impl Session { pub fn start_autosave(&mut self, cfg: autosave::AutosaveConfig) -> std::io::Result<()> { let prefs_path = cfg.data_dir.join("preferences.json"); self.prefs = autosave::Preferences::load(&prefs_path); + self.media.set_use_proxies(self.prefs.media.enable_proxies); self.shortcuts.set_dir(&cfg.data_dir); self.prefs_path = Some(prefs_path); self.persistence = Some(autosave::Persistence::start(&cfg, self.prefs.auto_save.clone())?); @@ -319,8 +324,10 @@ impl Session { p.last_sent = Some(cur); } - /// Apply worker notifications (call regularly, e.g. once per UI frame). + /// Apply worker notifications (call regularly, e.g. once per UI frame). Also applies the + /// results of finished proxy / ingest jobs. pub fn poll_persistence(&mut self) { + proxies::poll(self); let Some(p) = self.persistence.as_mut() else { return }; for ev in p.drain_events() { match ev { @@ -339,6 +346,10 @@ impl Session { /// Replace preferences (persisting them and updating the worker). pub fn set_prefs(&mut self, p: autosave::Preferences) -> std::io::Result<()> { self.prefs = p; + if self.media.use_proxies() != self.prefs.media.enable_proxies { + self.media.set_use_proxies(self.prefs.media.enable_proxies); + self.bump_view(); + } if let Some(w) = &self.persistence { w.set_prefs(self.prefs.auto_save.clone()); } @@ -357,6 +368,16 @@ impl Session { } } + /// A change of what the project *looks like* that isn't an edit (proxies toggled): bumps the + /// revision so frame caches refresh, without marking a clean project as modified. + pub fn bump_view(&mut self) { + let clean = !self.is_dirty(); + self.bump(); + if clean { + self.saved_revision = self.revision; + } + } + pub fn is_dirty(&self) -> bool { self.revision != self.saved_revision } @@ -590,6 +611,8 @@ mod mixer_tests; #[cfg(test)] mod previews_tests; #[cfg(test)] +mod proxies_tests; +#[cfg(test)] mod relink_tests; #[cfg(test)] mod shortcuts_tests; diff --git a/crates/engine/src/previews.rs b/crates/engine/src/previews.rs index c49db99c..2117dc08 100644 --- a/crates/engine/src/previews.rs +++ b/crates/engine/src/previews.rs @@ -459,6 +459,7 @@ pub fn render(s: &mut Session, mode: RenderMode, p: &Value) -> Result { // Captions stay live over previews and are not part of the preview hash. burn_captions: false, part_of_batch: true, + prores_profile: String::new(), }; match filmcraft_export::export(&project, seq, &settings, &provider, &prog) { Ok(r) => { diff --git a/crates/engine/src/proxies.rs b/crates/engine/src/proxies.rs new file mode 100644 index 00000000..beff7d65 --- /dev/null +++ b/crates/engine/src/proxies.rs @@ -0,0 +1,587 @@ +//! Proxies and ingest (Clip ▸ Proxy ▸ Create / Attach / Reconnect Full Resolution, Toggle Proxies, +//! Project Settings ▸ Ingest Settings). +//! +//! - **Create.** A background job transcodes each clip with our own encoders (ProRes 422 Proxy / +//! LT or H.264 at ¼ or ½ size, audio included) and attaches the results when it finishes +//! ([`poll`] applies finished jobs; the frontend calls it every frame through +//! `Session::poll_persistence`). +//! - **Attach.** A proxy must have the original's duration (±1 frame) and frame rate; its frame +//! size and aspect may differ. +//! - **Toggle.** `media.toggleProxies` flips Preferences ▸ Media ▸ Enable proxies; the media pool +//! then hands proxies to monitors and playback. Export always uses full resolution. +//! - **Ingest.** With ingest enabled, `file.import` copies (verified by fingerprint), transcodes +//! and/or creates proxies. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use filmcraft_export::{ExportSettings, Format, Progress, Report}; +use filmcraft_media::{MediaKind, SharedSource}; +use filmcraft_project::{IngestAction, ItemId, ItemKind, MediaClip, MediaRef, Project, SequenceSettings, TrackKind}; +use filmcraft_time::{Tick, TimeRange}; +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::commands::{CommandSpec, always, bad, bool_p, str_p, u64_p}; +use crate::{EngineError, Result, Session}; + +/// A proxy / transcode preset. +#[derive(Clone, Copy, Debug, Serialize)] +pub struct Preset { + pub id: &'static str, + pub label: &'static str, + #[serde(skip)] + pub format: Format, + /// Frame size relative to the original. + pub scale: f32, + /// ProRes flavour (`proxy`, `lt`, `hq`) for ProRes presets. + pub prores: &'static str, + pub extension: &'static str, + /// Offered for proxies (else only for transcodes). + pub proxy: bool, +} + +pub const PRESETS: &[Preset] = &[ + Preset { + id: "prores_proxy_quarter", + label: "ProRes 422 Proxy, ¼ size", + format: Format::ProRes, + scale: 0.25, + prores: "proxy", + extension: "mov", + proxy: true, + }, + Preset { id: "prores_proxy_half", label: "ProRes 422 Proxy, ½ size", format: Format::ProRes, scale: 0.5, prores: "proxy", extension: "mov", proxy: true }, + Preset { id: "prores_lt_half", label: "ProRes 422 LT, ½ size", format: Format::ProRes, scale: 0.5, prores: "lt", extension: "mov", proxy: true }, + Preset { id: "h264_quarter", label: "H.264, ¼ size", format: Format::H264, scale: 0.25, prores: "", extension: "mp4", proxy: true }, + Preset { id: "h264_half", label: "H.264, ½ size", format: Format::H264, scale: 0.5, prores: "", extension: "mp4", proxy: true }, + Preset { id: "prores_lt", label: "ProRes 422 LT (full size)", format: Format::ProRes, scale: 1.0, prores: "lt", extension: "mov", proxy: false }, + Preset { id: "prores_hq", label: "ProRes 422 HQ (full size)", format: Format::ProRes, scale: 1.0, prores: "hq", extension: "mov", proxy: false }, + Preset { id: "h264", label: "H.264 (full size)", format: Format::H264, scale: 1.0, prores: "", extension: "mp4", proxy: false }, +]; + +pub const DEFAULT_PROXY_PRESET: &str = "prores_proxy_quarter"; +pub const DEFAULT_TRANSCODE_PRESET: &str = "prores_lt"; + +pub fn preset(id: &str) -> Option<&'static Preset> { + PRESETS.iter().find(|p| p.id == id) +} + +/// What to do with a finished job's outputs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OnDone { + /// Attach outputs as proxies. + AttachProxies, + /// Point the clips at the outputs (ingest transcode). + ReplaceMedia, + /// Nothing (the job's own result is the product, e.g. Project Manager). + Nothing, +} + +/// A media job whose outputs still have to be applied to the project. +pub struct PendingJob { + pub job: u64, + pub on_done: OnDone, + /// (item, output path) for each finished output. + pub outputs: Arc>>, +} + +/// Transcode `range` (whole media when None) of one media clip to `out` with `preset`, blocking. +/// `progress.total` is the caller's; this adds one per frame to `progress.done`. +pub fn transcode( + clip: &MediaClip, + name: &str, + src: SharedSource, + range: Option, + out: &str, + preset: &Preset, + progress: &Progress, +) -> std::result::Result { + let info = &clip.info; + let rate = clip.frame_rate(); + let mut st = SequenceSettings::default(); + if let Some(v) = &info.video { + st.width = v.width; + st.height = v.height; + } + st.frame_rate = rate; + st.sample_rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate.max(8000)); + let mut p = Project::new("transcode"); + let mut c = clip.clone(); + c.offline = false; + c.proxy = None; + let item = p.add_item(name, filmcraft_project::Label::Iris, ItemKind::Media(c), None); + let has_v = info.video.is_some(); + let has_a = info.audio.is_some(); + let seq = p.new_sequence("transcode", st, usize::from(has_v), usize::from(has_a), None); + let range = range.unwrap_or(TimeRange { start: Tick::ZERO, duration: info.duration }); + let mut placed = Vec::new(); + if has_v && let Some(ti) = p.make_track_item(item, TrackKind::Video, Tick::ZERO, range, rate) { + placed.push((TrackKind::Video, ti)); + } + if has_a && let Some(ti) = p.make_track_item(item, TrackKind::Audio, Tick::ZERO, range, rate) { + placed.push((TrackKind::Audio, ti)); + } + let q = p.sequence_mut(seq).ok_or("no sequence")?; + for (k, ti) in placed { + let t = if k == TrackKind::Video { &mut q.video_tracks[0] } else { &mut q.audio_tracks[0] }; + t.items.push(ti); + } + let format = if has_v { preset.format } else { Format::Wav }; + let settings = ExportSettings { + format, + path: out.to_string(), + scale: preset.scale, + include_audio: has_a, + quality: 85, + bitrate_kbps: ((info.video.as_ref().map_or(1920 * 1080, |v| v.width * v.height) as f64 * (preset.scale * preset.scale) as f64) / 200.0).max(800.0) + as u32, + part_of_batch: true, + prores_profile: preset.prores.to_string(), + ..Default::default() + }; + let project = Arc::new(p); + let provider = move |id: ItemId| (id == item).then(|| src.clone()); + filmcraft_export::export(&project, seq, &settings, &provider, progress).map_err(|e| e.to_string()) +} + +/// Frames a transcode of `clip` will write (for progress totals). +pub fn frame_count(clip: &MediaClip, range: Option) -> u64 { + let rate = clip.frame_rate(); + let d = range.map_or(clip.info.duration, |r| r.duration); + if clip.info.video.is_none() { 1 } else { rate.frame_at(d).max(1) as u64 } +} + +/// `/.`, numbered when the name is taken. +pub fn unique_path(dir: &Path, stem: &str, suffix: &str, ext: &str, taken: &[String]) -> PathBuf { + let mut n = 1; + loop { + let name = if n == 1 { format!("{stem}{suffix}.{ext}") } else { format!("{stem}{suffix} {n}.{ext}") }; + let p = dir.join(name); + if !p.exists() && !taken.contains(&p.to_string_lossy().into_owned()) { + return p; + } + n += 1; + } +} + +fn media_path(p: &Project, item: ItemId) -> Option<(MediaClip, String, String)> { + let it = p.item(item)?; + let m = it.as_media()?; + match &m.media { + MediaRef::File { path } => Some((m.clone(), path.clone(), it.name.clone())), + MediaRef::Generator(_) => None, + } +} + +fn items_param(s: &Session, p: &Value) -> Vec { + match p.get("items").and_then(Value::as_array) { + Some(a) => a.iter().filter_map(Value::as_u64).map(ItemId).collect(), + None => u64_p(p, "item").map(|i| vec![ItemId(i)]).unwrap_or_else(|| s.state.project_selection.clone()), + } +} + +/// Start a background transcode job over `work` (item, output path, preset). Returns the job id. +pub fn start_job(s: &mut Session, label: String, work: Vec<(ItemId, String, &'static Preset)>, on_done: OnDone, wait: bool) -> Result { + let mut tasks = Vec::new(); + let mut total = 0; + for (item, out, pr) in work { + let (clip, _, name) = media_path(&s.project, item).ok_or_else(|| bad("media.createProxies", format!("item {} is not file media", item.0)))?; + let src = s.media.full_res_source(&s.project, item, &*s.services).ok_or_else(|| EngineError::Other(format!("{name}: no source")))?; + if s.media.offline_status(item).is_some() { + return Err(EngineError::Other(format!("{name} is offline; link it first"))); + } + total += frame_count(&clip, None); + tasks.push((item, clip, name, src, out, pr)); + } + let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1; + let job = crate::Job { id, label, progress: Default::default(), result: Default::default() }; + job.progress.total.store(total, std::sync::atomic::Ordering::Relaxed); + let outputs: Arc>> = Arc::default(); + let (prog, res, outs) = (job.progress.clone(), job.result.clone(), outputs.clone()); + let run = move || { + let t0 = std::time::Instant::now(); + let mut bytes = 0; + let mut err = None; + let n = tasks.len(); + for (k, (item, clip, name, src, out, pr)) in tasks.into_iter().enumerate() { + if prog.cancel.load(std::sync::atomic::Ordering::Relaxed) { + err = Some("cancelled".to_string()); + break; + } + *prog.status.lock().unwrap_or_else(|e| e.into_inner()) = format!("{name} ({} of {n})", k + 1); + if let Some(d) = Path::new(&out).parent() { + let _ = std::fs::create_dir_all(d); + } + match transcode(&clip, &name, src, None, &out, pr, &prog) { + Ok(r) => { + bytes += r.bytes; + outs.lock().unwrap_or_else(|e| e.into_inner()).push((item, out)); + } + Err(e) => { + let _ = std::fs::remove_file(&out); + err = Some(format!("{name}: {e}")); + break; + } + } + } + let secs = t0.elapsed().as_secs_f64(); + let frames = prog.done.load(std::sync::atomic::Ordering::Relaxed); + let r = match err { + Some(e) => { + *prog.error.lock().unwrap_or_else(|x| x.into_inner()) = Some(e.clone()); + Err(e) + } + None => Ok(Report { path: String::new(), frames, seconds: secs, bytes, render_fps: frames as f64 / secs.max(1e-6) }), + }; + *prog.status.lock().unwrap_or_else(|e| e.into_inner()) = match &r { + Ok(_) => format!("Done in {secs:.1}s"), + Err(e) => e.clone(), + }; + prog.finished.store(true, std::sync::atomic::Ordering::Relaxed); + *res.lock().unwrap_or_else(|x| x.into_inner()) = Some(r); + }; + s.jobs.push(job); + s.media_jobs.push(PendingJob { job: id, on_done, outputs }); + if wait || cfg!(target_arch = "wasm32") { + run(); + poll(s); + } else { + std::thread::Builder::new().name("filmcraft-proxies".into()).spawn(run).map_err(|e| EngineError::Other(e.to_string()))?; + } + Ok(id) +} + +/// Apply the outputs of finished media jobs (attach proxies, switch to transcodes). +pub fn poll(s: &mut Session) { + use std::sync::atomic::Ordering; + let finished: Vec = s + .media_jobs + .iter() + .enumerate() + .filter(|(_, pj)| s.jobs.iter().find(|j| j.id == pj.job).is_none_or(|j| j.progress.finished.load(Ordering::Relaxed))) + .map(|(i, _)| i) + .collect(); + for i in finished.into_iter().rev() { + let pj = s.media_jobs.remove(i); + let outs = std::mem::take(&mut *pj.outputs.lock().unwrap_or_else(|e| e.into_inner())); + if outs.is_empty() { + continue; + } + let r = match pj.on_done { + OnDone::AttachProxies => attach(s, &outs, false).map(|_| ()), + OnDone::ReplaceMedia => replace_media(s, &outs), + OnDone::Nothing => Ok(()), + }; + if let Err(e) = r { + s.events.push(crate::Event::Toast { message: e.to_string(), error: true }); + } + } +} + +/// Check a proxy file against its clip: duration within one frame, same frame rate. +fn check_proxy(s: &Session, item: ItemId, path: &str) -> Result<()> { + let (clip, _, name) = media_path(&s.project, item).ok_or_else(|| bad("media.attachProxies", format!("item {} is not file media", item.0)))?; + let src = s.media.open_file(path, &*s.services).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; + let pi = src.info(); + let fd = clip.info.frame_rate().frame_duration(); + if (pi.duration - clip.info.duration).0.abs() > fd.0 { + return Err(EngineError::Other(format!( + "{}: the proxy is {:.3}s long but {name} is {:.3}s", + Path::new(path).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(), + pi.duration.seconds(), + clip.info.duration.seconds() + ))); + } + if let (Some(a), Some(b)) = (&pi.video, &clip.info.video) + && a.frame_rate != b.frame_rate + { + return Err(EngineError::Other(format!("the proxy runs at {} fps but {name} at {} fps", a.frame_rate.label(), b.frame_rate.label()))); + } + if pi.video.is_none() && clip.info.video.is_some() { + return Err(EngineError::Other("the proxy has no video".into())); + } + Ok(()) +} + +/// Attach proxies (checked unless `force`). One undo step. +pub fn attach(s: &mut Session, pairs: &[(ItemId, String)], force: bool) -> Result { + if !force { + for (i, p) in pairs { + check_proxy(s, *i, p)?; + } + } + let pairs = pairs.to_vec(); + s.edit("Attach Proxies", |proj, _| { + for (i, p) in &pairs { + if let Some(m) = proj.item_mut(*i).and_then(|it| it.as_media_mut()) { + m.proxy = Some(MediaRef::File { path: p.clone() }); + } + } + Ok(()) + })?; + Ok(json!({"attached": pairs.iter().map(|(i, p)| json!({"item": i.0, "path": p})).collect::>()})) +} + +fn replace_media(s: &mut Session, outs: &[(ItemId, String)]) -> Result<()> { + for (i, p) in outs { + let q = json!({"item": i.0, "path": p, "force": true, "relinkOthers": false}); + crate::relink::relink(s, &q)?; + } + Ok(()) +} + +/// Where proxies of `media` go: the chosen folder, else `/Proxies`. +pub fn proxy_dir(media: &str, dest: Option<&str>) -> PathBuf { + match dest { + Some(d) if !d.is_empty() => PathBuf::from(d), + _ => Path::new(media).parent().unwrap_or(Path::new(".")).join("Proxies"), + } +} + +/// `media.createProxies {items?, preset?, destination?, wait?}`. +pub fn create(s: &mut Session, p: &Value) -> Result { + let pr = preset(str_p(p, "preset").unwrap_or(DEFAULT_PROXY_PRESET)).ok_or_else(|| bad("media.createProxies", "unknown preset (see media.proxyPresets)"))?; + let items = items_param(s, p); + let dest = str_p(p, "destination").map(str::to_string); + let mut work = Vec::new(); + let mut taken = Vec::new(); + let mut skipped = Vec::new(); + for i in items { + let Some((clip, path, name)) = media_path(&s.project, i) else { + skipped.push(json!({"item": i.0, "reason": "not file media"})); + continue; + }; + if !matches!(clip.info.kind, MediaKind::Movie) || clip.info.video.is_none() { + skipped.push(json!({"item": i.0, "reason": format!("{name} has no video to proxy")})); + continue; + } + let stem = Path::new(&path).file_stem().map(|n| n.to_string_lossy().into_owned()).unwrap_or(name); + let out = unique_path(&proxy_dir(&path, dest.as_deref()), &stem, "_Proxy", pr.extension, &taken).to_string_lossy().into_owned(); + taken.push(out.clone()); + work.push((i, out, pr)); + } + if work.is_empty() { + return Err(EngineError::Other(format!("nothing to proxy: {}", skipped.iter().filter_map(|v| v["reason"].as_str()).collect::>().join("; ")))); + } + let outputs: Vec = work.iter().map(|(i, o, _)| json!({"item": i.0, "path": o})).collect(); + let n = work.len(); + let label = format!("Create Proxies ({n} clip{})", if n == 1 { "" } else { "s" }); + let attach = bool_p(p, "attach").unwrap_or(true); + let job = start_job(s, label, work, if attach { OnDone::AttachProxies } else { OnDone::Nothing }, bool_p(p, "wait").unwrap_or(false))?; + Ok(json!({"job": job, "preset": pr.id, "outputs": outputs, "skipped": skipped})) +} + +/// Ingest freshly imported items per the project's ingest settings. +pub fn ingest(s: &mut Session, items: &[ItemId]) -> Result { + let ing = s.project.settings.ingest.clone(); + if !ing.enabled || items.is_empty() { + return Ok(Value::Null); + } + let mut copied = Vec::new(); + let copy = matches!(ing.action, IngestAction::Copy | IngestAction::CopyAndCreateProxies); + if copy { + for &i in items { + let Some((_, path, _)) = media_path(&s.project, i) else { continue }; + let dir = match &ing.destination { + Some(d) if !d.is_empty() => PathBuf::from(d), + _ => Path::new(&path).parent().unwrap_or(Path::new(".")).join("Ingested Media"), + }; + let fname = Path::new(&path).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let out = dir.join(&fname); + if Path::new(&path) == out { + continue; + } + std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{}: {e}", dir.display())))?; + std::fs::copy(&path, &out).map_err(|e| EngineError::Other(format!("copying {fname}: {e}")))?; + let out_s = out.to_string_lossy().into_owned(); + // verify the copy before switching to it + let a = crate::relink::identity_of(&*s.services, &path).map_err(|e| EngineError::Other(e.to_string()))?; + let b = crate::relink::identity_of(&*s.services, &out_s).map_err(|e| EngineError::Other(e.to_string()))?; + if a != b { + return Err(EngineError::Other(format!("the copy of {fname} doesn't match the original"))); + } + crate::relink::relink(s, &json!({"item": i.0, "path": out_s, "relinkOthers": false}))?; + copied.push(json!({"item": i.0, "path": out_s})); + } + } + let mut job = Value::Null; + match ing.action { + IngestAction::CreateProxies | IngestAction::CopyAndCreateProxies => { + let p = json!({"items": items.iter().map(|i| i.0).collect::>(), "preset": if ing.preset.is_empty() { DEFAULT_PROXY_PRESET } else { ing.preset.as_str() }, "destination": ing.destination}); + if items.iter().any(|i| s.project.item(*i).and_then(|x| x.as_media()).is_some_and(|m| m.info.video.is_some() && m.info.kind == MediaKind::Movie)) { + job = create(s, &p)?; + } + } + IngestAction::Transcode => { + let pr = + preset(if ing.preset.is_empty() { DEFAULT_TRANSCODE_PRESET } else { ing.preset.as_str() }).ok_or_else(|| bad("ingest", "unknown preset"))?; + let mut work = Vec::new(); + for &i in items { + let Some((clip, path, name)) = media_path(&s.project, i) else { continue }; + if clip.info.video.is_none() { + continue; + } + let dir = match &ing.destination { + Some(d) if !d.is_empty() => PathBuf::from(d), + _ => Path::new(&path).parent().unwrap_or(Path::new(".")).join("Ingested Media"), + }; + let stem = Path::new(&path).file_stem().map(|n| n.to_string_lossy().into_owned()).unwrap_or(name); + work.push((i, unique_path(&dir, &stem, "", pr.extension, &[]).to_string_lossy().into_owned(), pr)); + } + if !work.is_empty() { + let id = start_job(s, format!("Ingest: transcode {} clip(s)", work.len()), work, OnDone::ReplaceMedia, false)?; + job = json!({"job": id}); + } + } + IngestAction::Copy => {} + } + Ok(json!({"copied": copied, "job": job})) +} + +pub fn commands() -> Vec { + vec![ + CommandSpec { + id: "media.createProxies", + label: "Create Proxies…", + menu: &["Clip", "Proxy"], + shortcut: None, + params: r#"{"items":[id]?,"preset":"prores_proxy_quarter|prores_proxy_half|prores_lt_half|h264_quarter|h264_half","destination":str?,"attach":bool=true,"wait":bool=false}"#, + enabled: has_file_media, + run: create, + journal: true, + }, + CommandSpec { + id: "media.attachProxies", + label: "Attach Proxies…", + menu: &["Clip", "Proxy"], + shortcut: None, + params: r#"{"item":id,"path":str}|{"items":[id],"paths":[str]},"force":bool=false"#, + enabled: has_file_media, + run: |s, p| { + let pairs: Vec<(ItemId, String)> = match (p.get("items").and_then(Value::as_array), p.get("paths").and_then(Value::as_array)) { + (Some(i), Some(ps)) => i.iter().zip(ps).filter_map(|(i, p)| Some((ItemId(i.as_u64()?), p.as_str()?.to_string()))).collect(), + _ => vec![( + u64_p(p, "item").map(ItemId).ok_or_else(|| bad("media.attachProxies", "need `item`"))?, + str_p(p, "path").ok_or_else(|| bad("media.attachProxies", "need `path`"))?.to_string(), + )], + }; + attach(s, &pairs, bool_p(p, "force").unwrap_or(false)) + }, + journal: true, + }, + CommandSpec { + id: "media.detachProxies", + label: "Detach Proxies", + menu: &["Clip", "Proxy"], + shortcut: None, + params: r#"{"items":[id]?}"#, + enabled: has_file_media, + run: |s, p| { + let items = items_param(s, p); + let n = s.edit("Detach Proxies", |proj, _| { + let mut n = 0; + for i in &items { + if let Some(m) = proj.item_mut(*i).and_then(|it| it.as_media_mut()) + && m.proxy.take().is_some() + { + n += 1; + } + } + Ok(n) + })?; + Ok(json!({"detached": n})) + }, + journal: true, + }, + CommandSpec { + id: "media.reconnectFullRes", + label: "Reconnect Full Resolution Media…", + menu: &["Clip", "Proxy"], + shortcut: None, + params: r#"{"item":id,"path":str}"#, + enabled: has_file_media, + run: |s, p| { + let item = u64_p(p, "item").map(ItemId).ok_or_else(|| bad("media.reconnectFullRes", "need `item`"))?; + if s.project.item(item).and_then(|i| i.as_media()).is_none_or(|m| m.proxy.is_none()) { + return Err(EngineError::Other("the clip has no proxy attached".into())); + } + let mut q = p.clone(); + q["match"] = json!({"fileName": false, "extension": false}); + q["relinkOthers"] = json!(false); + crate::relink::relink(s, &q) + }, + journal: true, + }, + CommandSpec { + id: "media.toggleProxies", + label: "Toggle Proxies", + menu: &["View"], + shortcut: None, + params: r#"{"enabled":bool?}"#, + enabled: always, + run: |s, p| { + let on = bool_p(p, "enabled").unwrap_or(!s.prefs.media.enable_proxies); + let mut next = s.prefs.clone(); + next.media.enable_proxies = on; + s.set_prefs(next).map_err(|e| EngineError::Other(format!("saving preferences: {e}")))?; + let attached = s.project.items.values().filter(|i| i.as_media().is_some_and(|m| m.proxy.is_some())).count(); + Ok(json!({"enabled": on, "clipsWithProxies": attached})) + }, + journal: false, + }, + CommandSpec { + id: "media.proxyPresets", + label: "Proxy Presets", + menu: &[], + shortcut: None, + params: "{}", + enabled: always, + run: |_, _| Ok(serde_json::to_value(PRESETS).unwrap_or_default()), + journal: false, + }, + CommandSpec { + id: "project.ingestSettings", + label: "Ingest Settings…", + menu: &[], + shortcut: None, + params: r#"{"enabled":bool?,"action":"copy|transcode|createProxies|copyAndCreateProxies"?,"destination":str?,"preset":str?}"#, + enabled: always, + run: |s, p| { + let mut ing = s.project.settings.ingest.clone(); + if let Some(b) = bool_p(p, "enabled") { + ing.enabled = b; + } + if let Some(a) = p.get("action") { + ing.action = serde_json::from_value(a.clone()).map_err(|e| bad("project.ingestSettings", e.to_string()))?; + } + if let Some(d) = p.get("destination") { + ing.destination = d.as_str().filter(|d| !d.is_empty()).map(str::to_string); + } + if let Some(pr) = str_p(p, "preset") { + if !pr.is_empty() && preset(pr).is_none() { + return Err(bad("project.ingestSettings", format!("unknown preset `{pr}`"))); + } + ing.preset = pr.to_string(); + } + if ing != s.project.settings.ingest { + let v = ing.clone(); + s.edit("Ingest Settings", |proj, _| { + proj.settings.ingest = v; + Ok(()) + })?; + } + Ok(serde_json::to_value(&s.project.settings.ingest).unwrap_or_default()) + }, + journal: true, + }, + ] +} + +fn has_file_media(s: &Session) -> std::result::Result<(), String> { + if s.project.items.values().any(|i| i.as_media().is_some_and(|m| matches!(m.media, MediaRef::File { .. }))) { + Ok(()) + } else { + Err("the project has no file-based media".into()) + } +} diff --git a/crates/engine/src/proxies_tests.rs b/crates/engine/src/proxies_tests.rs new file mode 100644 index 00000000..f782092a --- /dev/null +++ b/crates/engine/src/proxies_tests.rs @@ -0,0 +1,206 @@ +//! Proxies end to end: create (background job) + attach + toggle gives the same composite modulo +//! resolution (scale-aware effects and Motion), attach checks, export ignores proxies, toggling +//! doesn't dirty the project, ingest copies on import. + +use filmcraft_media::DemoScene; +use filmcraft_project::{ParamValue, find_effect}; +use serde_json::json; + +use crate::Session; +use crate::media_test_util::{frame_rgba, make_movie, psnr, session_with, tmp_dir}; + +const W: u32 = 320; +const H: u32 = 180; + +/// Give V1's clip a Motion move and a blur, so proxies must render effects scale-aware. +fn add_effects(s: &mut Session) { + let seq = s.state.active_sequence.unwrap(); + let mut p = (*s.project).clone(); + let q = p.sequence_mut(seq).unwrap(); + let it = &mut q.video_tracks[0].items[0]; + let m = it.effect_mut("motion").unwrap(); + m.params.get_mut("scale").unwrap().value = ParamValue::Float(70.0); + m.params.get_mut("rotation").unwrap().value = ParamValue::Float(8.0); + let mut blur = find_effect("gaussian_blur").unwrap().instance(); + if let Some(b) = blur.params.get_mut("blurriness") { + b.value = ParamValue::Float(6.0); + } + it.effects.push(blur); + s.project = std::sync::Arc::new(p); +} + +#[test] +fn create_attach_toggle_renders_the_same_picture() { + let root = tmp_dir("proxy-create"); + let a = root.join("a.mov"); + make_movie(&a, DemoScene::OceanSunset, W, H, 12); + let (mut s, items, _) = session_with(&[&a]); + add_effects(&mut s); + let full = frame_rgba(&mut s, 5, 1.0); + let full_half = frame_rgba(&mut s, 5, 0.5); + let r = s.execute("media.createProxies", json!({"items": [items[0].0], "preset": "prores_proxy_half", "wait": true})).unwrap(); + let out = r["outputs"][0]["path"].as_str().unwrap().to_string(); + assert!(out.ends_with("Proxies/a_Proxy.mov"), "{out}"); + let job = s.execute("jobs.list", json!({})).unwrap(); + assert_eq!(job[0]["finished"], json!(true), "{job}"); + let m = s.project.item(items[0]).unwrap().as_media().unwrap().clone(); + assert_eq!(m.proxy, Some(filmcraft_project::MediaRef::File { path: out.clone() }), "attached when the job finished"); + // proxies off: unchanged + assert_eq!(frame_rgba(&mut s, 5, 1.0), full); + let rev = s.revision; + let dirty = s.is_dirty(); + s.execute("media.toggleProxies", json!({"enabled": true})).unwrap(); + assert!(s.revision > rev, "frame caches refresh"); + assert_eq!(s.is_dirty(), dirty, "toggling proxies is not an edit"); + let px = frame_rgba(&mut s, 5, 1.0); + assert_eq!((px.0, px.1), (full.0, full.1)); + assert_ne!(px.2, full.2, "the proxy is really used"); + let q = psnr(&px.2, &full.2); + let q_half = psnr(&frame_rgba(&mut s, 5, 0.5).2, &full_half.2); + eprintln!("proxy vs full: {q:.1} dB at full scale, {q_half:.1} dB at ½"); + assert!(q > 30.0, "proxy composite matches full res modulo resolution: {q:.1} dB"); + assert!(q_half > 33.0, "at ½ playback resolution the proxy is near-identical: {q_half:.1} dB"); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn attach_checks_duration_and_export_ignores_proxies() { + let root = tmp_dir("proxy-attach"); + let a = root.join("a.mov"); + make_movie(&a, DemoScene::Aurora, W, H, 12); + let short = root.join("short.mov"); + make_movie(&short, DemoScene::Aurora, W / 2, H / 2, 6); + // a "proxy" with other pictures but the right duration and rate + let other = root.join("other.mp4"); + make_movie(&other, DemoScene::Plasma, W / 4, H / 4, 12); + let (mut s, items, _) = session_with(&[&a]); + let e = s.execute("media.attachProxies", json!({"item": items[0].0, "path": short.to_string_lossy()})).unwrap_err().to_string(); + assert!(e.contains("long"), "{e}"); + s.execute("media.attachProxies", json!({"item": items[0].0, "path": other.to_string_lossy()})).unwrap(); + let full = frame_rgba(&mut s, 4, 1.0); + s.execute("media.toggleProxies", json!({"enabled": true})).unwrap(); + assert!(psnr(&frame_rgba(&mut s, 4, 1.0).2, &full.2) < 20.0, "monitors show the (different) proxy"); + let png = root.join("out.png").to_string_lossy().into_owned(); + s.execute("file.exportMedia", json!({"path": png, "format": "png", "wait": true})).unwrap(); + let jobs = s.execute("jobs.list", json!({})).unwrap(); + assert!(jobs.as_array().unwrap().iter().all(|j| j["result"].get("error").is_none()), "{jobs}"); + let f = image::open(root.join("out_00004.png")).unwrap().to_rgba8(); + assert_eq!(psnr(f.as_raw(), &full.2), f64::INFINITY, "export renders full-resolution media"); + s.execute("media.detachProxies", json!({"items": [items[0].0]})).unwrap(); + assert_eq!(frame_rgba(&mut s, 4, 1.0), full); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn ingest_copies_and_creates_proxies_on_import() { + let root = tmp_dir("proxy-ingest"); + let cards = root.join("Card"); + std::fs::create_dir_all(&cards).unwrap(); + let a = cards.join("A001.mov"); + make_movie(&a, DemoScene::Forest, W, H, 8); + let dest = root.join("Ingest"); + let mut s = Session::default(); + s.execute( + "project.ingestSettings", + json!({"enabled": true, "action": "copyAndCreateProxies", "destination": dest.to_string_lossy(), "preset": "h264_quarter"}), + ) + .unwrap(); + let r = s.execute("file.import", json!({"paths": [a.to_string_lossy()]})).unwrap(); + let item = filmcraft_project::ItemId(r["items"][0].as_u64().unwrap()); + let job = r["ingest"]["job"]["job"].as_u64().unwrap_or_else(|| panic!("{r}")); + // wait for the background proxy job, then let the session apply it + for _ in 0..600 { + if s.jobs.iter().any(|j| j.id == job && j.progress.finished.load(std::sync::atomic::Ordering::Relaxed)) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + s.poll_persistence(); + let m = s.project.item(item).unwrap().as_media().unwrap(); + let copy = dest.join("A001.mov").to_string_lossy().into_owned(); + assert_eq!(m.media, filmcraft_project::MediaRef::File { path: copy.clone() }, "the clip uses the verified copy"); + assert_eq!(m.identity, Some(crate::relink::identity_of(&crate::FsServices, &a.to_string_lossy()).unwrap())); + assert_eq!(m.proxy, Some(filmcraft_project::MediaRef::File { path: dest.join("A001_Proxy.mp4").to_string_lossy().into_owned() })); + let _ = std::fs::remove_dir_all(&root); +} + +/// Measure: Program-monitor playback cost of 4K H.264 with and without proxies (sequential frames +/// at ½ resolution). Run with `RAYON_NUM_THREADS=1` so wall time ≈ CPU time: +/// `cargo test --release -p filmcraft-engine proxy_playback_perf_4k -- --ignored --nocapture`. +#[test] +#[ignore = "benchmark: 4K fixture via ffmpeg"] +fn proxy_playback_perf_4k() { + let Some(ffmpeg) = filmcraft_testkit::oracle::ffmpeg_or_skip("proxy_playback_perf_4k") else { return }; + let out = filmcraft_testkit::fixtures_dir("proxies").join("uhd_h264.mp4"); + let src = filmcraft_testkit::fixtures::generate(&out, |tmp| { + std::process::Command::new(&ffmpeg) + .args([ + "-y", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=3840x2160:rate=24:duration=2", + "-c:v", + "libx264", + "-preset", + "fast", + "-pix_fmt", + "yuv420p", + "-g", + "48", + ]) + .arg(tmp) + .status() + .is_ok_and(|s| s.success()) + }) + .expect("fixture"); + let dir = tmp_dir("proxy-perf"); + let media = dir.join("uhd.mp4"); + std::fs::copy(&src, &media).unwrap(); + let (mut s, items, _) = session_with(&[&media]); + let n = 24; + // (decode ms/frame as the GPU playback path pays it, CPU composite ms/frame at ½ and ¼) + let measure = |s: &mut Session| { + let rate = s.sequence_rate(); + let src = s.source(items[0]).unwrap(); + let _ = src.video_frame(filmcraft_media::FrameRequest { time: rate.tick_of(0), scale: 0.5 }); + let t0 = std::time::Instant::now(); + for f in 1..n { + let _ = src.video_frame(filmcraft_media::FrameRequest { time: rate.tick_of(f), scale: 0.5 }).unwrap(); + } + let decode = t0.elapsed().as_secs_f64() * 1000.0 / (n - 1) as f64; + let mut comp = [0.0; 2]; + for (k, scale) in [0.5f32, 0.25].into_iter().enumerate() { + s.media.clear(); + frame_rgba(s, 0, scale); + let t0 = std::time::Instant::now(); + for f in 1..n { + frame_rgba(s, f, scale); + } + comp[k] = t0.elapsed().as_secs_f64() * 1000.0 / (n - 1) as f64; + } + (decode, comp[0], comp[1]) + }; + let full = measure(&mut s); + eprintln!("4K H.264, full res: decode {:.1} ms/frame; CPU composite {:.1} ms/frame at 1/2, {:.1} at 1/4", full.0, full.1, full.2); + for preset in ["prores_proxy_quarter", "h264_quarter"] { + let t0 = std::time::Instant::now(); + s.execute("media.createProxies", json!({"items": [items[0].0], "preset": preset, "wait": true})).unwrap(); + let make = t0.elapsed().as_secs_f64(); + s.execute("media.toggleProxies", json!({"enabled": true})).unwrap(); + let px = measure(&mut s); + s.execute("media.toggleProxies", json!({"enabled": false})).unwrap(); + eprintln!( + "{preset} proxy (made in {make:.1}s): decode {:.1} ms/frame ({:.1}x less); CPU composite {:.1} ms/frame at 1/2 ({:.1}x less), {:.1} at 1/4 ({:.1}x less)", + px.0, + full.0 / px.0, + px.1, + full.1 / px.1, + px.2, + full.2 / px.2 + ); + } + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index c3a926eb..4cece607 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -101,6 +101,9 @@ pub struct ExportSettings { /// `progress.total`, `finished` and the final status; this call only adds to `done`. #[serde(default)] pub part_of_batch: bool, + /// ProRes flavour: `proxy`, `lt`, `standard` or `hq` (empty = HQ). + #[serde(default)] + pub prores_profile: String, } impl Default for ExportSettings { @@ -115,6 +118,7 @@ impl Default for ExportSettings { bitrate_kbps: 20_000, burn_captions: false, part_of_batch: false, + prores_profile: String::new(), } } } @@ -332,8 +336,19 @@ pub fn rgba_to_yuv422_10(rgba: &[u8], w: usize, h: usize, y: &mut [u16], cb: &mu }); } -fn prores_factory(format: Format, w: u32, h: u32, rate: FrameRate, _s: &ExportSettings) -> Option>> { - let profile = filmcraft_prores::Profile::Hq; +/// The ProRes profile named by [`ExportSettings::prores_profile`]. +pub fn prores_profile(name: &str) -> filmcraft_prores::Profile { + use filmcraft_prores::Profile; + match name.to_ascii_lowercase().as_str() { + "proxy" => Profile::Proxy, + "lt" => Profile::Lt, + "standard" | "422" => Profile::Standard, + _ => Profile::Hq, + } +} + +fn prores_factory(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Option>> { + let profile = prores_profile(&s.prores_profile); (format == Format::ProRes) .then(|| Ok(Box::new(ProResEncoder { enc: filmcraft_prores::Encoder::new(profile, w, h), profile, w, h, rate }) as Box)) } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 3c0c678b..43b540d8 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -784,6 +784,36 @@ pub struct ProjectSettings { pub default_still_duration: Tick, pub default_transition_duration_frames: i64, pub default_audio_transition_duration: Tick, + /// Project Settings ▸ Ingest Settings: what happens to media on import. + #[serde(default)] + pub ingest: IngestSettings, +} + +/// Ingest on import (Project Settings ▸ Ingest Settings). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct IngestSettings { + pub enabled: bool, + pub action: IngestAction, + /// Destination folder for copies, transcodes and proxies (None = next to the media: + /// `/Proxies` for proxies, `/Ingest` for copies and transcodes). + pub destination: Option, + /// Proxy / transcode preset id (the engine's `proxies::PRESETS`); empty = the default. + pub preset: String, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum IngestAction { + /// Copy the files to the destination (verified), then use the copies. + Copy, + /// Transcode to the preset's format, then use the transcodes. + Transcode, + /// Create proxies (in the background) and attach them. + #[default] + CreateProxies, + /// Copy, then create proxies of the copies. + CopyAndCreateProxies, } impl Default for ProjectSettings { @@ -795,6 +825,7 @@ impl Default for ProjectSettings { default_still_duration: Tick(5 * TICKS_PER_SECOND), default_transition_duration_frames: 24, default_audio_transition_duration: Tick(TICKS_PER_SECOND), + ingest: IngestSettings::default(), } } } diff --git a/crates/ui-egui/src/icons.rs b/crates/ui-egui/src/icons.rs index fdb8cc7d..251667a3 100644 --- a/crates/ui-egui/src/icons.rs +++ b/crates/ui-egui/src/icons.rs @@ -83,6 +83,8 @@ pub enum Icon { Sparkle, Grid, Square, + /// Toggle Proxies: a small frame inside a large one. + Proxy, /// Offline media: a broken link. Offline, } @@ -492,6 +494,12 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { pen.line(&[(2.0, 10.0), (14.0, 10.0)]); } Square => pen.rect(3.0, 3.0, 13.0, 13.0), + Proxy => { + pen.rect(1.5, 3.0, 14.5, 13.0); + pen.rect_fill(3.5, 8.0, 8.5, 11.5); + pen.line(&[(9.5, 7.0), (12.5, 4.5)]); + pen.line(&[(10.5, 4.5), (12.5, 4.5), (12.5, 6.5)]); + } Offline => { pen.line(&[(7.0, 4.0), (4.5, 4.0), (2.5, 6.0), (2.5, 7.5), (4.0, 9.0)]); pen.line(&[(9.0, 12.0), (11.5, 12.0), (13.5, 10.0), (13.5, 8.5), (12.0, 7.0)]); diff --git a/crates/ui-egui/src/panels/media_dialogs.rs b/crates/ui-egui/src/panels/media_dialogs.rs index c9f92956..86647d19 100644 --- a/crates/ui-egui/src/panels/media_dialogs.rs +++ b/crates/ui-egui/src/panels/media_dialogs.rs @@ -1,4 +1,4 @@ -//! Media management dialogs: Link Media (offline media / relink) and Make Offline. +//! Media management dialogs: Link Media (offline media / relink), Make Offline and Create Proxies. //! Their state lives in `UiState` (serde), so agents can open and fill them as well as click them; //! the actions run engine commands (`media.*`). //! @@ -8,12 +8,13 @@ //! `linkMedia.exactName`, `linkMedia.search`, `linkMedia.candidate.`, `linkMedia.preview`, //! `linkMedia.link`, `linkMedia.locate`, `linkMedia.offline`, `linkMedia.offlineAll`, `linkMedia.cancel`. //! - Make Offline: `makeOffline.keep`, `makeOffline.delete`, `makeOffline.ok`, `makeOffline.cancel`. +//! - Create Proxies: `proxies.preset.`, `proxies.destination`, `proxies.browse`, `proxies.ok`, `proxies.cancel`. use egui::{Color32, RichText}; use serde_json::{Value, json}; use crate::FilmcraftApp; -use crate::state::LinkMediaDraft; +use crate::state::{LinkMediaDraft, ProxyDraft}; type Elems = Vec<(String, egui::Rect, String)>; @@ -32,6 +33,9 @@ pub fn show(app: &mut FilmcraftApp, ctx: &egui::Context) { if app.ui.make_offline.is_some() { make_offline(app, ctx); } + if app.ui.create_proxies.is_some() { + create_proxies(app, ctx); + } } /// Menu / shortcut entry points without params open a dialog. Returns None for other ids. @@ -57,6 +61,19 @@ pub fn route(app: &mut FilmcraftApp, id: &str, params: &Value) -> Option Some(enabled(app, id).map(|_| { + let items = app.session.state.project_selection.iter().map(|i| i.0).collect(); + app.ui.create_proxies = Some(ProxyDraft { items, preset: filmcraft_engine::proxies::DEFAULT_PROXY_PRESET.into(), destination: String::new() }); + json!({"dialog": "createProxies"}) + })), + "media.attachProxies" | "media.reconnectFullRes" => { + let item = app.session.state.project_selection.first().copied(); + let Some(item) = item else { return Some(Err("select a clip in the Project panel".into())) }; + let exts: Vec<&str> = filmcraft_media::VIDEO_EXTENSIONS.to_vec(); + let path = app.hooks.pick_files.as_mut().and_then(|f| f(&exts).into_iter().next()); + let Some(path) = path else { return Some(Ok(Value::Null)) }; + Some(app.session.execute(id, json!({"item": item.0, "path": path})).map_err(|e| e.to_string())) + } _ => None, } } @@ -348,3 +365,56 @@ fn make_offline(app: &mut FilmcraftApp, ctx: &egui::Context) { } app.ui.make_offline = (keep && !ctx.input(|i| i.key_pressed(egui::Key::Escape))).then_some(delete); } + +fn create_proxies(app: &mut FilmcraftApp, ctx: &egui::Context) { + let Some(mut d) = app.ui.create_proxies.clone() else { return }; + let mut elems: Elems = Vec::new(); + let mut keep = true; + let mut ok = false; + let mut browse = false; + egui::Window::new("Create Proxies").collapsible(false).resizable(false).anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]).show(ctx, |ui| { + ui.label(format!("{} clip(s) selected.", d.items.len())); + ui.add_space(4.0); + ui.label(RichText::new("Format and size").strong()); + for p in filmcraft_engine::proxies::PRESETS.iter().filter(|p| p.proxy) { + let r = ui.radio(d.preset == p.id, p.label); + push(&mut elems, format!("proxies.preset.{}", p.id), &r, p.label); + if r.clicked() { + d.preset = p.id.into(); + } + } + ui.add_space(4.0); + ui.label(RichText::new("Destination").strong()); + ui.horizontal(|ui| { + let r = ui.add(egui::TextEdit::singleline(&mut d.destination).desired_width(320.0).hint_text("Next to the original media, in a Proxies folder")); + push(&mut elems, "proxies.destination", &r, "destination"); + let r = ui.button("Browse…"); + push(&mut elems, "proxies.browse", &r, "Browse…"); + browse = r.clicked(); + }); + ui.label(RichText::new("Proxies are made in the background and attached when done. Export always uses full-resolution media.").weak()); + ui.horizontal(|ui| { + let r = ui.button("Cancel"); + push(&mut elems, "proxies.cancel", &r, "Cancel"); + keep &= !r.clicked(); + let r = ui.add(egui::Button::new(RichText::new("OK").color(Color32::WHITE)).fill(app.tokens.accent)); + push(&mut elems, "proxies.ok", &r, "OK"); + ok = r.clicked(); + }); + }); + for (id, r, l) in elems { + app.auto.add(&id, r, &l); + } + if browse && let Some(f) = app.hooks.pick_folder.as_mut().and_then(|f| f()) { + d.destination = f; + } + if ok { + let p = json!({"items": d.items, "preset": d.preset, "destination": if d.destination.is_empty() { Value::Null } else { json!(d.destination) }}); + match app.session.execute("media.createProxies", p) { + Ok(v) => app.ui.status = format!("Creating {} proxy file(s)…", v["outputs"].as_array().map_or(0, Vec::len)), + Err(e) => app.ui.status = e.to_string(), + } + keep = false; + } + app.ui.create_proxies = (keep && !ctx.input(|i| i.key_pressed(egui::Key::Escape))).then_some(d); +} diff --git a/crates/ui-egui/src/panels/monitor.rs b/crates/ui-egui/src/panels/monitor.rs index 362b6f2f..c9b1e1e7 100644 --- a/crates/ui-egui/src/panels/monitor.rs +++ b/crates/ui-egui/src/panels/monitor.rs @@ -322,6 +322,7 @@ fn transport(app: &mut FilmcraftApp, ui: &mut egui::Ui, row: Rect, which: Which) (Icon::Insert, "source.insert", "Insert (,)"), (Icon::Overwrite, "source.overwrite", "Overwrite (.)"), (Icon::Camera, "exportFrame", "Export Frame (Shift+E)"), + (Icon::Proxy, "media.toggleProxies", "Toggle Proxies"), ] } else { vec![ @@ -336,6 +337,7 @@ fn transport(app: &mut FilmcraftApp, ui: &mut egui::Ui, row: Rect, which: Which) (Icon::Lift, "sequence.lift", "Lift (;)"), (Icon::Extract, "sequence.extract", "Extract (')"), (Icon::Camera, "exportFrame", "Export Frame (Shift+E)"), + (Icon::Proxy, "media.toggleProxies", "Toggle Proxies"), ] }; let bw = 30.0; diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index 7c40c40a..5682cf99 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -367,6 +367,20 @@ fn item_interactions( ui.close(); } } + ui.menu_button("Proxy", |ui| { + for (label, cmd) in [ + ("Create Proxies…", "media.createProxies"), + ("Attach Proxies…", "media.attachProxies"), + ("Reconnect Full Resolution Media…", "media.reconnectFullRes"), + ("Detach Proxies", "media.detachProxies"), + ] { + if ui.button(label).clicked() { + actions.push(("project.select".into(), json!({"items": [id.0]}))); + actions.push((cmd.into(), json!({}))); + ui.close(); + } + } + }); ui.separator(); } if ui.button("Clear").clicked() { diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 56d157c8..33bf40b5 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -295,6 +295,9 @@ pub struct UiState { /// Link Media dialog (open when Some). #[serde(default)] pub link_media: Option, + /// Create Proxies dialog (open when Some). + #[serde(default)] + pub create_proxies: Option, /// Make Offline dialog: Some(delete files?) while open. #[serde(default)] pub make_offline: Option, @@ -347,6 +350,16 @@ impl Default for LinkMediaDraft { } } +/// Clip ▸ Proxy ▸ Create Proxies…. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ProxyDraft { + pub items: Vec, + pub preset: String, + /// Empty = a Proxies folder next to the original media. + pub destination: String, +} + /// The Audio Gain dialog (Clip ▸ Audio Gain…, G). #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(default)] @@ -410,6 +423,7 @@ impl Default for UiState { gfx_edit: None, pen_points: vec![], link_media: None, + create_proxies: None, make_offline: None, } } diff --git a/crates/ui-egui/tests/media_ui.rs b/crates/ui-egui/tests/media_ui.rs index 14b1eea2..60e42169 100644 --- a/crates/ui-egui/tests/media_ui.rs +++ b/crates/ui-egui/tests/media_ui.rs @@ -1,5 +1,6 @@ //! Headless UI tests of media management: the Link Media dialog opening for a project whose media -//! moved (search a folder, link, the others follow) and the Project panel's offline markers. +//! moved (search a folder, link, the others follow), Project panel offline / proxy badges, the +//! monitors' Toggle Proxies button and the Create Proxies dialog. //! //! Set `FILMCRAFT_UI_SNAPSHOT_DIR=` to also render the window offscreen with wgpu and write PNGs //! there (`media-*.png`). @@ -206,7 +207,7 @@ fn link_media_dialog_relinks_moved_media() { } #[test] -fn offline_markers_in_the_project_panel() { +fn offline_badges_proxy_toggle_and_dialogs() { let root = tmp_dir("badges"); let path = moved_project(&root); let mut s = Session::default(); @@ -221,9 +222,29 @@ fn offline_markers_in_the_project_panel() { let offline_badges = d.ids("project.item.").into_iter().filter(|i| i.ends_with(".offline")).count(); assert_eq!(offline_badges, 2, "icon-view offline markers"); d.snapshot("media-offline-project"); - // relinking clears the markers + // relink by folder remap, create proxies through the dialog, toggle them from the monitor d.exec("media.autoRelink", json!({"from": root.join("Media").to_string_lossy(), "to": root.join("Moved/Media").to_string_lossy()})); - d.frames(10); - assert_eq!(d.ids("project.item.").into_iter().filter(|i| i.ends_with(".offline")).count(), 0); + let items: Vec = d.exec("media.status", json!({})).as_array().unwrap().iter().map(|m| m["item"].as_u64().unwrap()).collect(); + d.exec("project.select", json!({"items": items})); + d.ok("ui.menu.invoke", json!({"id": "media.createProxies"})); + d.frames(3); + assert!(d.app().ui.create_proxies.is_some()); + d.snapshot("media-create-proxies"); + d.click("proxies.preset.prores_proxy_quarter"); + d.click("proxies.ok"); + for _ in 0..400 { + d.frames(1); + let attached = d.app().session.project.items.values().filter(|i| i.as_media().is_some_and(|m| m.proxy.is_some())).count(); + if attached == 2 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert_eq!(d.app().session.project.items.values().filter(|i| i.as_media().is_some_and(|m| m.proxy.is_some())).count(), 2, "proxies attached"); + d.click("program.transport.media.toggleProxies"); + assert!(d.app().session.prefs.media.enable_proxies); + assert_eq!(d.ids("project.item.").into_iter().filter(|i| i.ends_with(".proxy")).count(), 2, "proxy badges"); + d.frames(20); + d.snapshot("media-proxies-on"); let _ = std::fs::remove_dir_all(&root); } diff --git a/docs/architecture.md b/docs/architecture.md index 84bfd35a..35bba5f0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -203,7 +203,7 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see flag), so relinking and undo take effect at once. Media that can't be opened renders the offline slate (`render::offline`) instead of failing. With proxies enabled, an item with a proxy reads it through `ProxySource`, which reports the original's size. Export always uses - `MediaPool::full_res_provider`. See [project-files.md](project-files.md#media-offline-and-relinking). + `MediaPool::full_res_provider`. See [project-files.md](project-files.md#media-offline-relinking-proxies-ingest). - **Seeking.** `codecs::Mp4Source` seeks to the preceding sync sample and decodes forward, caching every frame of the GOP. Sequential playback reuses the decoder. Decoders implement `codecs::VideoDecoder`. `register_video_decoder` puts a factory in front of the built-in ones, so a diff --git a/docs/control-protocol.md b/docs/control-protocol.md index b45cecba..22349d22 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -46,9 +46,11 @@ record|soloSafe|output|input|fx.|send.|meter|name>` (popup entries below t `timeline.track.A1.lane[.kf.]`, `audioGain.`. Project files, auto-save, crash recovery and preferences commands (`file.recover`, `prefs.set`, …) and their -automation ids are listed in [project-files.md](project-files.md). That file also lists the offline -media and relinking commands and dialog ids (`media.findMissing`, -`media.relink`, `media.autoRelink`, `media.search`, `media.makeOffline`, `media.status`, `linkMedia.*`). +automation ids are listed in [project-files.md](project-files.md). That file also lists the media +management commands and dialog ids: offline media and relinking (`media.findMissing`, +`media.relink`, `media.autoRelink`, `media.search`, `media.makeOffline`, `media.status`, `linkMedia.*`), +proxies (`media.createProxies`, `media.attachProxies`, `media.toggleProxies`, `proxies.*`) and ingest +(`project.ingestSettings`). **Trim mode** (`crates/engine/src/trim.rs`): `trim.selectEditPoint` / `trim.selectNearest` enter trim mode (the Program monitor becomes the Trim Monitor, ids `trimMonitor.*`); `trim.monitor` returns what diff --git a/docs/project-files.md b/docs/project-files.md index 41938f7b..8230ace7 100644 --- a/docs/project-files.md +++ b/docs/project-files.md @@ -170,13 +170,13 @@ UI automation ids: `prefs.autoSave.enabled`, `prefs.autoSave.intervalMinutes`, `recovery.recover`, `recovery.later`, `recovery.discard`, `recovery.item.`; `revert.yes`, `revert.no` (File ▸ Revert asks first when invoked without params). -## Media: offline and relinking +## Media: offline, relinking, proxies, ingest Media is referenced by path. Each imported file also records its **identity** (`MediaClip::identity`): the size and a 64-bit fingerprint of the size, the first MiB and the last MiB. Reading 2 MiB is cheap on any file, and it tells a moved original from a different take with the same name. The field is optional (`skip_serializing_if`), and older builds ignore it, so it -needed no schema bump. Projects from older builds +needed no schema bump. The same applies to `ProjectSettings::ingest`. Projects from older builds have no identity, so relinking them skips the fingerprint check. ### Offline media @@ -227,5 +227,68 @@ Link Media dialog automation ids: `linkMedia.row.`, `linkMedia.search`, `linkMedia.candidate.`, `linkMedia.preview`, `linkMedia.link`, `linkMedia.locate`, `linkMedia.offline`, `linkMedia.offlineAll`, `linkMedia.cancel`. Make Offline: `makeOffline.keep|delete|ok|cancel`. In the Project panel, offline items get a -broken-link badge (`project.item..offline` in Icon view). +broken-link badge (`project.item..offline` in Icon view), and items with a proxy get a **P** +badge (`project.item..proxy`). + +### Proxies + +| Command | Menu | | +|---|---|---| +| `media.createProxies {items?, preset?, destination?, attach?=true, wait?}` | Clip ▸ Proxy ▸ Create Proxies… | background job; attaches when done | +| `media.attachProxies {item, path}` / `{items, paths}` | Attach Proxies… | duration within one frame and the same frame rate are required; size and aspect may differ | +| `media.detachProxies {items?}` | Detach Proxies | | +| `media.reconnectFullRes {item, path}` | Reconnect Full Resolution Media… | link the full-resolution file of a clip that has a proxy | +| `media.toggleProxies {enabled?}` | View ▸ Toggle Proxies, monitor button | Preferences ▸ Media ▸ `media.enableProxies` | +| `media.proxyPresets` | | | + +The presets use our own encoders: + +- `prores_proxy_quarter` (default), `prores_proxy_half`, `prores_lt_half`: ProRes 422 Proxy or LT + in MOV with PCM audio; +- `h264_quarter`, `h264_half`: H.264 + AAC in MP4. + +Proxies go to `/Proxies/_Proxy.`, or to the chosen folder. Job progress is +in `jobs.list`. `Session::poll_persistence`, which the app calls every frame, attaches finished +proxies. + +With proxies enabled, monitors, thumbnails and playback read the proxy. The proxy source reports +the original's size. The compositor derives its pixel scale from the frame it gets, so Motion and +pixel-size effect parameters (blur radii…) give the same picture at the proxy's resolution. A +320×180 clip with Motion and Gaussian Blur and a ½-size ProRes Proxy measures 44 dB PSNR against +full resolution (48 dB at ½ playback resolution). **Export always uses full resolution** +(`MediaPool::full_res_provider`). Toggling proxies bumps the revision so caches refresh, but it is +not an edit and doesn't mark the project as modified. + +**Cost.** These numbers are from `proxy_playback_perf_4k`, an ignored test: `RAYON_NUM_THREADS=1 +cargo test --release -p filmcraft-engine proxy_playback_perf_4k -- --ignored --nocapture`. It plays a +2 s 3840×2160 H.264 clip (ffmpeg `testsrc2`, libx264) on one core of the development Mac and reports +milliseconds per frame: + +| | decode (GPU path) | CPU composite at ½ | CPU composite at ¼ | proxy creation | +|---|---|---|---|---| +| full resolution | 9.2 | 277 | 201 | | +| ProRes Proxy ¼ | 8.9 | 150 | 35 | 17.8 s | +| H.264 ¼ | 0.7 | 131 | 48 | 8.9 s | + +The test pattern is unusually cheap for H.264 to decode, so real camera footage gains more from +proxies. The CPU composite (YUV→linear conversion and resampling) is what proxies cut most: +5.7× at ¼ resolution, where the proxy needs no resampling. + +Create Proxies dialog ids: `proxies.preset.`, `proxies.destination`, `proxies.browse`, +`proxies.ok`, `proxies.cancel`. Monitor button: `program.transport.media.toggleProxies` (and +`source.…`). It is lit while proxies are on. + +### Ingest settings + +`project.ingestSettings {enabled, action: copy|transcode|createProxies|copyAndCreateProxies, +destination?, preset?}` (Project Settings ▸ Ingest) acts on every `file.import`: + +- **copy** copies the file to the destination (default `/Ingested Media`), checks the + copy's fingerprint against the original, and then uses the copy; +- **transcode** writes the preset's format (default ProRes 422 LT) in the background and switches the + clip to the result when it is done; +- **createProxies** makes proxies in the background and attaches them; +- **copyAndCreateProxies** does both. + +`file.import` reports this under `ingest`.