diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index 650b1dc8..bf7c41d3 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -35,6 +35,8 @@ jobs: "$RUNNER_TEMP/actionlint" -color - name: shellcheck run: shellcheck packaging/*.sh packaging/*/*.sh + - name: Linux tarball installer + run: python3 packaging/linux/test_install.py - name: PowerShell shell: pwsh run: | diff --git a/AGENTS.md b/AGENTS.md index bea60585..68486045 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ panic path, and fix a crash before building on top of it. The cross-app standard code alone (a literal that always parses): a single-item `#[allow(clippy::expect_used)]` with the reason. 3. **No `unsafe`** (`unsafe_code = "forbid"` for the workspace). The single exception is `crates/platform`, which needs `unsafe` to call the operating system's media APIs (hardware - video decoding through VideoToolbox and Media Foundation / Direct3D 11; later encoding and Linux) and does nothing else. It + video decoding through VideoToolbox, Media Foundation / Direct3D 11 and VA-API; hardware encoding) and does nothing else. It uses `unsafe_code = "deny"` with `#[allow(unsafe_code)]` only on its FFI modules, a `// SAFETY:` comment on every `unsafe` block, a safe `Result`-returning public API, and the pure-Rust decoder as the tested fallback ([ADR 0001](docs/adr/0001-platform-ffi.md)). diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 4b28b953..83dfef26 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -26,6 +26,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `docs/images/filmcraft-effects.png` | FilmCraft contributors | Original work: FilmCraft screenshot crop; NASA Earth Views (US Government work, public domain) | MIT OR Apache-2.0; footage public domain | | `docs/images/filmcraft-timeline.png` | FilmCraft contributors | Original work: FilmCraft UI crop; thumbnails from Night of the Living Dead (1968) and Carnival of Souls (1962), CC0 Chopin waveform | MIT OR Apache-2.0; film frames public domain | | `assets/app-icon/filmcraft.svg` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; canonical vector | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) | +| `packaging/macos/dmg/background.svg` | @XusBadia | Original work: FilmCraft macOS DMG window background, derived from the FilmCraft app icon (`assets/app-icon/filmcraft.svg`); rendered to `background.tiff` by `packaging/macos/dmg/generate.py` | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`) | +| `packaging/macos/dmg/background.tiff` | @XusBadia | Rendered from `packaging/macos/dmg/background.svg` by `packaging/macos/dmg/generate.py` | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`) | | `assets/app-icon/filmcraft-small.svg` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; lighter vector | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) | | `assets/app-icon/filmcraft-1024.png` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; 1024 px render | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) | | `assets/app-icon/filmcraft-macos-512.png` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; macOS runtime Dock icon (512 px) | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) | diff --git a/Cargo.lock b/Cargo.lock index 7dda81f4..6a65be27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1844,6 +1844,7 @@ dependencies = [ name = "filmcraft-edit" version = "0.4.0" dependencies = [ + "filmcraft-geom", "filmcraft-media", "filmcraft-project", "filmcraft-time", @@ -2098,16 +2099,20 @@ dependencies = [ name = "filmcraft-platform" version = "0.4.0" dependencies = [ + "filmcraft-bitstream", "filmcraft-codecs", "filmcraft-color", "filmcraft-export", "filmcraft-frame", + "filmcraft-h264", + "filmcraft-hevc", "filmcraft-isobmff", "filmcraft-media", "filmcraft-project", "filmcraft-render", "filmcraft-testkit", "filmcraft-time", + "libloading", "log", "objc2-core-foundation", "objc2-core-media", @@ -2203,6 +2208,7 @@ dependencies = [ "skrifa", "unicode-bidi", "unicode-linebreak", + "unicode-segmentation", ] [[package]] diff --git a/README.md b/README.md index 0c91ed07..7ed0c17c 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ FilmCraft is a non-linear editor for people who know Premiere: the same panels, - **Premiere 26's full Video Effects bin (93 effects in 16 folders), the Legacy bin and the obsolete effects old projects use, plus around 30 transitions.** Blurs (Bokeh, Focus, Compound), keys (Ultra Key, Track Matte), distortions (Corner Pin, Turbulent Displace, Warp Stabilizer), Lights & Glows, Immersive Video (VR) effects on equirectangular footage, Posterize Time, Echo and more. Cross dissolve, dip to black or white, film dissolve, wipes, irises, pushes, slides, zooms, page peel, cube spin and more. - **Motion and opacity on every clip:** position, scale, rotation, anchor point and anti-flicker, plus 26 blend modes. -- **Keyframes like Premiere's:** linear, Bezier, auto and continuous Bezier, hold, ease in and ease out. Effect Controls shows a keyframe lane for every parameter, and each animated parameter opens into **value and velocity graphs** with draggable influence handles. +- **Keyframes like Premiere's:** linear, Bezier, auto and continuous Bezier, hold, ease in and ease out. Effect Controls shows a keyframe lane for every parameter under a time ruler with the playhead's handle, and each animated parameter opens into **value and velocity graphs** with draggable influence handles. Effect Controls and the Properties panel share the keyframe navigator (◀ ◆ ▶): add or remove the keyframe at the playhead, step to the previous or next one. - **A GPU compositor** built on wgpu (Metal, Vulkan, DirectX 12, WebGPU). It samples YUV straight from the decoder with footprint supersampling and blends in linear light. A CPU path renders the same frames, and the two are tested against each other.

@@ -185,7 +185,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus | **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata | | **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF | -Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: HEVC and AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox) and Windows (Media Foundation), with Linux to follow; H.264 export can use NVIDIA's encoder on Windows (opt-in) ([#30](https://github.com/storytold/filmcraft/issues/30)). +Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: HEVC and AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows (opt-in) ([#30](https://github.com/storytold/filmcraft/issues/30)).
@@ -287,7 +287,7 @@ We track two numbers ([ROADMAP.md](ROADMAP.md#honest-assessment-2026-10-05)): The biggest gaps today: -- **Speed on big footage.** Hardware decoding works on macOS and Windows; Linux has no hardware path yet. Blend modes and the most common effects run on the GPU, but Lumetri, keys and export rendering still run on the CPU; H.264 encoding can use the hardware encoder on macOS and on Windows with an NVIDIA GPU (opt-in, Export ▸ Hardware encoding), and H.265 export is hardware-only there ([#30](https://github.com/storytold/filmcraft/issues/30)). +- **Speed on big footage.** Hardware decoding works on macOS and Windows, and for H.264 and HEVC on Linux (VA-API; VP9 and AV1 decode in software there). Blend modes and the most common effects run on the GPU, but Lumetri, keys and export rendering still run on the CPU; H.264 encoding can use the hardware encoder on macOS and on Windows with an NVIDIA GPU (opt-in, Export ▸ Hardware encoding), and H.265 export is hardware-only there ([#30](https://github.com/storytold/filmcraft/issues/30)). - **No plugins.** No VST3 / Audio Units or OpenFX hosting. - **Delivery codecs.** H.264 is our only delivery-codec export; no HEVC or AV1 export yet. - **Real-world media and platforms.** Our decoders are bit-exact on conformance streams, but camera and phone files in the wild are less tested. Windows and Linux get far less testing than macOS. @@ -342,7 +342,11 @@ Installers and executables are code-signed. | Flatpak | `filmcraft--linux-x86_64.flatpak` | `filmcraft--linux-aarch64.flatpak` | Sandboxed; `flatpak install --user ` | | Debian/Ubuntu | `filmcraft--linux-x86_64.deb` | `filmcraft--linux-aarch64.deb` | | | Fedora/RHEL/openSUSE | `filmcraft--linux-x86_64.rpm` | `filmcraft--linux-aarch64.rpm` | | -| Tarball | `filmcraft--linux-x86_64.tar.gz` | `filmcraft--linux-aarch64.tar.gz` | Unpack anywhere | +| Tarball | `filmcraft--linux-x86_64.tar.gz` | `filmcraft--linux-aarch64.tar.gz` | Unpack and run `./install.sh` | + +The tarball installer places FilmCraft, its command-line tool and desktop integration in +`~/.local`, without administrator permissions. For all users, run +`sudo ./install.sh --prefix /usr/local` instead. Running the installer again updates the installation. ### FreeBSD diff --git a/ROADMAP.md b/ROADMAP.md index 29c0409f..2eefe6e0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,7 +36,7 @@ before choosing work. | **Correctness in depth** | not tracked | **~60–70%** | A first-time contributor found four real bugs in core paths within hours (#5–#8: AIFF import failed, Slide left linked audio behind, transitions didn't follow ripple trims, export ignored start/end times). User reports: Color Matte always grey with no picker (#29); ALSA audio error on Linux (#23). Settings are "most wired", not all. | | **Codecs and media** | ~90% | **~75%** | Our decoders are bit-exact on conformance streams. Real camera and phone media (variable frame rate, damaged files, unusual containers) is far less tested. No camera RAW (RED, BRAW, ARRIRAW), no E-AC-3. | | **Export** | ~92% | **~70%** | Delivery: H.264 only (ProRes, DNxHR and image sequences cover mastering). No HEVC or AV1 export. AAF and OMF have never been validated in Avid Media Composer or Pro Tools. Export renders and encodes on the CPU only. | -| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux has no hardware path. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | +| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30), and for H.264 and HEVC on Linux (VA-API: 4K H.264 at 11 ms of CPU per frame instead of 417 ms, 4K HEVC at 10 ms instead of 203 ms). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux decodes H.264 and HEVC in hardware (VP9 and AV1 there are software) and has no hardware encoder. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. | | **Stability** | not tracked | **improving, unproven** | The never-crash pass (no panics in product code, last-resort guards) landed on 2026-10-04. There's no field record from real users' projects yet. | | **Plugins and pro ecosystem** | not tracked | **~0–10%** | No audio plugin hosting (VST3 / Audio Units) and no third-party video effects (OpenFX). For many professional editors this alone rules FilmCraft out. Team Projects and Productions are out of scope by design. | | **AI features** | partial | **~25–35%** | Speech to text exists but is off in default builds (the `whisper` feature). Enhance Speech is a DSP chain, not a model. Auto Reframe is approximate. No Generative Extend, and no media-intelligence search or auto colour. | @@ -48,7 +48,7 @@ before choosing work. In priority order. Agents should prefer this work over adding more checklist items. 1. **Hardware acceleration** (#30), the most visible gap to users: - - hardware decode on Linux (VA-API); + - hardware decode on Linux (VA-API): H.264 and HEVC are in; VP9 and AV1 remain; - zero-copy decoded frames into wgpu; - the remaining effects on the GPU: Lumetri, keys, Vignette, Video Limiter, masks (31 common effects are done); - GPU export, then hardware encode: H.264 and H.265 on macOS are in (H.264 opt-in; H.265 has no other encoder), H.264 on NVENC (Windows, opt-in); HEVC on Windows and Linux remain. @@ -100,7 +100,7 @@ matched against `filmcraft-cli commands` and the UI command table. Effects: the | Graphics and captions | 8% | text engine, shapes, per-character styles, our own graphics templates (.fcgt, 8 built-ins, export/install/edit), rolls/crawls, responsive pins + time, captions SRT/VTT/SCC/MCC/STL/TTML/DFXP, transcripts | ~85% | | Export | 8% | own H.264/AAC, ProRes, DNxHR, MXF OP1a/OP-Atom, image sequences, GIF, WAV, 5.1; preset library + user presets, full Export-mode settings, queue, Quick Export; AAF (Edit Protocol) import/export, OMF 2.0 export; no AAF/OMF validation against Avid/Pro Tools yet | ~92% | | Preferences and project management | 5% | Settings dialog with 16 categories (most settings wired), project settings, scratch disks, search bins, templates | ~80% | -| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; blend modes on the GPU (#32); 8K and AV1 not yet real-time on a loaded machine; no hardware path on Linux, and on Windows only decode and opt-in NVENC H.264 export (see the honest assessment: ~35–40% against Premiere) | ~62% | +| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; blend modes on the GPU (#32); 8K and AV1 not yet real-time on a loaded machine; H.264 and HEVC decode in hardware on Linux (VA-API), and on Windows only decode and opt-in NVENC H.264 export (see the honest assessment: ~35–40% against Premiere) | ~62% | | **Weighted total** | | | **~87%** | ## Estimate to parity @@ -154,11 +154,13 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining ## Running now -- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Next: VA-API (Linux) decode, zero-copy upload, Lumetri on the GPU. +- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Linux: VA-API H.264 + HEVC decode (HW5). Next: VA-API VP9 / AV1, zero-copy upload, Lumetri on the GPU. - **Next:** [Where we are lacking](#where-we-are-lacking) items 1–5. GPU export waits for the export-crate work (#19 and the never-crash follow-up) to land. ## Log +- **2026-10-08 (#285):** `filmcraft-cli` no longer panics with "Broken pipe" when its stdout is closed early (`filmcraft-cli commands | head`): later output is dropped, the rest of the work (script lines, `--save`, `--save-as`) still runs, and the exit status still reports failures. Other stdout write errors are reported and make the exit status 1. No change to the checklist or estimates. + - **2026-10-06:** 21 community PRs landed (#19, #28, #38, #39, #41–#60). Highlights: hardware decoding is now actually on in the desktop app (#41: `register()` sat inside a `log::info!` that never ran, so #33's VideoToolbox path only worked in the CLI and benchmarks); decode memory on long timelines cut sharply (frame pool, shared GOP-cache budget, positional file reads); exports are byte-identical across machines (#19); crash fix for odd track names over MCP (#44); ripple trim, ripple delete and speed changes move split-edit clips once (#56–#58); Japanese interface using an installed system font (#38; no fonts bundled); MCP annotations, resources, export progress and cancellation (#28). - **2026-10-05:** honest assessment added (checklist ~87% vs ready for real work ~50–60%). Hardware acceleration started (#30): blend modes on the GPU (#32); VideoToolbox decode + `platform` crate, the one crate allowed `unsafe` (#33). Community fixes landed: AIFF import, export start/end ranges, transitions following ripple trims, Slide moving linked audio (#5–#8). diff --git a/apps/filmcraft-cli/src/main.rs b/apps/filmcraft-cli/src/main.rs index 13bebb85..03ff62f0 100644 --- a/apps/filmcraft-cli/src/main.rs +++ b/apps/filmcraft-cli/src/main.rs @@ -4,6 +4,32 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] +/// `println!` that doesn't panic when stdout is closed (`filmcraft-cli commands | head`) with +/// "failed printing to stdout: Broken pipe (os error 32)". Once stdout has failed, later output is +/// dropped and the work goes on: see `stdout_failed`. +macro_rules! outln { + ($($arg:tt)*) => {{ + use std::io::Write as _; + if !$crate::stdout_gone() { + if let Err(e) = writeln!(std::io::stdout(), $($arg)*) { + $crate::stdout_failed(e); + } + } + }}; +} + +/// `print!` counterpart of `outln!`. +macro_rules! out { + ($($arg:tt)*) => {{ + use std::io::Write as _; + if !$crate::stdout_gone() { + if let Err(e) = write!(std::io::stdout(), $($arg)*) { + $crate::stdout_failed(e); + } + } + }}; +} + mod args; mod probe; @@ -11,6 +37,28 @@ use args::{Args, parse_value}; use filmcraft_automation::BridgeClient; use filmcraft_engine::Session; use serde_json::{Value, json}; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Set once a write to stdout has failed; later output is dropped. +static STDOUT_GONE: AtomicBool = AtomicBool::new(false); +/// Set when stdout failed for a reason other than a closed reader: the exit status becomes 1. +static STDOUT_ERROR: AtomicBool = AtomicBool::new(false); + +fn stdout_gone() -> bool { + STDOUT_GONE.load(Ordering::Relaxed) +} + +/// stdout went away. The program doesn't stop there: the rest of `exec`/`run`/`import` (later +/// script lines, `--save`, `--save-as`) still runs and the exit status still tells how it went, so +/// `exec … --save | head -1` keeps the edit. A reader that stopped early (a closed pipe) is not an +/// error, as in ripgrep; any other write error is reported and makes the exit status 1. +fn stdout_failed(e: std::io::Error) { + if STDOUT_GONE.swap(true, Ordering::Relaxed) || e.kind() == std::io::ErrorKind::BrokenPipe { + return; + } + eprintln!("filmcraft-cli: can't write to stdout: {e}"); + STDOUT_ERROR.store(true, Ordering::Relaxed); +} const HELP: &str = "\ filmcraft-cli: drive FilmCraft from the shell (headless engine, or the live app with --bridge) @@ -134,7 +182,7 @@ impl Backend { fn print(a: &Args, v: &Value) { let s = if a.flag("--compact") { serde_json::to_string(v) } else { serde_json::to_string_pretty(v) }; - println!("{}", s.unwrap_or_default()); + outln!("{}", s.unwrap_or_default()); } /// Export format from the output file's extension. @@ -162,16 +210,23 @@ fn register_hardware_decoders() -> filmcraft_platform::Availability { #[tokio::main] async fn main() { + cli().await; + if STDOUT_ERROR.load(Ordering::Relaxed) { + std::process::exit(1); + } +} + +async fn cli() { if matches!(std::env::args().nth(1).as_deref(), Some("--version" | "-V")) { - println!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")); + outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")); return; } register_hardware_decoders(); let a = Args::parse(std::env::args().skip(1)); let Some(cmd) = a.pos(0) else { usage("missing subcommand") }; match cmd { - "help" | "--help" | "-h" => print!("{HELP}"), - "version" => println!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")), + "help" | "--help" | "-h" => out!("{HELP}"), + "version" => outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")), "probe" => { let path = a.pos(1).unwrap_or_else(|| usage("probe [--image-sequence]")); match probe::probe(path, a.flag("--image-sequence")) { @@ -189,11 +244,11 @@ async fn main() { src.video_frame(filmcraft_media::FrameRequest::full(rate.tick_of(f))).unwrap_or_else(|e| fail(e)); } let dt = t0.elapsed().as_secs_f64(); - println!("{n} frames in {dt:.2}s → {:.1} fps (sequential, via Mp4Source)", n as f64 / dt); + outln!("{n} frames in {dt:.2}s → {:.1} fps (sequential, via Mp4Source)", n as f64 / dt); let t1 = std::time::Instant::now(); let f = src.video_frame(filmcraft_media::FrameRequest::full(rate.tick_of(n / 2))).unwrap_or_else(|e| fail(e)); let (w, h, _) = f.to_linear_f32_decimated(2); - println!("½-res linear conversion {w}x{h}: {:.1} ms", t1.elapsed().as_secs_f64() * 1000.0); + outln!("½-res linear conversion {w}x{h}: {:.1} ms", t1.elapsed().as_secs_f64() * 1000.0); } "commands" | "describe" => { let mut b = Backend::open(&a); @@ -219,7 +274,7 @@ async fn main() { } else { for c in &list { let s = |k: &str| c[k].as_str().unwrap_or("").to_string(); - println!("{:<32} {:<34} {:<14} {}", s("id"), s("label"), s("shortcut"), s("params")); + outln!("{:<32} {:<34} {:<14} {}", s("id"), s("label"), s("shortcut"), s("params")); } } } @@ -246,9 +301,9 @@ async fn main() { let v: Value = serde_json::from_str(line).unwrap_or_else(|e| usage(format!("line {}: {e}", n + 1))); let id = v["id"].as_str().unwrap_or_else(|| usage(format!("line {}: missing \"id\"", n + 1))); match b.exec(id, v.get("params").cloned().unwrap_or(json!({}))).await { - Ok(r) => println!("{}", json!({"line": n + 1, "id": id, "ok": true, "result": r})), + Ok(r) => outln!("{}", json!({"line": n + 1, "id": id, "ok": true, "result": r})), Err(e) => { - println!("{}", json!({"line": n + 1, "id": id, "ok": false, "error": e})); + outln!("{}", json!({"line": n + 1, "id": id, "ok": false, "error": e})); failed += 1; if !a.flag("--keep-going") { std::process::exit(1); @@ -344,7 +399,7 @@ async fn main() { match r { Ok(v) => { print(&a, &v); - eprintln!("exported {out} in {:.1}s", t0.elapsed().as_secs_f64()); + eprintln!("exported {} in {:.1}s", written_paths(&v, out).join(", "), t0.elapsed().as_secs_f64()); } Err(e) => fail(format!("export: {e}")), } @@ -381,6 +436,24 @@ async fn main() { } } +/// The files an export wrote, from its result: `path` of a direct export, or each queued item's +/// `path` (the format may have changed the extension: `out.mp4` exported as APV is `out.mp4.mov`). +/// `requested` when the result names none. +fn written_paths(result: &Value, requested: &str) -> Vec { + let paths: Vec = match result.get("path").and_then(Value::as_str) { + Some(p) => vec![p.to_string()], + None => result + .get("items") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|i| i.get("path").and_then(Value::as_str)) + .map(str::to_string) + .collect(), + }; + if paths.is_empty() { vec![requested.to_string()] } else { paths } +} + #[cfg(test)] mod format_tests { /// The CLI (and MCP / headless runs, which share its entry point) registers the hardware @@ -388,7 +461,21 @@ mod format_tests { #[test] fn startup_registers_the_hardware_decoders() { let hardware = super::register_hardware_decoders(); - assert_eq!(filmcraft_platform::registered(), cfg!(any(target_os = "macos", target_os = "windows")), "{hardware:?}"); + // always on macOS and Windows; on Linux when a VA-API driver is there + let expected = cfg!(any(target_os = "macos", target_os = "windows")) + || (cfg!(target_os = "linux") && matches!(hardware, filmcraft_platform::Availability::Available(_))); + assert_eq!(filmcraft_platform::registered(), expected, "{hardware:?}"); + } + + #[test] + fn export_message_names_the_written_files() { + use serde_json::json; + let direct = json!({"job": 1, "path": "out-apv.mp4.mov", "result": {"path": "out-apv.mp4.mov"}}); + assert_eq!(super::written_paths(&direct, "out-apv.mp4"), ["out-apv.mp4.mov"]); + let queued = json!({"added": [1, 2], "items": [{"path": "a.mp4.mov"}, {"path": "b.wav"}]}); + assert_eq!(super::written_paths(&queued, "a.mp4"), ["a.mp4.mov", "b.wav"]); + assert_eq!(super::written_paths(&json!({}), "x.mov"), ["x.mov"]); + assert_eq!(super::written_paths(&json!({"items": []}), "x.mov"), ["x.mov"]); } #[test] diff --git a/apps/filmcraft-cli/tests/cli.rs b/apps/filmcraft-cli/tests/cli.rs index b7fedfc2..22653338 100644 --- a/apps/filmcraft-cli/tests/cli.rs +++ b/apps/filmcraft-cli/tests/cli.rs @@ -179,9 +179,89 @@ fn export_with_presets() { let v = json_out(&cli(&["--demo", "--data-dir", data_s, "export", q.to_str().unwrap(), "--preset", "Tiny WAV", "--queue", "--range", "entire"])); assert_eq!(v["items"][0]["status"], "done", "{v}"); assert!(q.exists()); + // a format that changes the extension: the completion message names the file written (#341) + let apv = dir.join("apv.mp4"); + let o = + cli(&["--demo", "--data-dir", data_s, "export", apv.to_str().unwrap(), "--preset", "APV 422-10", "--start", "0", "--end", "0.1", "--scale", "0.125"]); + let v = json_out(&o); + let written = v["path"].as_str().unwrap().to_string(); + assert!(written.ends_with("apv.mp4.mov") && std::path::Path::new(&written).exists(), "{v}"); + let msg = String::from_utf8_lossy(&o.stderr); + assert!(msg.contains(&format!("exported {written} in ")), "{msg}"); + let qa = dir.join("queued-apv.mp4"); + let o = cli(&[ + "--demo", + "--data-dir", + data_s, + "export", + qa.to_str().unwrap(), + "--preset", + "APV 422-10", + "--queue", + "--start", + "0", + "--end", + "0.1", + "--scale", + "0.125", + ]); + let v = json_out(&o); + let written = v["items"][0]["path"].as_str().unwrap().to_string(); + assert!(written.ends_with("queued-apv.mp4.mov"), "{v}"); + assert!(String::from_utf8_lossy(&o.stderr).contains(&format!("exported {written} in ")), "{}", String::from_utf8_lossy(&o.stderr)); // unknown preset: exit status 1 and a message naming it let bad = cli(&["--demo", "--data-dir", data_s, "export", dir.join("x.mp4").to_str().unwrap(), "--preset", "No Such Preset"]); assert_eq!(bad.status.code(), Some(1)); assert!(String::from_utf8_lossy(&bad.stderr).contains("No Such Preset")); let _ = std::fs::remove_dir_all(&dir); } + +/// `filmcraft-cli commands | head -c 10` panicked with "failed printing to stdout: Broken pipe +/// (os error 32)" and exit status 101 (#285). +#[test] +fn closed_stdout_ends_quietly() { + use std::io::Read; + for args in [&["commands"][..], &["help"], &["--demo", "commands", "--json"]] { + let mut child = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(args).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); + let mut first = [0u8; 10]; + child.stdout.as_mut().unwrap().read_exact(&mut first).unwrap(); + drop(child.stdout.take()); + let out = child.wait_with_output().unwrap(); + let err = String::from_utf8_lossy(&out.stderr); + assert!(out.status.success() && err.is_empty(), "{args:?}: {:?}: {err}", out.status); + } +} + +/// `run -` with stdout already closed: `run` reads its script to the end before it prints, so its +/// first write fails for sure. The rest of the script and the save still happen, and the exit status +/// still reports a failing command. +fn run_with_closed_stdout(args: &[&str], script: &[u8]) -> Output { + let mut child = + Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); + drop(child.stdout.take()); + child.stdin.take().unwrap().write_all(script).unwrap(); + child.wait_with_output().unwrap() +} + +#[test] +fn closed_stdout_still_runs_the_script_and_saves() { + let dir = std::env::temp_dir().join(format!("fc-cli-pipe-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let proj = dir.join("p.fcproj"); + let p = proj.to_str().unwrap(); + let script = b"{\"id\":\"file.newBin\",\"params\":{\"name\":\"Piped One\"}}\n{\"id\":\"file.newBin\",\"params\":{\"name\":\"Piped Two\"}}\n"; + let out = run_with_closed_stdout(&["--demo", "--save-as", p, "run", "-"], script); + assert!(out.status.success(), "{:?}: {}", out.status, String::from_utf8_lossy(&out.stderr)); + let tree = json_out(&cli(&["--project", p, "inspect", "project"])).to_string(); + assert!(tree.contains("Piped One") && tree.contains("Piped Two"), "{tree}"); + + // a failing command is still a failure, whether or not anyone reads stdout + let out = run_with_closed_stdout(&["--demo", "run", "-"], b"{\"id\":\"no.such.command\"}\n"); + assert_eq!(out.status.code(), Some(1)); + let out = run_with_closed_stdout( + &["--demo", "--keep-going", "run", "-"], + b"{\"id\":\"no.such.command\"}\n{\"id\":\"file.newBin\",\"params\":{\"name\":\"x\"}}\n", + ); + assert_eq!(out.status.code(), Some(1)); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/apps/filmcraft-web/src/fs.rs b/apps/filmcraft-web/src/fs.rs index b959bdcb..adf417ca 100644 --- a/apps/filmcraft-web/src/fs.rs +++ b/apps/filmcraft-web/src/fs.rs @@ -70,9 +70,8 @@ fn file_name(path: &str) -> &str { path.rsplit('/').next().unwrap_or(path) } -/// Register a browser `File` (or `Blob` with a name) and return its virtual path. A file with -/// the same name and size as an existing entry replaces it (the same file picked again keeps -/// its path, so projects referencing it come back online); otherwise a numbered name is used. +/// Register a browser `File` (or named `Blob`) at an unused virtual path; collisions get numbered names. +/// Name + size do not identify content: replacing an occupied path changes every project item referencing it. pub fn register_blob(name: &str, blob: web_sys::Blob) -> String { let size = blob.size() as u64; let clean: String = name.chars().map(|c| if c == '/' || c == '\\' { '_' } else { c }).collect(); @@ -85,18 +84,11 @@ pub fn register_blob(name: &str, blob: web_sys::Blob) -> String { let mut n = 1; let path = loop { let p = if n == 1 { format!("/files/{clean}") } else { format!("/files/{stem} ({n}){ext}") }; - match fs.entries.get(&p) { - None => break p, - Some(Entry::Blob { size: s, .. }) if *s == size => break p, - Some(Entry::Mem(b)) if b.len() as u64 == size => break p, - _ => n += 1, + if !fs.entries.contains_key(&p) { + break p; } + n += 1; }; - if let Some(Entry::Blob { id, .. }) = fs.entries.get(&path) { - let id = *id; - fs.blobs.remove(&id); - fs.chunks.retain(|k, _| k.0 != id); - } let id = fs.next_id; fs.next_id += 1; fs.blobs.insert(id, blob); diff --git a/apps/filmcraft-web/src/import.rs b/apps/filmcraft-web/src/import.rs index e837a371..6b41a2cd 100644 --- a/apps/filmcraft-web/src/import.rs +++ b/apps/filmcraft-web/src/import.rs @@ -81,12 +81,12 @@ async fn prewarm(path: &str, size: u64) { } /// `file.import` one path, retried while its bytes are loading. -async fn import_path(path: String) -> Result { +async fn import_path(path: String, bin: u64) -> Result { for _ in 0..2000 { let p = path.clone(); let (r, pending) = on_ui(move |w, _| { let _ = filmcraft_media::pending::take(); - let r = w.app.session.execute("file.import", json!({"paths": [p]})).map_err(|e| e.to_string()); + let r = w.app.session.execute("file.import", json!({"paths": [p], "bin": bin})).map_err(|e| e.to_string()); (r, filmcraft_media::pending::take()) }) .await; @@ -105,6 +105,8 @@ async fn import_path(path: String) -> Result { /// Import browser files (no copies): returns `{items, errors, paths}`. pub async fn import_files(files: Vec) -> Value { + // Capture once per batch: navigation while prewarming must not redirect pending imports. + let bin = on_ui(|w, _| w.app.import_bin().0).await; let mut items = Vec::new(); let mut errors = Vec::new(); let mut paths = Vec::new(); @@ -113,7 +115,7 @@ pub async fn import_files(files: Vec) -> Value { let size = f.size() as u64; let path = fs::register_blob(&name, f.clone().into()); prewarm(&path, size).await; - match import_path(path.clone()).await { + match import_path(path.clone(), bin).await { Ok(v) => { items.extend(v.get("items").and_then(Value::as_array).cloned().unwrap_or_default()); errors.extend(v.get("errors").and_then(Value::as_array).cloned().unwrap_or_default()); diff --git a/apps/filmcraft-web/src/lib.rs b/apps/filmcraft-web/src/lib.rs index 469d2107..80d6eb00 100644 --- a/apps/filmcraft-web/src/lib.rs +++ b/apps/filmcraft-web/src/lib.rs @@ -24,6 +24,7 @@ pub mod fs; pub mod import; pub mod opfs; pub mod recovery; +pub mod recovery_policy; pub mod webcodecs; use std::cell::RefCell; @@ -58,6 +59,13 @@ pub fn repaint() { }); } +/// Give a host that builds its own [`WebApp`] the egui context that [`post`] and [`repaint`] wake. +/// [`start`] calls it; an embedding app that runs `WebApp` in its own eframe runner calls it once +/// from its app creator, or posted actions and JS API requests wait for the next input event. +pub fn set_context(ctx: egui::Context) { + CTX.with(|c| *c.borrow_mut() = Some(ctx)); +} + /// Environment facts reported by `filmcraft.info()` (backend, isolation, codecs…). pub fn set_info(key: &str, v: serde_json::Value) { INFO.with(|i| { @@ -150,7 +158,7 @@ pub async fn start(canvas_id: String) -> Result<(), JsValue> { canvas, web_options, Box::new(move |cc| { - CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone())); + set_context(cc.egui_ctx.clone()); let ctx = cc.egui_ctx.clone(); fs::set_on_data(move || ctx.request_repaint()); fs::set_on_write(|path, data| { diff --git a/apps/filmcraft-web/src/opfs.rs b/apps/filmcraft-web/src/opfs.rs index b1907108..a246a52d 100644 --- a/apps/filmcraft-web/src/opfs.rs +++ b/apps/filmcraft-web/src/opfs.rs @@ -10,8 +10,13 @@ use wasm_bindgen::{JsCast, JsValue}; use wasm_bindgen_futures::JsFuture; use web_sys::{FileSystemDirectoryHandle, FileSystemFileHandle}; +/// Called with whether a queued write reached OPFS. +pub type Done = Box; + thread_local! { - static QUEUE: RefCell<(BTreeMap>, bool)> = const { RefCell::new((BTreeMap::new(), false)) }; + /// Pending writes per path (bytes and the callbacks of every write coalesced into them), and + /// whether the writer task is running. + static QUEUE: RefCell<(BTreeMap, Vec)>, bool)> = const { RefCell::new((BTreeMap::new(), false)) }; } /// Whether this browser has OPFS (`navigator.storage.getDirectory`). @@ -60,9 +65,17 @@ async fn write_now(rel: &str, data: &[u8]) -> Result<(), JsValue> { /// Queue a write of `data` to `rel` (relative to the OPFS root). pub fn write(rel: String, data: Vec) { + write_then(rel, data, None); +} + +/// [`write`], then call `done` with whether it succeeded. A write replaced by a newer one to the +/// same path reports the newer one's result. +pub fn write_then(rel: String, data: Vec, done: Option) { let start = QUEUE.with(|q| { let mut q = q.borrow_mut(); - q.0.insert(rel, data); + let e = q.0.entry(rel).or_default(); + e.0 = data; + e.1.extend(done); !std::mem::replace(&mut q.1, true) }); if !start { @@ -81,10 +94,14 @@ pub fn write(rel: String, data: Vec) { } } }); - let Some((rel, data)) = next else { break }; - if let Err(e) = write_now(&rel, &data).await { + let Some((rel, (data, done))) = next else { break }; + let r = write_now(&rel, &data).await; + if let Err(e) = &r { log::warn!("OPFS write {rel}: {e:?}"); } + for d in done { + d(r.is_ok()); + } } }); } diff --git a/apps/filmcraft-web/src/recovery.rs b/apps/filmcraft-web/src/recovery.rs index 939559a4..6001531e 100644 --- a/apps/filmcraft-web/src/recovery.rs +++ b/apps/filmcraft-web/src/recovery.rs @@ -6,24 +6,32 @@ //! - Imported media are copied to `media/` (the browser streams the copy; nothing goes //! through wasm memory), so the snapshot's `/files/` references resolve after a reload. //! - At start-up a dirty snapshot is reopened (`?norecover` skips it, `?fresh` also skips the media). +//! A skipped snapshot stays dirty until this session writes its own ([`crate::recovery_policy`]). + +use std::cell::{Cell, RefCell}; +use std::rc::Rc; use filmcraft_engine::Session; use serde_json::{Value, json}; use crate::opfs; +use crate::recovery_policy::{Action, Policy}; const SNAPSHOT: &str = "recovery/snapshot.fcproj"; const META: &str = "recovery/meta.json"; -/// Seconds between snapshots of a changing project. -const INTERVAL_S: f64 = 5.0; /// Media larger than this are not copied to OPFS. const KEEP_MAX: f64 = 4.0 * 1024.0 * 1024.0 * 1024.0; +/// A finished OPFS write, reported back to the next [`Autosave::tick`]. +enum Outcome { + Snapshot { revision: u64, ok: bool }, + Clean { ok: bool }, +} + #[derive(Default)] pub struct Autosave { - last_revision: u64, - last_time: f64, - last_dirty: Option, + policy: Policy, + outcomes: Rc>>, } impl Autosave { @@ -33,22 +41,44 @@ impl Autosave { return; } let now = ctx.input(|i| i.time); - let dirty = s.revision != s.saved_revision; - if dirty { - if s.revision != self.last_revision && now - self.last_time >= INTERVAL_S { - opfs::write(SNAPSHOT.into(), filmcraft_format::encode(&s.project, false)); - let meta = json!({"dirty": true, "name": s.project.name, "revision": s.revision, "time": js_sys::Date::now()}); - opfs::write(META.into(), meta.to_string().into_bytes()); - self.last_revision = s.revision; - self.last_time = now; - } else if s.revision != self.last_revision { - // the UI may go idle: wake up when the snapshot is due - ctx.request_repaint_after(std::time::Duration::from_secs_f64((INTERVAL_S - (now - self.last_time)).max(0.05))); + for o in self.outcomes.borrow_mut().drain(..) { + match o { + Outcome::Snapshot { revision, ok } => self.policy.written(revision, ok, now), + Outcome::Clean { ok } => self.policy.cleaned(ok, now), + } + } + match self.policy.tick(now, s.revision, s.saved_revision) { + Action::Nothing => {} + Action::WakeAfter(secs) => ctx.request_repaint_after(std::time::Duration::from_secs_f64(secs)), + Action::WriteSnapshot(revision) => { + // Snapshot and meta both have to land; either failing retries the pair. + let failed = Rc::new(Cell::new(false)); + let left = Rc::new(Cell::new(2u8)); + let done = |out: Rc>>, failed: Rc>, left: Rc>, ctx: egui::Context| -> opfs::Done { + Box::new(move |ok| { + failed.set(failed.get() || !ok); + left.set(left.get().saturating_sub(1)); + if left.get() == 0 { + out.borrow_mut().push(Outcome::Snapshot { revision, ok: !failed.get() }); + ctx.request_repaint(); + } + }) + }; + let meta = json!({"dirty": true, "name": s.project.name, "revision": s.revision, "time": js_sys::Date::now()}); + let d1 = done(self.outcomes.clone(), failed.clone(), left.clone(), ctx.clone()); + let d2 = done(self.outcomes.clone(), failed, left, ctx.clone()); + opfs::write_then(SNAPSHOT.into(), filmcraft_format::encode(&s.project, false), Some(d1)); + opfs::write_then(META.into(), meta.to_string().into_bytes(), Some(d2)); + } + Action::WriteClean => { + let (out, ctx) = (self.outcomes.clone(), ctx.clone()); + let done: opfs::Done = Box::new(move |ok| { + out.borrow_mut().push(Outcome::Clean { ok }); + ctx.request_repaint(); + }); + opfs::write_then(META.into(), json!({"dirty": false}).to_string().into_bytes(), Some(done)); } - } else if self.last_dirty != Some(false) { - opfs::write(META.into(), json!({"dirty": false}).to_string().into_bytes()); } - self.last_dirty = Some(dirty); } } diff --git a/apps/filmcraft-web/src/recovery_policy.rs b/apps/filmcraft-web/src/recovery_policy.rs new file mode 100644 index 00000000..8426d3c3 --- /dev/null +++ b/apps/filmcraft-web/src/recovery_policy.rs @@ -0,0 +1,88 @@ +//! When the web recovery snapshot is written or marked clean (pure, so it is tested natively in +//! `tests/recovery_policy.rs`; the OPFS side lives in [`crate::recovery`]). +//! +//! - A dirty project is snapshotted at most every [`INTERVAL_S`] seconds, once per revision. A write +//! that fails is retried after the interval, without waiting for another edit (#197). +//! - The meta is marked clean only to retire a snapshot this session wrote. A snapshot left by an +//! earlier session that was not reopened (`?norecover`, `?fresh`) stays dirty (#193). + +/// Seconds between snapshots of a changing project. +pub const INTERVAL_S: f64 = 5.0; + +/// What the caller should do this frame. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Action { + Nothing, + /// Write the snapshot and a dirty meta for this revision, then report with [`Policy::written`]. + WriteSnapshot(u64), + /// Write a clean meta, then report with [`Policy::cleaned`]. + WriteClean, + /// Nothing now; wake up after this many seconds. + WakeAfter(f64), +} + +#[derive(Debug, Default)] +pub struct Policy { + /// Revision whose snapshot was last written or is being written. + last_revision: Option, + last_time: Option, + /// This session wrote a dirty meta that is not yet retired. + owns_dirty_meta: bool, + /// A clean-meta write is in flight. + cleaning: bool, + /// Snapshot writes started so far, and how many had started when the clean write did (a + /// snapshot started after it makes the meta dirty again). + snapshots: u64, + snapshots_at_clean: u64, +} + +impl Policy { + /// Called every frame with the session's revisions and the time in seconds. + pub fn tick(&mut self, now: f64, revision: u64, saved_revision: u64) -> Action { + if revision != saved_revision { + if self.last_revision == Some(revision) { + return Action::Nothing; + } + let since = self.last_time.map_or(f64::INFINITY, |t| now - t); + if since >= INTERVAL_S { + self.last_revision = Some(revision); + self.last_time = Some(now); + self.owns_dirty_meta = true; + self.snapshots += 1; + return Action::WriteSnapshot(revision); + } + return Action::WakeAfter((INTERVAL_S - since).max(0.05)); + } + if self.owns_dirty_meta && !self.cleaning { + let since = self.last_time.map_or(f64::INFINITY, |t| now - t); + if self.last_revision.is_some() || since >= INTERVAL_S { + self.cleaning = true; + self.snapshots_at_clean = self.snapshots; + return Action::WriteClean; + } + return Action::WakeAfter((INTERVAL_S - since).max(0.05)); + } + Action::Nothing + } + + /// The snapshot write for `revision` finished. + pub fn written(&mut self, revision: u64, ok: bool, now: f64) { + if !ok && self.last_revision == Some(revision) { + // retry after the interval, even if nothing else changes + self.last_revision = None; + self.last_time = Some(now); + } + } + + /// The clean-meta write finished. + pub fn cleaned(&mut self, ok: bool, now: f64) { + self.cleaning = false; + if ok { + self.owns_dirty_meta = self.snapshots != self.snapshots_at_clean; + } else { + // retry after the interval + self.last_revision = None; + self.last_time = Some(now); + } + } +} diff --git a/apps/filmcraft-web/tests/import-bins.mjs b/apps/filmcraft-web/tests/import-bins.mjs new file mode 100644 index 00000000..6e181171 --- /dev/null +++ b/apps/filmcraft-web/tests/import-bins.mjs @@ -0,0 +1,67 @@ +// Web import destinations: active bin tab, browser drop + navigation during pending reads. +// Synthetic browser drop event exercises the production handler; physical OS drag/picker UI is outside this test. +export async function checkImportBins({ evaluate, media, timeoutMs = 120000 }) { + const report = {gates: {}, cases: []}; + await evaluate('filmcraft.execute("file.newProject", {name: "Import bins"})'); + const parent = (await evaluate('filmcraft.execute("file.newBin", {name: "Footage"})')).bin; + const child = (await evaluate(`filmcraft.execute("file.newBin", {name: "Selects", parent: ${parent}})`)).bin; + await evaluate(`(async () => {window.__importBinFixture = new Uint8Array(await (await fetch(${JSON.stringify(media)})).arrayBuffer());})()`); + const setBin = bin => evaluate(`filmcraft.request("ui.set", {projectPanel: {bin: ${JSON.stringify(bin)}, activeTab: null}})`); + const importExpression = name => `filmcraft.importFiles([new File([window.__importBinFixture], ${JSON.stringify(name)}, {type: "video/mp4"})])`; + const locate = async name => { + const deadline = Date.now() + timeoutMs; + const pollMs = 100; + for (;;) { + const project = await evaluate('filmcraft.execute("project.inspect")'); + const find = bin => { + for (const entry of bin.children) { + if (entry.item !== undefined && entry.name === name) return bin.id; + if (entry.children) {const found = find(entry); if (found !== undefined) return found;} + } + }; + const found = find(project.root); + if (found !== undefined) return found; + if (Date.now() >= deadline) throw new Error(`Import never appeared: ${name}`); + await new Promise(resolve => setTimeout(resolve, pollMs)); + } + }; + const check = async (key, expected, name) => { + const actual = await locate(name); + report.cases.push({key, expected, actual, name}); report.gates[key] = actual === expected; + }; + await evaluate(`filmcraft.request("ui.set", {projectPanel: {bin: ${child}, activeTab: 0, tabs: [{bin: ${parent}}]}})`); + await evaluate(importExpression("tab.mp4")); await check("activeTab", parent, "tab.mp4"); + await setBin(child); + await evaluate(`(() => { + const transfer = new DataTransfer(); + transfer.items.add(new File([window.__importBinFixture], "drop.mp4", {type: "video/mp4"})); + window.dispatchEvent(new DragEvent("drop", {dataTransfer: transfer, bubbles: true, cancelable: true})); + })()`); + await check("browserDrop", child, "drop.mp4"); + await setBin(child); + await evaluate(`(() => { + const original = Blob.prototype.arrayBuffer; + let release; + const pending = new Promise(resolve => {release = resolve;}); + const gate = window.__importBinGate = {hits: 0, release, original}; + Blob.prototype.arrayBuffer = function () {gate.hits++; return pending.then(() => original.call(this));}; + window.__pendingBinImport = ${importExpression("pending.mp4")}; + })()`); + try { + const deadline = Date.now() + timeoutMs; + const pollMs = 100; + for (;;) { + if (await evaluate("window.__importBinGate.hits > 0")) break; + if (Date.now() >= deadline) throw new Error("Read-delay probe did not intercept any Blob reads"); + await new Promise(resolve => setTimeout(resolve, pollMs)); + } + report.delayedReadHits = await evaluate("window.__importBinGate.hits"); + await setBin(parent); + await evaluate("window.__importBinGate.release(); window.__pendingBinImport"); + await check("destinationCapturedBeforeRead", child, "pending.mp4"); + } finally { + await evaluate("Blob.prototype.arrayBuffer = window.__importBinGate.original; window.__importBinGate.release(); delete window.__importBinGate; delete window.__pendingBinImport; delete window.__importBinFixture"); + } + report.ok = Object.values(report.gates).every(Boolean); + return report; +} diff --git a/apps/filmcraft-web/tests/import-collision.mjs b/apps/filmcraft-web/tests/import-collision.mjs new file mode 100644 index 00000000..dfde3b66 --- /dev/null +++ b/apps/filmcraft-web/tests/import-collision.mjs @@ -0,0 +1,78 @@ +// #194 regression: same-name/same-size imports must retain independent sources across OPFS recovery. +// evaluate(expression) awaits a browser expression; navigate(url) waits for FilmCraft readiness. Generated PCM16 mono WAVs only; no binary fixtures. +import { createHash } from "node:crypto"; + +function tone(frequency, seconds) { + // RIFF/WAVE PCM16 mono: 44-byte header + 2 bytes/sample; fixture matches #194 (48 kHz, amplitude 12000). + const rate = 48000, bytesPerSample = 2, samples = rate * seconds; + const b = Buffer.alloc(44 + samples * bytesPerSample); + b.write("RIFF", 0); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8); + b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); + b.writeUInt32LE(rate, 24); b.writeUInt32LE(rate * bytesPerSample, 28); + b.writeUInt16LE(bytesPerSample, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * bytesPerSample, 40); + for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(12000 * Math.sin(2 * Math.PI * frequency * i / rate)), 44 + bytesPerSample * i); + return b; +} + +export async function checkImportCollision({ evaluate, navigate, url, timeoutMs = 120000 }) { + const fixtures = [tone(440, 2), tone(880, 2)]; + const hashes = fixtures.map(b => createHash("sha256").update(b).digest("hex")); + const report = { hashes, fixtureBytes: fixtures.map(b => b.length), gates: {} }; + const hash = expression => evaluate(`(async () => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", ${expression})), b => b.toString(16).padStart(2, "0")).join(""))()`); + const readHash = path => hash(`await filmcraft.readFile(${JSON.stringify(path)})`); + const importFile = index => evaluate(`filmcraft.importFiles([new File([Uint8Array.from(atob(${JSON.stringify(fixtures[index].toString("base64"))}), c => c.charCodeAt(0))], "collision.wav", {type: "audio/wav"})])`); + await evaluate('filmcraft.execute("file.newProject", {name: "Import collision"})'); + const a = await importFile(0); report.a = a; + if (a.errors?.length || !a.items?.length) throw new Error("baseline import failed: " + JSON.stringify(a)); + report.initialHash = await readHash(a.paths[0]); + report.gates.healthyInitial = report.initialHash === hashes[0]; + await evaluate(`filmcraft.execute("file.newSequence", {name: "Original A", fromItem: ${a.items[0]}})`); + const b = await importFile(1); report.b = b; + if (b.errors?.length || !b.items?.length) throw new Error("second import failed: " + JSON.stringify(b)); + report.afterBHash = await readHash(a.paths[0]); + report.gates.distinctPaths = a.paths[0] !== b.paths[0]; + report.gates.originalPreserved = report.afterBHash === hashes[0]; + report.beforeProject = await evaluate('filmcraft.execute("project.inspect")'); + const revision = report.beforeProject.revision; + const deadline = Date.now() + timeoutMs; + const persistencePollMs = 100; + for (;;) { + const snapshot = await evaluate(`(async () => { + try { + const dir = await (await navigator.storage.getDirectory()).getDirectoryHandle("recovery"); + const read = async name => JSON.parse(await (await (await dir.getFileHandle(name)).getFile()).text()); + return {meta: await read("meta.json"), project: await read("snapshot.fcproj")}; + } catch (e) {return {error: String(e)};} + })()`); + report.recoveryMeta = snapshot.meta; + if (snapshot.meta?.dirty && snapshot.meta.revision === revision && JSON.stringify(snapshot.project).includes("Original A")) break; + if (Date.now() >= deadline) throw new Error("Recovery persistence gate failed: " + JSON.stringify(snapshot)); + await new Promise(resolve => setTimeout(resolve, persistencePollMs)); + } + // Media writes run separately from the snapshot queue; gate their actual bytes before navigation. + for (;;) { + report.opfs = await evaluate(`(async () => { + const dir = await (await navigator.storage.getDirectory()).getDirectoryHandle("media"); + const files = []; + for await (const [name, handle] of dir.entries()) { + const bytes = await (await handle.getFile()).arrayBuffer(); + files.push({name, hash: Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), b => b.toString(16).padStart(2, "0")).join("")}); + } + return files; + })()`); + // Before the fix, A and B intentionally share the same path; qualify the last write there. + const expected = new Map([[a.paths[0], hashes[0]], [b.paths[0], hashes[1]]]); + if ([...expected].every(([path, digest]) => report.opfs.some(f => f.name === path.slice("/files/".length) && f.hash === digest))) break; + if (Date.now() >= deadline) throw new Error("Imported media did not reach OPFS: " + JSON.stringify(report.opfs)); + await new Promise(resolve => setTimeout(resolve, persistencePollMs)); + } + const recoveryUrl = new URL(url); + recoveryUrl.searchParams.delete("fresh"); recoveryUrl.searchParams.delete("norecover"); + await navigate(recoveryUrl.href); + report.recovered = await evaluate('filmcraft.execute("project.inspect")'); + report.afterReload = {a: await readHash(a.paths[0]), b: await readHash(b.paths[0])}; + report.gates.recoveredProject = JSON.stringify(report.recovered).includes("Original A"); + report.gates.recoveredSources = report.afterReload.a === hashes[0] && report.afterReload.b === hashes[1]; + report.ok = Object.values(report.gates).every(Boolean); + return report; +} diff --git a/apps/filmcraft-web/tests/recovery_policy.rs b/apps/filmcraft-web/tests/recovery_policy.rs new file mode 100644 index 00000000..c60b34d5 --- /dev/null +++ b/apps/filmcraft-web/tests/recovery_policy.rs @@ -0,0 +1,77 @@ +//! The web recovery snapshot policy. It is pure, so it runs natively even though the rest of the +//! crate is wasm32-only. + +#[path = "../src/recovery_policy.rs"] +mod recovery_policy; + +use recovery_policy::{Action, INTERVAL_S, Policy}; + +/// #197: a snapshot write that fails is retried after the interval without another edit. +#[test] +fn failed_snapshot_write_is_retried_without_another_edit() { + let mut p = Policy::default(); + assert_eq!(p.tick(0.0, 3, 0), Action::WriteSnapshot(3)); + p.written(3, false, 0.1); + // not immediately (no spinning on a failing disk)… + assert!(matches!(p.tick(1.0, 3, 0), Action::WakeAfter(_))); + // …but once the interval has passed, with the revision unchanged + assert_eq!(p.tick(0.1 + INTERVAL_S, 3, 0), Action::WriteSnapshot(3)); + p.written(3, true, 5.2); + assert_eq!(p.tick(20.0, 3, 0), Action::Nothing); +} + +/// A failure reported for an older revision doesn't make the newer one be written again. +#[test] +fn stale_failure_is_ignored() { + let mut p = Policy::default(); + assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1)); + assert_eq!(p.tick(INTERVAL_S, 2, 0), Action::WriteSnapshot(2)); + p.written(1, false, INTERVAL_S); + assert_eq!(p.tick(3.0 * INTERVAL_S, 2, 0), Action::Nothing); +} + +/// #193: a clean session (started with `?norecover` / `?fresh`, so the snapshot left by an +/// earlier session was not reopened) never marks that snapshot clean. +#[test] +fn clean_start_leaves_an_unconsumed_snapshot_dirty() { + let mut p = Policy::default(); + for t in 0..100 { + assert_eq!(p.tick(t as f64, 0, 0), Action::Nothing); + } +} + +/// The snapshot this session wrote is retired once the project is saved. +#[test] +fn saving_marks_own_snapshot_clean() { + let mut p = Policy::default(); + assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1)); + p.written(1, true, 0.1); + assert_eq!(p.tick(1.0, 1, 1), Action::WriteClean); + assert_eq!(p.tick(1.1, 1, 1), Action::Nothing); + p.cleaned(true, 1.2); + assert_eq!(p.tick(30.0, 1, 1), Action::Nothing); +} + +/// A failed clean write is retried after the interval. +#[test] +fn failed_clean_write_is_retried() { + let mut p = Policy::default(); + assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1)); + p.written(1, true, 0.1); + assert_eq!(p.tick(1.0, 1, 1), Action::WriteClean); + p.cleaned(false, 1.1); + assert!(matches!(p.tick(2.0, 1, 1), Action::WakeAfter(_))); + assert_eq!(p.tick(1.1 + INTERVAL_S, 1, 1), Action::WriteClean); +} + +/// An edit made while the clean write is in flight keeps the meta dirty after it lands. +#[test] +fn edit_during_clean_write_keeps_meta_owned() { + let mut p = Policy::default(); + assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1)); + assert_eq!(p.tick(10.0, 1, 1), Action::WriteClean); + assert_eq!(p.tick(20.0, 2, 1), Action::WriteSnapshot(2)); + p.cleaned(true, 20.1); + // saved again: the newer snapshot must be retired too + assert_eq!(p.tick(30.0, 2, 2), Action::WriteClean); +} diff --git a/apps/filmcraft-web/tests/smoke.mjs b/apps/filmcraft-web/tests/smoke.mjs index f43ceec0..71951edf 100644 --- a/apps/filmcraft-web/tests/smoke.mjs +++ b/apps/filmcraft-web/tests/smoke.mjs @@ -13,6 +13,8 @@ import { spawn } from "node:child_process"; import { mkdirSync, writeFileSync, readdirSync, statSync, mkdtempSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; +import { checkImportBins } from "./import-bins.mjs"; +import { checkImportCollision } from "./import-collision.mjs"; const arg = (k, d) => { const i = process.argv.indexOf(`--${k}`); @@ -162,6 +164,19 @@ try { await sleep(500); await send("Emulation.setDeviceMetricsOverride", { width: 1600, height: 1000, deviceScaleFactor: 1, mobile: false }); report.steps.modes = { screenshot: await shot("05-modes") }; + report.steps.importBins = await checkImportBins({evaluate: js, media}); + if (!report.steps.importBins.ok) throw new Error("import bin regression: " + JSON.stringify(report.steps.importBins.gates)); + report.steps.importCollision = await checkImportCollision({ + evaluate: js, + url, + navigate: async (nextUrl) => { + await send("Page.navigate", { url: nextUrl }); + await until("!!(window.filmcraftLoad && (window.filmcraftLoad.readyMs || window.filmcraftLoad.error))", 120000); + const load = await js("window.filmcraftLoad"); + if (load.error) throw new Error(load.error); + }, + }); + if (!report.steps.importCollision.ok) throw new Error("import collision regression: " + JSON.stringify(report.steps.importCollision.gates)); // still alive? (a panicked app never answers) await Promise.race([js("filmcraft.inspect().then(() => true)"), sleep(5000).then(() => { throw new Error("app stopped answering"); })]); const fatal = logs.filter((l) => /panicked at|RuntimeError|EXCEPTION/.test(l)); diff --git a/apps/filmcraft/src/logging.rs b/apps/filmcraft/src/logging.rs new file mode 100644 index 00000000..3d7c713f --- /dev/null +++ b/apps/filmcraft/src/logging.rs @@ -0,0 +1,562 @@ +//! The desktop app's logger: `log::` records go to standard error and to a log file. +//! +//! Without a logger every `log::warn!`/`log::error!` in the workspace (GPU compositor fallbacks, +//! audio output problems, a panicked frame job or audio mixer, container warnings) vanished. A +//! launch from a desktop menu has no terminal, so the file is what a bug report can attach: +//! `/Logs/filmcraft.log`, next to the crash logs (`--data-dir`, `FILMCRAFT_DATA_DIR` or +//! `filmcraft_engine::autosave::default_data_dir`; Help ▸ Reveal Log Files opens the folder). Each +//! start moves the previous log to `filmcraft.1.log` (and that one to `.2`), so the log of a run +//! that crashed survives the next launch. +//! +//! Levels: `info` for FilmCraft's own crates, `warn` for everything else (wgpu and naga are +//! chatty). `RUST_LOG` replaces that with env_logger-style directives: `debug`, +//! `warn,filmcraft_codecs=trace`, `wgpu_core=info`. A directive ending in `*` matches every target +//! that starts with it (`filmcraft*=debug`). +//! +//! Records logged before the data directory is known are kept (up to [`MAX_PENDING`]) and written +//! once the file is attached. Writing never panics: a file that can't be created or written leaves +//! standard error as the only sink. +//! +//! Realtime audio: the cpal callbacks (`audio.rs`, `audio_in.rs`, the play-ahead reader) do not +//! log; they report through flags and a one-time `eprintln!`. A record takes the sink lock only for +//! one `write_all` of an already formatted line (standard error is written outside the lock), so a +//! logging thread such as the audio mixer never waits on more than one other line being written. + +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock, PoisonError}; +use std::time::SystemTime; + +use log::LevelFilter; + +/// The current log file's name inside the log directory. +pub const LOG_FILE: &str = "filmcraft.log"; +/// The log directory inside the data directory (shared with the crash logs and Reveal Log Files). +pub const LOG_DIR: &str = "Logs"; +/// How many previous logs are kept (`filmcraft.1.log` … `filmcraft..log`). +pub const KEEP: usize = 2; +/// The log file stops growing past this size (a runaway warning can't fill the disk). +pub const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024; +/// Records kept in memory until the log file is attached. +pub const MAX_PENDING: usize = 512; +/// The built-in filter when `RUST_LOG` is unset or empty. +pub const DEFAULT_FILTER: &str = "warn,filmcraft*=info"; + +/// Per-target level filter parsed from env_logger-style directives. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Filter { + default: LevelFilter, + directives: Vec<(String, LevelFilter)>, +} + +impl Filter { + /// Parse `spec`; unknown levels and empty parts are skipped. Without a bare level, targets no + /// directive names log errors only. + pub fn parse(spec: &str) -> Filter { + let mut f = Filter { default: LevelFilter::Error, directives: Vec::new() }; + for part in spec.split(',').map(str::trim).filter(|p| !p.is_empty()) { + match part.split_once('=') { + Some((name, level)) => { + let name = name.trim(); + if let (false, Some(level)) = (name.is_empty(), parse_level(level)) { + f.directives.push((name.to_owned(), level)); + } + } + None => match parse_level(part) { + Some(level) => f.default = level, + // A bare target name: everything from it (env_logger does the same). + None => f.directives.push((part.to_owned(), LevelFilter::Trace)), + }, + } + } + f + } + + /// The level that applies to `target` (the most specific matching directive wins). + pub fn level_for(&self, target: &str) -> LevelFilter { + self.directives.iter().filter(|(name, _)| matches(name, target)).max_by_key(|(name, _)| name.len()).map_or(self.default, |(_, level)| *level) + } + + /// The most verbose level any target can reach (for `log::set_max_level`). + pub fn max(&self) -> LevelFilter { + self.directives.iter().map(|(_, level)| *level).fold(self.default, Ord::max) + } +} + +fn parse_level(s: &str) -> Option { + s.trim().parse().ok() +} + +/// `name` matches `target` itself and its submodules (`a` matches `a` and `a::b`, not `ab`); +/// `name*` matches every target starting with `name`. +fn matches(name: &str, target: &str) -> bool { + match name.strip_suffix('*') { + Some(prefix) => target.starts_with(prefix), + None => target.strip_prefix(name).is_some_and(|rest| rest.is_empty() || rest.starts_with("::")), + } +} + +/// `2026-10-08T07:59:17.728Z` (UTC, milliseconds). A clock before 1970 reads as the epoch. +pub fn timestamp(t: SystemTime) -> String { + let since = t.duration_since(std::time::UNIX_EPOCH).unwrap_or_default(); + let secs = since.as_secs(); + let (days, rem) = (secs / 86_400, secs % 86_400); + let (y, m, d) = civil_from_days(days); + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.{:03}Z", rem / 3600, rem % 3600 / 60, rem % 60, since.subsec_millis()) +} + +/// Days since 1970-01-01 to a proleptic Gregorian (year, month, day) (Howard Hinnant's +/// `civil_from_days`, unsigned because `timestamp` clamps at the epoch; saturating so an absurd +/// clock can't overflow). +fn civil_from_days(days: u64) -> (u64, u64, u64) { + let z = days.saturating_add(719_468); + let era = z / 146_097; + let doe = z % 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = era.saturating_mul(400).saturating_add(yoe).saturating_add(u64::from(m <= 2)); + (y, m, d) +} + +fn numbered(dir: &Path, n: usize) -> PathBuf { + if n == 0 { dir.join(LOG_FILE) } else { dir.join(format!("filmcraft.{n}.log")) } +} + +/// Shift the previous logs up one (`.log` → `.1.log` → … → `..log`, the oldest dropped) +/// and create a fresh, empty log file in `dir` (created if missing). +pub fn rotate(dir: &Path) -> std::io::Result<(PathBuf, File)> { + std::fs::create_dir_all(dir)?; + for n in (1..=KEEP).rev() { + match std::fs::rename(numbered(dir, n - 1), numbered(dir, n)) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e), + _ => {} + } + } + let path = numbered(dir, 0); + let file = File::create(&path)?; + Ok((path, file)) +} + +/// One formatted log line: ` [] : \n`. +pub fn format_line(ts: &str, level: log::Level, thread: &str, target: &str, message: &str) -> String { + let message = message.strip_suffix('\n').unwrap_or(message); + format!("{ts} {level:<5} [{thread}] {target}: {message}\n") +} + +/// Where formatted lines go besides standard error: kept in memory until a file is attached, +/// then appended to it until it reaches its size cap. +pub struct Sink { + file: Option, + pending: Vec, + dropped: usize, + written: u64, + max_bytes: u64, + capped: bool, +} + +impl Sink { + pub fn new(max_bytes: u64) -> Sink { + Sink { file: None, pending: Vec::new(), dropped: 0, written: 0, max_bytes, capped: false } + } + + /// Append `line` to the file, or keep it until [`Sink::attach`]. + pub fn write(&mut self, line: &str) { + if self.file.is_none() { + if self.capped { + return; + } + if self.pending.len() < MAX_PENDING { + self.pending.push(line.to_owned()); + } else { + self.dropped = self.dropped.saturating_add(1); + } + return; + } + self.append(line); + } + + /// Start writing to `file`, first the lines kept so far. + pub fn attach(&mut self, file: File) { + self.file = Some(file); + for line in std::mem::take(&mut self.pending) { + self.append(&line); + } + if self.dropped > 0 { + let note = format!("{} earlier log lines were dropped before the log file was opened\n", self.dropped); + self.dropped = 0; + self.append(¬e); + } + } + + /// No file will come: drop the kept lines and keep none from now on. + pub fn close(&mut self) { + self.file = None; + self.pending = Vec::new(); + self.capped = true; + } + + fn append(&mut self, line: &str) { + if self.capped { + return; + } + let Some(file) = self.file.as_mut() else { return }; + let len = u64::try_from(line.len()).unwrap_or(u64::MAX); + let text = if self.written.saturating_add(len) > self.max_bytes { + self.capped = true; + format!("log file reached {} bytes; later records go to standard error only\n", self.max_bytes) + } else { + self.written = self.written.saturating_add(len); + line.to_owned() + }; + // A full disk or a vanished file must not take the app down: drop the file sink (and + // stop buffering, since no file will come). + if file.write_all(text.as_bytes()).is_err() { + self.file = None; + self.capped = true; + } + } +} + +/// The installed logger: a [`Filter`], standard error and a [`Sink`]. +pub struct AppLogger { + filter: Filter, + stderr: bool, + sink: Mutex, +} + +impl AppLogger { + pub fn new(filter: Filter, stderr: bool) -> AppLogger { + AppLogger { filter, stderr, sink: Mutex::new(Sink::new(MAX_FILE_BYTES)) } + } + + /// Rotate the logs in `dir` and send records to the fresh file; returns its path. + /// An unusable directory leaves standard error as the only sink ([`AppLogger::stderr_only`]). + pub fn attach_dir(&self, dir: &Path) -> Result { + match rotate(dir) { + Ok((path, file)) => { + self.sink.lock().unwrap_or_else(PoisonError::into_inner).attach(file); + Ok(path) + } + Err(e) => { + self.stderr_only(); + Err(format!("{}: {e}", dir.display())) + } + } + } + + /// No log file this run (no data directory, or it can't be used): stop keeping records for one. + pub fn stderr_only(&self) { + self.sink.lock().unwrap_or_else(PoisonError::into_inner).close(); + } +} + +impl log::Log for AppLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= self.filter.level_for(metadata.target()) + } + + fn log(&self, record: &log::Record<'_>) { + if !self.enabled(record.metadata()) { + return; + } + // Format before taking the lock, so the lock covers one write only. + let thread = std::thread::current(); + let line = format_line(×tamp(SystemTime::now()), record.level(), thread.name().unwrap_or("?"), record.target(), &record.args().to_string()); + if self.stderr { + let _ = std::io::stderr().write_all(line.as_bytes()); + } + self.sink.lock().unwrap_or_else(PoisonError::into_inner).write(&line); + } + + fn flush(&self) {} +} + +/// Install the logger (filter from `RUST_LOG`, else [`DEFAULT_FILTER`]); `None` when another +/// logger was installed first. Call [`AppLogger::attach_dir`] once the data directory is known. +pub fn install() -> Option<&'static AppLogger> { + static LOGGER: OnceLock = OnceLock::new(); + let spec = std::env::var("RUST_LOG").ok().filter(|s| !s.trim().is_empty()); + let filter = Filter::parse(spec.as_deref().unwrap_or(DEFAULT_FILTER)); + let max = filter.max(); + let logger = LOGGER.get_or_init(|| AppLogger::new(filter, true)); + log::set_logger(logger).ok()?; + log::set_max_level(max); + Some(logger) +} + +#[cfg(test)] +mod tests { + use super::*; + use log::Log; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::{Duration, UNIX_EPOCH}; + + fn temp_dir(tag: &str) -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = std::env::temp_dir().join(format!("filmcraft-logging-{tag}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + let _ = std::fs::remove_dir_all(&d); + d + } + + fn read(path: &Path) -> String { + std::fs::read_to_string(path).expect("read log") + } + + fn record(logger: &AppLogger, level: log::Level, target: &'static str, msg: &'static str) { + logger.log(&log::Record::builder().level(level).target(target).args(format_args!("{msg}")).build()); + } + + #[test] + fn the_default_filter_shows_filmcraft_info_and_other_crates_warnings() { + let f = Filter::parse(DEFAULT_FILTER); + assert_eq!(f.level_for("filmcraft"), LevelFilter::Info); + assert_eq!(f.level_for("filmcraft::audio"), LevelFilter::Info); + assert_eq!(f.level_for("filmcraft_ui_egui::play_ahead"), LevelFilter::Info); + assert_eq!(f.level_for("filmcraft_codecs::mxf"), LevelFilter::Info); + assert_eq!(f.level_for("wgpu_core::device"), LevelFilter::Warn); + assert_eq!(f.level_for("naga"), LevelFilter::Warn); + assert_eq!(f.max(), LevelFilter::Info); + } + + #[test] + fn directives_follow_env_logger_and_the_most_specific_one_wins() { + let f = Filter::parse("info,wgpu_core=error,filmcraft_codecs=trace,filmcraft_codecs::mxf=off"); + assert_eq!(f.level_for("eframe"), LevelFilter::Info); + assert_eq!(f.level_for("wgpu_core::instance"), LevelFilter::Error); + assert_eq!(f.level_for("filmcraft_codecs::ogg"), LevelFilter::Trace); + assert_eq!(f.level_for("filmcraft_codecs::mxf"), LevelFilter::Off); + assert_eq!(f.max(), LevelFilter::Trace); + // A module name is matched at `::` boundaries, not as a bare prefix. + assert_eq!(f.level_for("wgpu_core_extra"), LevelFilter::Info); + // A trailing `*` is a prefix. + assert_eq!(Filter::parse("warn,filmcraft_ui*=debug").level_for("filmcraft_ui_egui::frames"), LevelFilter::Debug); + assert_eq!(Filter::parse("warn,filmcraft_ui*=debug").level_for("filmcraft_engine"), LevelFilter::Warn); + // A bare target name sets that target to the most verbose level, as env_logger does. + assert_eq!(Filter::parse("naga").level_for("naga::front"), LevelFilter::Trace); + assert_eq!(Filter::parse("naga").level_for("eframe"), LevelFilter::Error); + // Levels are case-insensitive; the last bare level wins. + assert_eq!(Filter::parse("debug,WARN").level_for("x"), LevelFilter::Warn); + } + + #[test] + fn hostile_specs_never_panic_and_fall_back_sensibly() { + let long = "a".repeat(100_000); + let specs = [ + "", + ",,,", + "=", + "==", + "=debug", + "filmcraft=", + "nonsense=loud", + "🦀=info", + "*", + "*=debug", + "a=b=c", + "warn,,filmcraft=DEBUG", + " \t ", + "\u{0}", + "é*=trace", + &long, + ]; + for spec in specs { + let f = Filter::parse(spec); + for target in ["filmcraft", "", "🦀", "é", "a::b"] { + let _ = f.level_for(target); + } + let _ = f.max(); + } + assert_eq!(Filter::parse("warn,,filmcraft=DEBUG").level_for("filmcraft"), LevelFilter::Debug); + assert_eq!(Filter::parse("nonsense=loud").level_for("nonsense"), LevelFilter::Error); + assert_eq!(Filter::parse("=debug").max(), LevelFilter::Error, "a directive without a name is skipped"); + assert_eq!(Filter::parse("*=debug").level_for("anything"), LevelFilter::Debug, "a lone `*` matches every target"); + assert_eq!(Filter::parse("é*=trace").level_for("éa"), LevelFilter::Trace); + } + + #[test] + fn timestamps_are_utc_with_milliseconds() { + assert_eq!(timestamp(UNIX_EPOCH), "1970-01-01T00:00:00.000Z"); + assert_eq!(timestamp(UNIX_EPOCH + Duration::from_millis(1_791_446_357_728)), "2026-10-08T07:59:17.728Z"); + // Leap day, and the day after it. + assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_164_800)), "2024-02-29T00:00:00.000Z"); + assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_251_199)), "2024-02-29T23:59:59.000Z"); + assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_251_200)), "2024-03-01T00:00:00.000Z"); + // 2000 is a leap year (divisible by 400), 2100 is not. + assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(951_782_400)), "2000-02-29T00:00:00.000Z"); + assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(4_107_542_400)), "2100-03-01T00:00:00.000Z"); + // Before the epoch (a clock set wrong) clamps instead of panicking. + assert_eq!(timestamp(UNIX_EPOCH - Duration::from_secs(5)), "1970-01-01T00:00:00.000Z"); + } + + #[test] + fn rotation_keeps_the_previous_logs_and_starts_an_empty_file() { + let dir = temp_dir("rotate"); + for run in 1..=4 { + let (path, _file) = rotate(&dir).expect("rotate"); + assert_eq!(path, dir.join(LOG_FILE)); + assert_eq!(read(&path), ""); + std::fs::write(&path, format!("run {run}")).expect("write"); + } + assert_eq!(read(&dir.join(LOG_FILE)), "run 4"); + assert_eq!(read(&dir.join("filmcraft.1.log")), "run 3"); + assert_eq!(read(&dir.join("filmcraft.2.log")), "run 2"); + assert!(!dir.join("filmcraft.3.log").exists(), "only {KEEP} old logs are kept"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn rotation_into_an_unusable_directory_is_an_error_not_a_panic() { + let dir = temp_dir("blocked"); + std::fs::create_dir_all(dir.parent().expect("parent")).expect("tmp"); + std::fs::write(&dir, "a file where the directory should be").expect("block"); + assert!(rotate(&dir).is_err()); + assert!(rotate(&dir.join("below")).is_err()); + let _ = std::fs::remove_file(&dir); + } + + #[test] + fn a_line_carries_time_level_thread_target_and_message() { + assert_eq!( + format_line("2026-10-08T07:59:17.728Z", log::Level::Warn, "main", "filmcraft_ui_egui", "audio output unavailable: no output device"), + "2026-10-08T07:59:17.728Z WARN [main] filmcraft_ui_egui: audio output unavailable: no output device\n" + ); + // A multi-line message keeps its lines; the record still ends in exactly one newline. + assert!(format_line("t", log::Level::Error, "w", "x", "a\nb\n").ends_with("x: a\nb\n")); + } + + #[test] + fn lines_logged_before_the_file_exists_are_written_when_it_is_attached() { + let dir = temp_dir("pending"); + let (path, file) = rotate(&dir).expect("rotate"); + let mut sink = Sink::new(MAX_FILE_BYTES); + sink.write("early 1\n"); + sink.write("early 2\n"); + sink.attach(file); + sink.write("late\n"); + assert_eq!(read(&path), "early 1\nearly 2\nlate\n"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn the_pending_buffer_is_bounded_and_says_how_much_it_dropped() { + let dir = temp_dir("bounded"); + let (path, file) = rotate(&dir).expect("rotate"); + let mut sink = Sink::new(MAX_FILE_BYTES); + for i in 0..MAX_PENDING + 5 { + sink.write(&format!("line {i}\n")); + } + sink.attach(file); + let text = read(&path); + assert_eq!(text.lines().filter(|l| l.starts_with("line ")).count(), MAX_PENDING); + assert_eq!(text.matches("5 earlier log lines were dropped").count(), 1, "{text}"); + assert!(text.contains("line 0\n") && !text.contains(&format!("line {MAX_PENDING}\n"))); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn exactly_max_pending_lines_drop_nothing() { + let dir = temp_dir("exact"); + let (path, file) = rotate(&dir).expect("rotate"); + let mut sink = Sink::new(MAX_FILE_BYTES); + for i in 0..MAX_PENDING { + sink.write(&format!("line {i}\n")); + } + sink.attach(file); + let text = read(&path); + assert_eq!(text.lines().count(), MAX_PENDING); + assert!(!text.contains("dropped"), "{text}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn the_file_stops_at_its_size_cap_with_one_note() { + let dir = temp_dir("cap"); + let (path, file) = rotate(&dir).expect("rotate"); + let mut sink = Sink::new(100); + sink.attach(file); + for i in 0..50 { + sink.write(&format!("line {i:02} with some padding\n")); + } + let text = read(&path); + assert_eq!(text.lines().filter(|l| l.starts_with("line ")).count(), 3, "{text}"); + assert_eq!(text.matches("log file reached 100 bytes").count(), 1, "{text}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn the_logger_filters_records_and_writes_them_to_the_attached_file() { + let dir = temp_dir("logger"); + let logger = AppLogger::new(Filter::parse(DEFAULT_FILTER), false); + record(&logger, log::Level::Info, "filmcraft", "FilmCraft starting"); + record(&logger, log::Level::Info, "wgpu_core::device", "too chatty"); + let path = logger.attach_dir(&dir).expect("attach"); + assert_eq!(path, dir.join(LOG_FILE)); + record(&logger, log::Level::Warn, "wgpu_hal::vulkan", "a real warning"); + record(&logger, log::Level::Debug, "filmcraft", "below info"); + let text = read(&path); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!(lines.len(), 2, "{text}"); + assert!(lines.first().is_some_and(|l| l.contains(" INFO [") && l.ends_with("filmcraft: FilmCraft starting")), "{text}"); + assert!(lines.get(1).is_some_and(|l| l.ends_with("wgpu_hal::vulkan: a real warning")), "{text}"); + assert!(logger.enabled(&log::Metadata::builder().level(log::Level::Info).target("filmcraft_engine::autosave").build())); + assert!(!logger.enabled(&log::Metadata::builder().level(log::Level::Info).target("naga").build())); + // A second launch rotates: the first log becomes `.1`. + let second = AppLogger::new(Filter::parse(DEFAULT_FILTER), false); + assert_eq!(second.attach_dir(&dir).expect("attach again"), dir.join(LOG_FILE)); + assert_eq!(read(&dir.join("filmcraft.1.log")), text); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn records_from_many_threads_are_whole_lines() { + let dir = temp_dir("threads"); + let logger = AppLogger::new(Filter::parse("info"), false); + let path = logger.attach_dir(&dir).expect("attach"); + std::thread::scope(|s| { + for _ in 0..4 { + s.spawn(|| { + for _ in 0..100 { + record(&logger, log::Level::Info, "filmcraft", "from a worker"); + } + }); + } + }); + let text = read(&path); + assert_eq!(text.lines().count(), 400); + assert!(text.lines().all(|l| l.ends_with("filmcraft: from a worker")), "{text}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn attaching_to_an_unusable_directory_reports_an_error_and_keeps_logging() { + let dir = temp_dir("unusable"); + std::fs::write(&dir, "not a directory").expect("block"); + let logger = AppLogger::new(Filter::parse("info"), false); + let e = logger.attach_dir(&dir).expect_err("a file is not a log directory"); + assert!(e.contains(&dir.display().to_string()), "{e}"); + record(&logger, log::Level::Error, "x", "still fine"); + let sink = logger.sink.lock().expect("sink"); + assert!(sink.file.is_none() && sink.pending.is_empty(), "nothing is kept for a file that will not come"); + drop(sink); + let _ = std::fs::remove_file(&dir); + } + + #[test] + fn a_poisoned_lock_does_not_stop_logging() { + let dir = temp_dir("poison"); + let logger = AppLogger::new(Filter::parse("info"), false); + let path = logger.attach_dir(&dir).expect("attach"); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = logger.sink.lock(); + panic!("poison the sink"); + })); + assert!(logger.sink.is_poisoned()); + record(&logger, log::Level::Info, "filmcraft", "after the panic"); + assert!(read(&path).contains("after the panic")); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/apps/filmcraft/src/main.rs b/apps/filmcraft/src/main.rs index b9b279a1..a1eef523 100644 --- a/apps/filmcraft/src/main.rs +++ b/apps/filmcraft/src/main.rs @@ -16,6 +16,9 @@ //! with unsaved changes the app asks to recover them; `--recover` recovers the newest without //! asking, `--no-recover` starts without asking (the changes stay available via File ▸ Recover //! Unsaved Changes…). +//! +//! `log` records go to standard error and `/Logs/filmcraft.log` (`RUST_LOG` sets the +//! levels); see [`logging`]. #![cfg_attr(all(target_os = "windows", not(debug_assertions)), windows_subsystem = "windows")] #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] @@ -24,6 +27,7 @@ mod args; mod audio; mod audio_in; mod control_server; +mod logging; #[cfg(target_os = "macos")] mod native_menu; mod window_raise; @@ -66,6 +70,8 @@ fn tell(text: &str, error: bool) { } fn main() -> eframe::Result { + // First, so the panic hook and every start-up record are logged (`logging`). + let logger = logging::install(); // lossy, so a value that is not Unicode is reported like any other bad port instead of ignored let env_port = std::env::var_os("FILMCRAFT_CONTROL_PORT").map(|p| p.to_string_lossy().into_owned()); let Launch { control_port, files, demo, startup_flag, recover, data_dir } = match args::parse(env_port, std::env::args().skip(1)) { @@ -83,10 +89,24 @@ fn main() -> eframe::Result { std::process::exit(2); } }; + let log_dir = data_dir.clone().or_else(default_data_dir).map(|d| d.join(logging::LOG_DIR)); + log::info!("FilmCraft {} ({} {})", env!("CARGO_PKG_VERSION"), std::env::consts::OS, std::env::consts::ARCH); + // The log file lives next to the crash logs; opened after the arguments, so `--help`, + // `--version` and usage errors leave no file behind. Records logged until now go in first. + if let Some(logger) = logger { + match log_dir.as_deref().map(|d| logger.attach_dir(d)) { + Some(Ok(path)) => log::info!("log file {}", path.display()), + Some(Err(e)) => log::warn!("no log file: {e}"), + None => { + logger.stderr_only(); + log::warn!("no log file: no data directory (set FILMCRAFT_DATA_DIR or --data-dir)"); + } + } + } app_nap::disable(); // Panics anywhere go to /Logs/crash-.log with a backtrace; the UI pass and // frame workers catch them and keep running (see filmcraft_ui_egui::crash). - filmcraft_ui_egui::crash::install(data_dir.clone().or_else(default_data_dir).map(|d| d.join("Logs"))); + filmcraft_ui_egui::crash::install(log_dir); // OS hardware video decoders (VideoToolbox on macOS) in front of our own; Settings ▸ Playback ▸ // Hardware decoding switches them off. Unsupported streams and failures use our decoders. register_hardware_decoders(); @@ -116,7 +136,7 @@ fn main() -> eframe::Result { let mut cfg = AutosaveConfig::new(dir); cfg.local_offset = local_offset; if let Err(e) = session.start_autosave(cfg) { - eprintln!("filmcraft: auto-save and crash recovery unavailable: {e}"); + log::warn!("auto-save and crash recovery unavailable: {e}"); } } // voice-over recording reads the microphone through cpal @@ -124,7 +144,7 @@ fn main() -> eframe::Result { let project = files.iter().find(|f| f.ends_with(".fcproj")).cloned(); if let Some(p) = project { if let Err(e) = session.execute("file.open", json!({"path": p})) { - eprintln!("filmcraft: {e}"); + log::error!("{e}"); } } else if !startup_flag && session.prefs.general.at_startup == "openMostRecent" { // Settings ▸ General ▸ At Startup ▸ Open Most Recent @@ -132,7 +152,7 @@ fn main() -> eframe::Result { match recent { Some(p) => { if let Err(e) = session.execute("file.open", json!({"path": p})) { - eprintln!("filmcraft: {e}"); + log::error!("{e}"); } } None => { @@ -150,8 +170,8 @@ fn main() -> eframe::Result { if recover == Some(true) && !session.recovery_candidates().is_empty() { let id = session.recovery_candidates()[0].id.clone(); match session.execute("file.recover", json!({"id": id})) { - Ok(r) => eprintln!("filmcraft: recovered {r}"), - Err(e) => eprintln!("filmcraft: recovery failed: {e}"), + Ok(r) => log::info!("recovered {r}"), + Err(e) => log::error!("recovery failed: {e}"), } } let mut app = FilmcraftApp::new(session); @@ -207,7 +227,7 @@ fn main() -> eframe::Result { // instead of exiting silently, and keep the reason in the crash log. if let Err(e) = &started { let msg = format!("FilmCraft could not start its window: {e}\n\nUpdating the graphics driver usually fixes this."); - eprintln!("filmcraft: {msg}"); + log::error!("{msg}"); filmcraft_ui_egui::crash::record(&msg); let _ = rfd::MessageDialog::new().set_title("FilmCraft").set_description(&msg).set_level(rfd::MessageLevel::Error).show(); } @@ -277,6 +297,9 @@ mod tests { fn startup_registers_the_hardware_decoders_without_a_logger() { assert!(!log::log_enabled!(log::Level::Info)); let hardware = super::register_hardware_decoders(); - assert_eq!(filmcraft_platform::registered(), cfg!(any(target_os = "macos", target_os = "windows")), "{hardware:?}"); + // always on macOS and Windows; on Linux when a VA-API driver is there + let expected = cfg!(any(target_os = "macos", target_os = "windows")) + || (cfg!(target_os = "linux") && matches!(hardware, filmcraft_platform::Availability::Available(_))); + assert_eq!(filmcraft_platform::registered(), expected, "{hardware:?}"); } } diff --git a/contributors/people.toml b/contributors/people.toml index 0fd5e8bf..07aa4a51 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -25,3 +25,13 @@ sync_github_name = false [people.sanyamsingh001] display_name = "sanyamsingh001" sync_github_name = false + +[people.swillunderscore] +display_name = "Swill Software" + + +[people.IppSec] +sync_github_name = true + +[people.ItzBloxy] +display_name = "Bloxy" diff --git a/crates/automation/src/long_job.rs b/crates/automation/src/long_job.rs index d03c4067..8d4035ab 100644 --- a/crates/automation/src/long_job.rs +++ b/crates/automation/src/long_job.rs @@ -107,10 +107,11 @@ impl FilmcraftMcp { } /// Delete what an interrupted export wrote, and nothing else: the output file itself, the numbered -/// stills of an image sequence (`.`) and the caption sidecar +/// stills of an image sequence (`<3+ digits>.`, only when the output is a +/// PNG/TIFF/BMP sequence) and the caption sidecar /// (`.srt` / `.vtt`), each only if it was written since `since`. Other files that -/// merely share the name's stem (`.aep`, `2.psd`, …) are never touched, even when they -/// were saved during the export. +/// merely share the name's stem (`.aep`, `2.psd`, `1.mp4` beside a movie, …) are +/// never touched, even when they were saved during the export. pub fn remove_partial(path: &str, since: SystemTime) { let p = Path::new(path); let (Some(dir), Some(stem), Some(file)) = (p.parent(), p.file_stem().and_then(|s| s.to_str()), p.file_name().and_then(|s| s.to_str())) else { @@ -139,9 +140,19 @@ fn is_export_output(name: &str, file: &str, stem: &str, ext: &str) -> bool { if let Some(sidecar) = rest.strip_prefix('.') { return sidecar.eq_ignore_ascii_case("srt") || sidecar.eq_ignore_ascii_case("vtt"); } - // image sequence: one or more digits, then the output's own extension + // image sequence: the frame number (at least three digits, like `image_sequence_path`), then the + // output's own extension; movies and audio are a single file, so numbered neighbours are not theirs + if !is_sequence_extension(ext) { + return false; + } let digits = rest.len() - rest.trim_start_matches(|c: char| c.is_ascii_digit()).len(); - digits > 0 && !ext.is_empty() && rest.get(digits..).is_some_and(|r| r.strip_prefix('.').is_some_and(|x| x == ext)) + digits >= 3 && rest.get(digits..).is_some_and(|r| r.strip_prefix('.').is_some_and(|x| x == ext)) +} + +/// Whether an export to a file with extension `ext` writes an image sequence (one still per frame). +fn is_sequence_extension(ext: &str) -> bool { + use filmcraft_engine::export::Format; + Format::from_name(ext).is_some_and(|f| matches!(f, Format::PngSequence | Format::TiffSequence | Format::BmpSequence)) } #[cfg(test)] @@ -170,6 +181,50 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + #[test] + fn numbered_files_beside_a_movie_are_kept() { + let dir = std::env::temp_dir().join(format!("fc-remove-partial-movie-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let started = SystemTime::now(); + // a movie export writes one file: `clip1.mp4` and `clip002.mp4` are other exports + let ours = ["clip.mp4", "clip.srt"]; + let theirs = ["clip1.mp4", "clip002.mp4", "clip2.mov"]; + for f in ours.iter().chain(theirs.iter()) { + std::fs::write(dir.join(f), b"x").unwrap(); + } + remove_partial(&dir.join("clip.mp4").to_string_lossy(), started); + for f in ours { + assert!(!dir.join(f).exists(), "{f} should be removed"); + } + for f in theirs { + assert!(dir.join(f).exists(), "{f} must not be removed"); + } + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn sequence_frames_have_at_least_three_digits() { + let dir = std::env::temp_dir().join(format!("fc-remove-partial-digits-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let started = SystemTime::now(); + // frames are numbered with at least three digits (`shot000.tiff`); `shot2.tiff` is not one + let ours = ["shot.tiff", "shot000.tiff", "shot1234.tiff"]; + let theirs = ["shot2.tiff", "shot12.tiff"]; + for f in ours.iter().chain(theirs.iter()) { + std::fs::write(dir.join(f), b"x").unwrap(); + } + remove_partial(&dir.join("shot.tiff").to_string_lossy(), started); + for f in ours { + assert!(!dir.join(f).exists(), "{f} should be removed"); + } + for f in theirs { + assert!(dir.join(f).exists(), "{f} must not be removed"); + } + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn files_older_than_the_export_are_kept() { let dir = std::env::temp_dir().join(format!("fc-remove-partial-old-{}", std::process::id())); diff --git a/crates/codecs/src/audio.rs b/crates/codecs/src/audio.rs index 4942f6a6..3287cc06 100644 --- a/crates/codecs/src/audio.rs +++ b/crates/codecs/src/audio.rs @@ -87,7 +87,7 @@ impl PacketDecoder { /// (HE-AAC, see [`aac_output_rate`]) the decoded core is upsampled to it. pub fn aac(asc: &[u8], sample_rate: u32) -> Result { if let Ok(dec) = filmcraft_aac::Decoder::new(asc) { - let up = (dec.sample_rate().checked_mul(2) == Some(sample_rate)).then(|| Box::new(Upsample2x::new(dec.channels()))); + let up = upsamples_core(dec.sample_rate(), sample_rate).then(|| Box::new(Upsample2x::new(dec.channels()))); return Ok(Self { inner: Inner::Aac { dec: Box::new(dec), asc: asc.to_vec(), up }, channels: 0 }); } Self::new(CODEC_TYPE_AAC, sample_rate, Some(asc.to_vec())) @@ -244,6 +244,77 @@ pub fn aac_output_rate<'a>(asc: &[u8], first_units: impl IntoIterator bool { + core.checked_mul(2) == Some(rate) +} + +/// A codec whose packets all decode to the same number of samples (see [`fixed_packet_samples`]). +#[derive(Clone, Copy, Debug)] +pub enum FixedFrames<'a> { + /// AAC with this AudioSpecificConfig. + Aac(&'a [u8]), + /// MPEG-1/2 audio, layer I, II or III. + MpegAudio, + /// AC-3 (and E-AC-3). + Ac3, +} + +/// Samples each packet decodes to at the stream's output `rate`, for codecs where that is the same +/// for every packet: AAC 1024 (2048 when [`PacketDecoder::aac`] upsamples an HE-AAC core); MPEG +/// audio and AC-3 from the frame header of `first`, the stream's first packet. `None` when it is +/// not known: an AAC configuration our decoder does not read, a header that does not parse, or a +/// header whose rate is not `rate`. +pub fn fixed_packet_samples(codec: FixedFrames, first: &[u8], rate: u32) -> Option { + let mpegts = match codec { + FixedFrames::Aac(asc) => { + let core = filmcraft_aac::Decoder::new(asc).ok()?.sample_rate(); + return Some(if upsamples_core(core, rate) { 2048 } else { 1024 }); + } + FixedFrames::MpegAudio => filmcraft_mpegts::Codec::MpegAudio, + FixedFrames::Ac3 => filmcraft_mpegts::Codec::Ac3, + }; + let f = filmcraft_mpegts::frame_info(&mpegts, first)?; + (f.sample_rate == rate && f.samples > 0).then_some(f.samples as i64) +} + +/// Where the container puts one audio packet, for [`contiguous_starts`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PacketTime { + /// Its timestamp, in output samples. + pub stamp: i64, + /// Whether `stamp` is the packet's own. A Matroska frame after the first in a laced block + /// without a duration repeats the block's timestamp. + pub stamped: bool, + /// Samples it decodes to, when known. + pub samples: Option, +} + +/// Packet start positions in output samples: each packet starts where the previous one ends, +/// unless its own timestamp is more than `tolerance` samples from there (a real gap or overlap in +/// the recording) or the previous packet's length is unknown; then it starts at its timestamp. +/// +/// Container timestamps are coarser than a sample. Matroska stores milliseconds, and a remux to MP4 +/// (OBS's) turns that rounding into uneven sample durations: a 1024-sample AAC frame at 48 kHz lasts +/// 21.33 ms, so frames are stamped at 0, 1008, 2064, 3072… Placing each decoded packet at its +/// stamp overwrote the end of one packet and left a hole of silence after another, a click every +/// few packets: crackling throughout playback and export (#236). +pub fn contiguous_starts(packets: impl IntoIterator, tolerance: i64) -> Vec { + let tolerance = tolerance.max(0).unsigned_abs(); + let mut next: Option = None; + packets + .into_iter() + .map(|p| { + let start = match next { + Some(n) if !p.stamped || n.abs_diff(p.stamp) <= tolerance => n, + _ => p.stamp, + }; + next = p.samples.and_then(|k| start.checked_add(k)); + start + }) + .collect() +} + /// Taps on each side of the interpolation point in [`Upsample2x`]. const UPSAMPLE_HALF: usize = 12; @@ -739,3 +810,82 @@ mod he_aac_tests { } } } + +#[cfg(test)] +mod packet_time_tests { + use super::*; + + /// 1024-sample frames at 48 kHz stamped to the nearest millisecond, then back in samples. + fn ms_stamps(count: i64) -> Vec { + (0..count).map(|k| (k * 1024 * 1000 + 24_000) / 48_000 * 48).collect() + } + + fn timed(stamps: &[i64], samples: Option) -> Vec { + stamps.iter().map(|&stamp| PacketTime { stamp, stamped: true, samples }).collect() + } + + #[test] + fn rounded_stamps_give_back_to_back_packets() { + let stamps = ms_stamps(200); + assert_eq!(&stamps[..4], [0, 1008, 2064, 3072]); + let starts = contiguous_starts(timed(&stamps, Some(1024)), 512); + assert_eq!(starts, (0..200).map(|k| k * 1024).collect::>()); + } + + #[test] + fn real_gaps_and_overlaps_resynchronise() { + // a 100 ms gap after packet 3, then a packet stamped 600 samples early + let mut stamps: Vec = (0..8).map(|k| k * 1024).collect(); + for s in &mut stamps[4..] { + *s += 4800; + } + stamps[7] -= 600; + let starts = contiguous_starts(timed(&stamps, Some(1024)), 512); + assert_eq!(starts, [0, 1024, 2048, 3072, 8896, 9920, 10944, 11368]); + // within the tolerance: runs on + stamps[7] += 200; + assert_eq!(contiguous_starts(timed(&stamps, Some(1024)), 512)[7], 11968); + } + + #[test] + fn unstamped_packets_follow_on_and_unknown_lengths_resynchronise() { + // a laced block of three frames that repeat its timestamp + let packets = [0, 1024, 1024, 1024, 4096].map(|stamp| PacketTime { stamp, stamped: true, samples: Some(1024) }); + let mut laced = packets; + laced[2].stamped = false; + laced[3].stamped = false; + assert_eq!(contiguous_starts(laced, 512), [0, 1024, 2048, 3072, 4096]); + // a packet whose length is unknown: the next one starts at its own stamp + let mut unknown = timed(&[0, 1024, 2100, 3124], Some(1024)); + unknown[1].samples = None; + assert_eq!(contiguous_starts(unknown, 512), [0, 1024, 2100, 3124]); + } + + #[test] + fn hostile_values_stay_finite() { + let packets = [ + PacketTime { stamp: i64::MAX, stamped: true, samples: Some(i64::MAX) }, + PacketTime { stamp: i64::MIN, stamped: false, samples: Some(-5) }, + PacketTime { stamp: 0, stamped: true, samples: Some(i64::MIN) }, + PacketTime { stamp: 7, stamped: true, samples: None }, + ]; + assert_eq!(contiguous_starts(packets, i64::MIN).len(), 4); + assert_eq!(contiguous_starts(packets, i64::MAX).len(), 4); + assert!(contiguous_starts([], 512).is_empty()); + } + + #[test] + fn fixed_frame_lengths() { + // AAC-LC 48 kHz stereo; HE-AAC (24 kHz core) played at 48 kHz is upsampled + assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[0x11, 0x90]), &[], 48_000), Some(1024)); + assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[0x13, 0x10]), &[], 48_000), Some(2048)); + assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[]), &[], 48_000), None); + // MPEG-1 layer III 44.1 kHz, 128 kbps; MPEG-2 layer III 22.05 kHz + assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xFB, 0x90, 0x44], 44_100), Some(1152)); + assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xF3, 0x90, 0x44], 22_050), Some(576)); + // the header's rate must be the stream's + assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xFB, 0x90, 0x44], 48_000), None); + assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF], 44_100), None); + assert_eq!(fixed_packet_samples(FixedFrames::Ac3, &[0x0B, 0x77], 48_000), None); + } +} diff --git a/crates/codecs/src/audio_timing_tests.rs b/crates/codecs/src/audio_timing_tests.rs new file mode 100644 index 00000000..40e43a2d --- /dev/null +++ b/crates/codecs/src/audio_timing_tests.rs @@ -0,0 +1,178 @@ +//! Audio packets placed by container timestamps (#236). +//! +//! An AAC access unit decodes to 1024 samples, 21.33 ms at 48 kHz. Matroska stores timestamps in +//! whole milliseconds, and OBS's remux to MP4 carries that rounding into sample durations of 1008, +//! 1008, 1056… Placing each decoded packet at its own timestamp overwrote the end of one packet and +//! left a hole of silence after another: playback and export crackled throughout. The streams here +//! are AAC from our own encoder muxed with such timestamps; read back block by block, as playback +//! reads them, they must be exactly the access units decoded back to back. + +use std::io::Cursor; +use std::sync::Arc; + +use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; +use filmcraft_matroska::{LacingMode, MkvWriter, MuxOptions, TrackKind, TrackSpec}; +use filmcraft_media::MediaSource; + +use crate::audio::PacketDecoder; +use crate::{MkvSource, Mp4Source}; + +/// A second of a stereo tone (440 Hz left, 660 Hz right) encoded with our AAC encoder: the +/// AudioSpecificConfig and the access units. +fn aac(rate: u32) -> (Vec, Vec>) { + let tone = |hz: f32| (0..rate as usize).map(|i| 0.5 * (std::f32::consts::TAU * hz * i as f32 / rate as f32).sin()).collect::>(); + let (l, r) = (tone(440.0), tone(660.0)); + let mut enc = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(rate, 2, 160_000)).unwrap(); + let mut units = enc.encode(&[&l, &r]); + units.extend(enc.flush()); + (enc.audio_specific_config(), units) +} + +/// The access units decoded back to back. +fn continuous(asc: &[u8], rate: u32, units: &[Vec]) -> Vec> { + let mut dec = PacketDecoder::aac(asc, rate).unwrap(); + let mut out = vec![Vec::new(), Vec::new()]; + for u in units { + for (o, c) in out.iter_mut().zip(dec.decode(u, 0).unwrap()) { + o.extend(c); + } + } + out +} + +/// Access unit `k`'s start to the nearest millisecond, as muxers stamp it. +fn stamp_ms(rate: u32, k: usize) -> i64 { + (k as i64 * 1024 * 2000 + rate as i64) / (2 * rate as i64) +} + +/// Matroska (1 ms timestamps), with the units after `gap` delayed by `gap_ms` (a real gap). +fn mkv(asc: &[u8], rate: u32, units: &[Vec], lacing: LacingMode, gap: usize, gap_ms: i64) -> Arc<[u8]> { + let mut spec = TrackSpec::new(TrackKind::Audio, "A_AAC"); + spec.codec_private = asc.to_vec(); + spec.audio = Some((rate as f64, 2, None)); + let opts = MuxOptions { lacing, ..MuxOptions::default() }; + assert_eq!(opts.timestamp_scale, 1_000_000, "1 ms timestamps"); + let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![spec], opts).unwrap(); + for (k, u) in units.iter().enumerate() { + let ms = stamp_ms(rate, k) + if k >= gap { gap_ms } else { 0 }; + w.write_frame(0, ms * 1_000_000, true, u, None).unwrap(); + } + w.finish().unwrap().into_inner().into() +} + +/// MP4 as OBS remuxes Matroska: each millisecond timestamp rescaled to the sample rate, the +/// differences written as sample durations. +fn mp4_remuxed(asc: &[u8], rate: u32, units: &[Vec]) -> Arc<[u8]> { + let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc.to_vec(), 2, rate), rate)).unwrap(); + let at = |k: usize| (stamp_ms(rate, k) * rate as i64 + 500) / 1000; + for (k, u) in units.iter().enumerate() { + let duration = (at(k + 1) - at(k)) as u32; + w.write_sample(t, WriteSample { data: u, duration, composition_offset: 0, is_sync: true }).unwrap(); + } + w.finish().unwrap().into_inner().into() +} + +/// `frames` samples per channel read in 512-sample blocks at `rate`, as playback reads them. +fn played(src: &dyn MediaSource, rate: u32, frames: usize) -> Vec> { + let mut out = vec![Vec::new(), Vec::new()]; + let mut pos = 0; + while pos < frames { + let n = 512.min(frames - pos); + let b = src.audio(pos as i64, n, rate).unwrap(); + for (o, c) in out.iter_mut().zip(&b.channels) { + o.extend_from_slice(c); + } + pos += n; + } + out +} + +/// Samples of `got` that differ from `want[k - shift]` (silence where that is out of range), and +/// the first few. +fn mismatches(got: &[Vec], want: &[Vec], shift: usize) -> (usize, Vec<(usize, usize)>) { + let mut bad = Vec::new(); + for (c, (g, w)) in got.iter().zip(want).enumerate() { + for (k, x) in g.iter().enumerate() { + let expect = k.checked_sub(shift).and_then(|j| w.get(j)).copied().unwrap_or(0.0); + if x.to_bits() != expect.to_bits() { + bad.push((c, k)); + } + } + } + (bad.len(), bad.into_iter().take(4).collect()) +} + +/// Runs of two or more exact zeros inside the signal: holes left between packets. +fn holes(got: &[Vec]) -> usize { + got.iter().map(|c| c.windows(2).filter(|w| w[0] == 0.0 && w[1] == 0.0).count()).sum() +} + +#[test] +fn matroska_millisecond_stamps_play_back_to_back() { + for rate in [48_000, 44_100] { + let (asc, units) = aac(rate); + let want = continuous(&asc, rate, &units); + let frames = units.len().saturating_sub(1) * 1024; + for lacing in [LacingMode::None, LacingMode::Xiph] { + let bytes = mkv(&asc, rate, &units, lacing, usize::MAX, 0); + let src = MkvSource::open("tone.mkv", bytes.clone()).unwrap(); + let got = played(&src, rate, frames); + let (n, first) = mismatches(&got, &want, 0); + assert_eq!(n, 0, "{rate} Hz, {lacing:?}: {n} samples differ from the continuous decode, first (channel, sample) {first:?}"); + // random access (a fresh source, so a cold decoder primed with the packet before) gives + // the same samples + let src = MkvSource::open("tone.mkv", bytes).unwrap(); + for start in [30_011usize, 7_000] { + let got = src.audio(start as i64, 4_800, rate).unwrap().channels; + let at: Vec> = want.iter().map(|c| c[start..start + 4_800].to_vec()).collect(); + assert_eq!(mismatches(&got, &at, 0).0, 0, "{rate} Hz, {lacing:?}: seek to {start}"); + } + } + } +} + +#[test] +fn obs_remuxed_mp4_plays_back_to_back() { + for rate in [48_000, 44_100] { + let (asc, units) = aac(rate); + let want = continuous(&asc, rate, &units); + let bytes = mp4_remuxed(&asc, rate, &units); + let src = Mp4Source::open("tone.mp4", bytes).unwrap(); + let got = played(&src, rate, units.len().saturating_sub(1) * 1024); + let (n, first) = mismatches(&got, &want, 0); + assert_eq!(n, 0, "{rate} Hz: {n} samples differ from the continuous decode, first (channel, sample) {first:?}"); + } +} + +#[test] +fn a_real_gap_stays_where_the_timestamps_put_it() { + let rate = 48_000; + let (asc, units) = aac(rate); + let want = continuous(&asc, rate, &units); + // 100 ms of the recording missing before unit 20: it plays as silence, and the units after it + // keep their timestamps (to the millisecond) + let src = MkvSource::open("gap.mkv", mkv(&asc, rate, &units, LacingMode::None, 20, 100)).unwrap(); + let shift = ((stamp_ms(rate, 20) + 100) * 48 - 20 * 1024) as usize; + assert!((4_700..4_900).contains(&shift), "{shift}"); + let got = played(&src, rate, units.len().saturating_sub(1) * 1024 + shift); + let before: Vec> = got.iter().map(|c| c[..20 * 1024].to_vec()).collect(); + assert_eq!(mismatches(&before, &want, 0).0, 0); + assert!(got.iter().all(|c| c[20 * 1024..20 * 1024 + shift].iter().all(|x| *x == 0.0)), "the gap is silent"); + let after: Vec> = got.iter().map(|c| c[20 * 1024 + shift..].to_vec()).collect(); + let rest: Vec> = want.iter().map(|c| c[20 * 1024..].to_vec()).collect(); + let (n, first) = mismatches(&after, &rest, 0); + assert_eq!(n, 0, "{n} samples after the gap differ, first {first:?}"); +} + +#[test] +fn playing_at_another_rate_leaves_no_holes() { + // a 44.1 kHz recording in a 48 kHz sequence (export's default rate) + let (asc, units) = aac(44_100); + let src = MkvSource::open("tone.mkv", mkv(&asc, 44_100, &units, LacingMode::None, usize::MAX, 0)).unwrap(); + let frames = (units.len().saturating_sub(2) * 1024) * 48_000 / 44_100; + let got = played(&src, 48_000, frames); + // skip the encoder's priming (the first frame decodes to near-silence) + let tail: Vec> = got.iter().map(|c| c[2048..].to_vec()).collect(); + assert_eq!(holes(&tail), 0); +} diff --git a/crates/codecs/src/gop.rs b/crates/codecs/src/gop.rs index b1ddd9a2..3972998e 100644 --- a/crates/codecs/src/gop.rs +++ b/crates/codecs/src/gop.rs @@ -140,6 +140,10 @@ struct State { bytes: usize, /// When a frame was last asked of this cache. last_used: Instant, + /// Only background requests (thumbnails, [`filmcraft_media::cancel::with_background`]) have + /// used the decoder since it was made: it is given up as soon as another background request + /// is done, whatever the caches' recency. + background: bool, } impl State { @@ -212,10 +216,27 @@ impl Drop for DecodingGuard { /// new one when the playhead reaches them, which costs what a seek costs. const MAX_LIVE_DECODERS: usize = 4; -/// A cache asked for a frame this recently is in use (a layer of the frame being composited, -/// the next clip being prefetched): it keeps its decoder whatever the count. +/// A cache asked for a frame this recently may be needed again soon (the next clip being +/// prefetched, a clip the user scrubs back to): it keeps its decoder up to [`HARD_DECODERS`], and +/// beyond [`MAX_LIVE_DECODERS`] only until it has been idle this long. Its frames are only kept +/// within [`FRAME_BUDGET`] once it has been idle for [`RECENT`]: going back to a clip left (or +/// paused) more than that long ago may cost a re-decode from its keyframe when other clips have +/// filled the budget meanwhile. Frames go least recently used cache first, earliest frames first. const IDLE: Duration = Duration::from_secs(3); +/// A cache asked for a frame this recently is in use (a layer of the frame being composited): it +/// keeps its decoder and frames whatever the count. Caches idle for less than [`IDLE`] but longer +/// than this are a burst of navigation (thumbnails of every clip, the playhead jumping from clip +/// to clip) that the playhead has already left: they give up decoders beyond [`HARD_DECODERS`] +/// and frames beyond [`FRAME_BUDGET`]. Without this, everything touched within [`IDLE`] counted +/// as in use, so a burst held every clip's decoder (42 live hardware sessions, 6.5 GB of VRAM) +/// and hundreds of MB of frames per clip. +const RECENT: Duration = Duration::from_millis(500); + +/// Most decoders caches idle for [`RECENT`] to [`IDLE`] keep together (a composite of a few +/// layers plus the clips just left behind). +const HARD_DECODERS: usize = 8; + /// Decoded frames all caches may hold together before idle ones give theirs up. Each cache also /// has its own budget, which alone let every clip on a timeline keep hundreds of MB of frames /// nobody was looking at. @@ -228,6 +249,10 @@ struct Pool { decoders: AtomicUsize, max_decoders: usize, idle: Duration, + /// Caches idle at least this long are no longer in use (see [`RECENT`]). + recent: Duration, + /// Decoders the caches idle for [`Self::recent`] but not [`Self::idle`] may keep. + hard_decoders: usize, /// Bytes of the frames the caches hold. bytes: AtomicUsize, budget: usize, @@ -258,13 +283,29 @@ fn try_state(c: &Shared) -> Option> { impl Pool { fn new(max_decoders: usize, idle: Duration, budget: usize) -> Self { - Self { caches: Mutex::new(Vec::new()), decoders: AtomicUsize::new(0), max_decoders, idle, bytes: AtomicUsize::new(0), budget } + Self { + caches: Mutex::new(Vec::new()), + decoders: AtomicUsize::new(0), + max_decoders, + idle, + recent: idle, + hard_decoders: max_decoders, + bytes: AtomicUsize::new(0), + budget, + } + } + + /// Caches idle for `recent` (< the idle time) are still trimmed, down to `hard_decoders`. + fn tiered(mut self, recent: Duration, hard_decoders: usize) -> Self { + self.recent = recent.min(self.idle); + self.hard_decoders = hard_decoders.max(self.max_decoders); + self } /// The pool of every cache in the process. fn global() -> Arc { static POOL: std::sync::OnceLock> = std::sync::OnceLock::new(); - POOL.get_or_init(|| Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET))).clone() + POOL.get_or_init(|| Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS))).clone() } fn register(&self, c: &Arc) { @@ -273,15 +314,15 @@ impl Pool { g.push(Arc::downgrade(c)); } - /// The idle caches other than `me` that `keep` selects, least recently used first. Other + /// The caches other than `me` idle for at least `idle` that `keep` selects, least recently used first. Other /// caches are only ever try-locked, so two caches trimming each other cannot deadlock. - fn idle_others(&self, me: &Shared, keep: impl Fn(&State) -> bool) -> Vec> { + fn idle_others(&self, me: &Shared, idle: Duration, keep: impl Fn(&State) -> bool) -> Vec> { let all: Vec> = self.caches.lock().unwrap_or_else(PoisonError::into_inner).iter().filter_map(Weak::upgrade).collect(); let mut idle: Vec<(Instant, Arc)> = all .into_iter() .filter(|c| !std::ptr::eq(Arc::as_ptr(c), me)) .filter_map(|c| { - let used = try_state(&c).filter(|st| keep(st) && st.last_used.elapsed() >= self.idle).map(|st| st.last_used)?; + let used = try_state(&c).filter(|st| keep(st) && st.last_used.elapsed() >= idle).map(|st| st.last_used)?; Some((used, c)) }) .collect(); @@ -295,29 +336,53 @@ impl Pool { #[must_use = "drop the released decoders with no cache lock held"] fn trim_decoders(&self, me: &Shared) -> Vec> { let mut released = Vec::new(); - if self.decoders.load(Ordering::Relaxed) <= self.max_decoders { - return released; - } - for c in self.idle_others(me, |st| st.decoder.is_some()) { - if self.decoders.load(Ordering::Relaxed) <= self.max_decoders { - break; + // first the caches idle for long, down to the cap; then those a burst has left behind + for (idle, cap) in [(self.idle, self.max_decoders), (self.recent, self.hard_decoders)] { + if self.decoders.load(Ordering::Relaxed) <= cap { + continue; } - if let Some(mut st) = try_state(&c) { + for c in self.idle_others(me, idle, |st| st.decoder.is_some()) { + if self.decoders.load(Ordering::Relaxed) <= cap { + break; + } + if let Some(mut st) = try_state(&c) { + released.extend(st.release_decoder(self)); + } + } + } + released + } + + /// Take the decoders of the caches other than `me` that only background requests have used + /// (see [`State::background`]) and that are not decoding now. A burst of thumbnails (an import + /// of many clips) touches every clip within moments, so the recency rules treat them all as + /// in use; each software 4K decoder holds ~0.5 GB. The cache `me` just served keeps its + /// decoder (hovering a clip asks for more of its frames). Drop the result with no cache lock + /// held. + #[must_use = "drop the released decoders with no cache lock held"] + fn release_background(&self, me: &Shared) -> Vec> { + let all: Vec> = self.caches.lock().unwrap_or_else(PoisonError::into_inner).iter().filter_map(Weak::upgrade).collect(); + let mut released = Vec::new(); + for c in all.iter().filter(|c| !std::ptr::eq(Arc::as_ptr(c), me)) { + if let Some(mut st) = try_state(c) + && st.background + && st.decoder.is_some() + { released.extend(st.release_decoder(self)); } } released } - /// Evict the frames of idle caches, least recently used first, down to the budget. Caches in - /// use keep theirs (each within its own budget): a frame evicted before it is shown costs a + /// Evict the frames of caches idle for [`RECENT`], least recently used first, down to the + /// budget. Caches in use keep theirs (each within its own budget): a frame evicted before it is shown costs a /// re-decode from the keyframe. fn trim_frames(&self, me: &Shared) { let over = || self.bytes.load(Ordering::Relaxed) > self.budget; if !over() { return; } - for c in self.idle_others(me, |st| !st.frames.is_empty()) { + for c in self.idle_others(me, self.recent, |st| !st.frames.is_empty()) { if !over() { break; } @@ -361,7 +426,7 @@ const CONTINUE_THROUGH: usize = 48; impl GopCache { pub fn new(explicit_color: Option) -> Self { // unit tests count decoder restarts: each of their caches gets a pool of its own - let pool = if cfg!(test) { Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET)) } else { Pool::global() }; + let pool = if cfg!(test) { Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS)) } else { Pool::global() }; Self::in_pool(explicit_color, pool) } @@ -378,6 +443,7 @@ impl GopCache { drafts: Default::default(), bytes: 0, last_used: Instant::now(), + background: false, }), pool: pool.clone(), }); @@ -405,6 +471,7 @@ impl GopCache { fn store(&self, st: &mut State, pts: i64, f: VideoFrame, draft: bool) { let f = self.finish(f); + st.last_used = Instant::now(); if draft { st.drafts.insert(pts); DRAFT.fetch_add(1, Ordering::Relaxed); @@ -467,6 +534,16 @@ impl GopCache { /// ([`VideoDecoder::set_draft`]); its draft frames are cached for draft requests only, and an /// exact request for one re-decodes it. pub fn frame_late(&self, s: &dyn VideoSamples, target: i64, late_before: Option) -> crate::Result> { + let r = self.frame_late_locked(s, target, late_before); + if filmcraft_media::cancel::background() { + // Outside the cache's lock (see `release_background`). A thumbnail leaves the decoders + // of earlier thumbnails no reason to stay: each costs hundreds of MB at 4K. + drop(self.shared.pool.release_background(&self.shared)); + } + r + } + + fn frame_late_locked(&self, s: &dyn VideoSamples, target: i64, late_before: Option) -> crate::Result> { let n = s.count(); let i = s.sample_at(target.max(0)).or_else(|| (n > 0).then(|| n - 1)).ok_or_else(|| CodecError::Decode("empty track".into()))?; let want_pts = s.pts(i); @@ -482,11 +559,23 @@ impl GopCache { let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner()); let _decoding = DecodingGuard::enter(me); st.last_used = Instant::now(); - if let Some(f) = st.cached(want_pts, draft) { + let background = filmcraft_media::cancel::background(); + if let Some(f) = st.cached(want_pts, draft).cloned() { HITS.fetch_add(1, Ordering::Relaxed); - return Ok(f.clone()); + // a monitor asking for a frame of a thumbnail's clip makes its decoder one in use + st.background &= background; + return Ok(f); } MISSES.fetch_add(1, Ordering::Relaxed); + if !background { + st.background = false; + } else if st.decoder.is_none() { + st.background = true; + } + if !background && st.decoder.as_ref().is_some_and(|d| d.thread_limited()) { + // made for a thumbnail (few threads): a monitor gets a full one + released.extend(st.release_decoder(&self.shared.pool)); + } if st.decoder.is_none() { let d = s.make_decoder()?; st.intra = d.intra_only(); @@ -683,6 +772,8 @@ mod tests { /// (pts, decoded as a draft picture) held: VecDeque<(i64, bool)>, draft: bool, + /// made for background work (few threads, like the H.264 decoder's) + limited: bool, resets: Arc, decodes: Arc, } @@ -717,6 +808,9 @@ mod tests { fn name(&self) -> &str { "test" } + fn thread_limited(&self) -> bool { + self.limited + } fn intra_only(&self) -> bool { self.intra } @@ -749,6 +843,7 @@ mod tests { intra: self.intra, held: VecDeque::new(), draft: false, + limited: filmcraft_media::cancel::background(), resets: self.resets.clone(), decodes: self.decodes.clone(), })) @@ -944,6 +1039,99 @@ mod tests { assert!(s.iter().all(|s| s.resets.load(Ordering::Relaxed) == 1), "no decoder restarted"); } + /// A burst of navigation (every clip of a timeline asked once within moments, as when + /// thumbnails are made or the playhead jumps clip to clip) must not keep every clip's decoder + /// and frames: only the caches of the frame being composited are in use. + #[test] + fn a_burst_of_single_requests_does_not_keep_every_decoder_and_frame() { + // the real decoder caps and idle time; RECENT scaled down from 500 ms to the test's pace + // (3 ms per clip). With the old rule (recent = idle) all 24 decoders stay live. + let pool = Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(Duration::from_millis(2), HARD_DECODERS)); + let n = 24; + let s: Vec = (0..n).map(|_| samples(300, 250, 3, false)).collect(); + let c: Vec = (0..n).map(|_| GopCache::in_pool(None, pool.clone())).collect(); + for k in 0..n { + assert_eq!(index_of(&c[k].frame(&s[k], 10_000).expect("frame")), 10); + std::thread::sleep(Duration::from_millis(3)); + } + assert!( + pool.decoders.load(Ordering::Relaxed) <= HARD_DECODERS + 1, + "{} live decoders after a burst over {n} clips", + pool.decoders.load(Ordering::Relaxed) + ); + // the clip asked last, and a clip in use together with it, keep theirs and carry on + let resets = s[n - 1].resets.load(Ordering::Relaxed); + assert_eq!(index_of(&c[n - 1].frame(&s[n - 1], 11_000).expect("frame")), 11); + assert_eq!(s[n - 1].resets.load(Ordering::Relaxed), resets); + } + + /// The thumbnails an import queues: every clip asked once, quickly (the real caps and recency, + /// no sleeping: all of them are "recent"). Each 4K software decoder costs ~0.5 GB, and 42 + /// clips used to hold 6+ GB at once. + #[test] + fn background_requests_leave_no_decoders_behind() { + use filmcraft_media::cancel::with_background; + let pool = Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS)); + let n = 24; + let s: Vec = (0..n).map(|_| samples(300, 250, 3, false)).collect(); + let c: Vec = (0..n).map(|_| GopCache::in_pool(None, pool.clone())).collect(); + // a clip in use by a monitor + assert_eq!(index_of(&c[0].frame(&s[0], 10_000).expect("frame")), 10); + for k in 1..n { + assert_eq!(index_of(&with_background(true, || c[k].frame(&s[k], 10_000)).expect("frame")), 10); + assert!(pool.decoders.load(Ordering::Relaxed) <= 2, "{} live decoders after thumbnail {k}", pool.decoders.load(Ordering::Relaxed)); + } + // the monitor's clip and the last thumbnail keep theirs and carry on without a seek + assert_eq!(pool.decoders.load(Ordering::Relaxed), 2); + assert_eq!(index_of(&c[0].frame(&s[0], 11_000).expect("frame")), 11); + assert_eq!(s[0].resets.load(Ordering::Relaxed), 1); + assert_eq!(index_of(&with_background(true, || c[n - 1].frame(&s[n - 1], 12_000)).expect("frame")), 12); + assert_eq!(s[n - 1].resets.load(Ordering::Relaxed), 1); + // a monitor asking for a frame of a thumbnail's clip makes its decoder one in use + assert_eq!(index_of(&c[n - 1].frame(&s[n - 1], 13_000).expect("frame")), 13); + for k in 1..4 { + with_background(true, || c[k].frame(&s[k], 20_000)).expect("frame"); + } + assert_eq!(index_of(&c[n - 1].frame(&s[n - 1], 14_000).expect("frame")), 14); + assert!(c[n - 1].shared.state.lock().unwrap().decoder.is_some(), "the thumbnails left its decoder"); + // a cache that lost its decoder still decodes (from the sync sample) + assert_eq!(index_of(&with_background(true, || c[1].frame(&s[1], 30_000)).expect("frame")), 30); + } + + #[test] + fn a_monitor_replaces_a_decoder_made_for_a_thumbnail() { + use filmcraft_media::cancel::with_background; + let pool = Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS)); + let (s, c) = (samples(300, 250, 3, false), GopCache::in_pool(None, pool.clone())); + assert_eq!(index_of(&with_background(true, || c.frame(&s, 10_000)).expect("frame")), 10); + // another thumbnail of the same clip carries on with the thumbnail decoder + assert_eq!(index_of(&with_background(true, || c.frame(&s, 12_000)).expect("frame")), 12); + assert_eq!(s.resets.load(Ordering::Relaxed), 1); + // a frame a monitor waits for (not cached: far ahead) is decoded by a full-thread decoder + assert_eq!(index_of(&c.frame(&s, 100_000).expect("frame")), 100); + assert!(!c.shared.state.lock().unwrap().decoder.as_ref().is_some_and(|d| d.thread_limited())); + assert_eq!(pool.decoders.load(Ordering::Relaxed), 1); + // and keeps it for the next frames + let resets = s.resets.load(Ordering::Relaxed); + assert_eq!(index_of(&c.frame(&s, 101_000).expect("frame")), 101); + assert_eq!(s.resets.load(Ordering::Relaxed), resets); + } + + #[test] + fn a_burst_of_single_requests_does_not_keep_every_clips_frames() { + // 1×1 RGBA pictures (4 bytes): a shared budget of ten of them + let pool = Arc::new(Pool::new(usize::MAX, Duration::from_secs(3600), 40).tiered(Duration::from_millis(2), usize::MAX)); + let n = 12; + let s: Vec = (0..n).map(|_| samples(300, 250, 0, false)).collect(); + let c: Vec = (0..n).map(|_| GopCache::in_pool(None, pool.clone())).collect(); + for k in 0..n { + assert_eq!(index_of(&c[k].frame(&s[k], 10_000).expect("frame")), 10); + std::thread::sleep(Duration::from_millis(5)); + } + let held = pool.bytes.load(Ordering::Relaxed); + assert!(held <= 40 + 11 * 4, "{held} bytes held after a burst over {n} clips"); + } + #[test] fn intra_caches_give_up_their_spare_decoders_too() { let pool = Arc::new(Pool::new(1, Duration::ZERO, FRAME_BUDGET)); diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs index a14f74d3..fb405cd7 100644 --- a/crates/codecs/src/lib.rs +++ b/crates/codecs/src/lib.rs @@ -173,6 +173,8 @@ pub fn open_bytes(name: &str, bytes: Arc<[u8]>) -> std::result::Result Vec { + use crate::audio::{FixedFrames, PacketTime, contiguous_starts, fixed_packet_samples}; + let Some(t) = file.tracks.get(ti) else { return Vec::new() }; + if let Some(h) = opus_head(&t.codec) { + return opus_starts(file, bytes, ti, &h, rate); + } + let (n, d) = tb(t); + let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64; + let first = || file.read_sample(bytes, ti, 0).unwrap_or_default(); + let out_rate = u32::try_from(rate).unwrap_or(0); + let frame = match &t.codec { + Codec::Aac { asc } => fixed_packet_samples(FixedFrames::Aac(asc), &[], out_rate), + Codec::Mp3 | Codec::Mp2 => fixed_packet_samples(FixedFrames::MpegAudio, &first(), out_rate), + Codec::Ac3 => fixed_packet_samples(FixedFrames::Ac3, &first(), out_rate), + _ => None, + }; + let Some(frame) = frame else { + return t.samples.iter().map(|s| at(s.pts)).collect(); + }; + let packets = t.samples.iter().map(|s| PacketTime { stamp: at(s.pts), stamped: own_stamp(s), samples: Some(frame) }); + contiguous_starts(packets, (frame / 2).max(stamp_tolerance(t, rate))) +} + /// Sample-exact Opus packet start positions (48 kHz frames, pre-skip removed). /// -/// Block timestamps are quantised to `TimestampScale` (usually 1 ms) and the demuxer subtracts a -/// rounded `CodecDelay`, so they are only accurate to a tick. Starts are therefore accumulated from -/// each packet's TOC duration, resynchronising to the timestamp only across real gaps (more than -/// two ticks off). Pre-skip is the exact `CodecDelay` (or the header's pre-skip when it is absent). +/// The demuxer also subtracts a rounded `CodecDelay` from the timestamps. Starts are accumulated +/// from each packet's TOC duration, resynchronising to the timestamp only across real gaps (more +/// than two ticks off). Pre-skip is the exact `CodecDelay` (or the header's pre-skip when it is +/// absent). fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_opus::OpusHead, rate: i64) -> Vec { - let t = &file.tracks[ti]; + let Some(t) = file.tracks.get(ti) else { return Vec::new() }; let (n, d) = tb(t); let scale_ns = (n as i128 * 1_000_000_000 / d as i128).max(1); let delay_ns = t.codec_delay_ns as i128; // The demuxer's rounding of CodecDelay to ticks (half away from zero), undone here. let delay_ticks = ((delay_ns + scale_ns / 2) / scale_ns) as i64; let skip = if t.codec_delay_ns > 0 { (delay_ns * rate as i128 / 1_000_000_000) as i64 } else { head.pre_skip as i64 }; - let at = |pts: i64| ((pts + delay_ticks) as i128 * n as i128 * rate as i128 / d as i128) as i64 - skip; - let tolerance = (2 * scale_ns * rate as i128 / 1_000_000_000) as i64 + 1; - let mut starts = Vec::with_capacity(t.samples.len()); - let mut next: Option = None; - for (i, s) in t.samples.iter().enumerate() { - let stamped = at(s.pts); - let start = match next { - Some(p) if (p - stamped).abs() <= tolerance => p, - _ => stamped, - }; - starts.push(start); - let dur = file.read_sample(bytes, ti, i).ok().and_then(|p| crate::audio::opus_packet_samples(&p)); - next = dur.map(|k| start + k as i64); - } - starts + let at = |pts: i64| ((pts.saturating_add(delay_ticks) as i128 * n as i128 * rate as i128 / d as i128) as i64).saturating_sub(skip); + let packets = t.samples.iter().enumerate().map(|(i, s)| crate::audio::PacketTime { + stamp: at(s.pts), + stamped: own_stamp(s), + samples: file.read_sample(bytes, ti, i).ok().and_then(|p| crate::audio::opus_packet_samples(&p)).map(|k| k as i64), + }); + crate::audio::contiguous_starts(packets, stamp_tolerance(t, rate)) +} + +/// Two timestamp ticks in sample frames at `rate`, plus one: how far a packet's Matroska timestamp +/// can be from its exact start. +fn stamp_tolerance(t: &filmcraft_matroska::Track, rate: i64) -> i64 { + let (n, d) = tb(t); + i64::try_from(2 * n as i128 * rate as i128 / d as i128).unwrap_or(i64::MAX).saturating_add(1) +} + +/// Whether a sample's timestamp is its own: frames after the first in a laced block without a +/// duration repeat the block's timestamp. +fn own_stamp(s: &filmcraft_matroska::Sample) -> bool { + s.lace == 0 || s.duration > 0 } /// The parsed `OpusHead` of an `A_OPUS` track. @@ -311,18 +350,7 @@ impl MkvSource { start_timecode: None, file_size: Some(bytes.0.len()), }; - let audio_starts = atrack - .map(|i| { - let t = &file.tracks[i]; - let rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64; - let (n, d) = tb(t); - let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64; - match opus_head(&t.codec) { - Some(h) => opus_starts(&file, &bytes, i, &h, rate), - None => t.samples.iter().map(|s| at(s.pts)).collect(), - } - }) - .unwrap_or_default(); + let audio_starts = atrack.map(|i| audio_starts(&file, &bytes, i, info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64)).unwrap_or_default(); let audio_preroll = atrack .map(|i| { let t = &file.tracks[i]; diff --git a/crates/codecs/src/mp4.rs b/crates/codecs/src/mp4.rs index 2fedf3be..2cdcad31 100644 --- a/crates/codecs/src/mp4.rs +++ b/crates/codecs/src/mp4.rs @@ -37,11 +37,11 @@ pub struct Mp4Source { atrack: Option, video: GopCache, audio: Mutex, - /// Cumulative sample start frames for the audio track (for packet lookup). + /// Audio packet start positions in source sample frames (see [`audio_starts`]). audio_starts: Vec, /// Presentation offset of the audio track in its timescale (edit list, or Opus pre-skip). audio_offset: i64, - /// Decoder pre-roll after a seek, in audio track timescale units (0: prime with one packet). + /// Decoder pre-roll after a seek, in source sample frames (0: prime with one packet). audio_preroll: i64, } @@ -49,6 +49,40 @@ pub fn sniff(b: &[u8]) -> bool { b.len() >= 12 && matches!(&b[4..8], b"ftyp" | b"moov" | b"mdat" | b"wide" | b"free" | b"skip") } +/// Audio packet start positions in source sample frames at `rate`: the running total of the sample +/// durations, except that packets which all decode to the same length (AAC, MPEG audio, AC-3) run +/// on from one another, resynchronising to the total only across gaps of more than half a packet. +/// A remux from Matroska (OBS's) carries the millisecond rounding of its timestamps into the +/// durations: 1008, 1008, 1056… for 1024-sample AAC frames at 48 kHz (see +/// [`crate::audio::contiguous_starts`]). +fn audio_starts(file: &Mp4File, bytes: &crate::Src, ti: usize, rate: u32) -> Vec { + use crate::audio::{FixedFrames, PacketTime, contiguous_starts, fixed_packet_samples}; + let Some(t) = file.tracks.get(ti) else { return Vec::new() }; + let ts = i128::from(t.timescale.max(1)); + let mut total = 0i128; + let stamps: Vec = t + .samples + .iter() + .map(|s| { + let at = total * i128::from(rate) / ts; + total += i128::from(s.duration); + i64::try_from(at).unwrap_or(i64::MAX) + }) + .collect(); + let first = || file.read_sample(bytes, ti, 0).unwrap_or_default(); + let frame = match t.entries.first().map(|e| &e.codec) { + Some(CodecConfig::Aac(a)) => fixed_packet_samples(FixedFrames::Aac(&a.asc), &[], rate), + Some(CodecConfig::Mp3) => fixed_packet_samples(FixedFrames::MpegAudio, &first(), rate), + Some(CodecConfig::Ac3 { .. }) => fixed_packet_samples(FixedFrames::Ac3, &first(), rate), + _ => None, + }; + let Some(frame) = frame else { return stamps }; + // samples per track timescale unit, rounded up: how coarse the running total is + let unit = i64::try_from((i128::from(rate) + ts - 1) / ts).unwrap_or(i64::MAX); + let packets = stamps.iter().map(|&stamp| PacketTime { stamp, stamped: true, samples: Some(frame) }); + contiguous_starts(packets, (frame / 2).max(unit.saturating_mul(2).saturating_add(1))) +} + /// HDR static metadata from the sample entry's `mdcv` / `clli` boxes (0 = unknown). fn hdr_metadata(md: Option<&filmcraft_isobmff::MasteringDisplay>, cll: Option<(u16, u16)>) -> Option { if md.is_none() && cll.is_none() { @@ -240,20 +274,8 @@ impl Mp4Source { start_timecode, file_size: Some(bytes.0.len()), }; - let audio_starts = atrack - .map(|i| { - let t = &file.tracks[i]; - let mut acc = 0i64; - t.samples - .iter() - .map(|s| { - let st = acc; - acc += s.duration as i64; - st - }) - .collect() - }) - .unwrap_or_default(); + let audio_rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate); + let audio_starts = atrack.map(|i| audio_starts(&file, &bytes, i, audio_rate)).unwrap_or_default(); let (audio_offset, audio_preroll) = atrack .map(|i| { let t = &file.tracks[i]; @@ -261,7 +283,8 @@ impl Mp4Source { match &t.entries[0].codec { CodecConfig::Opus(o) => { let off = if t.edit_offset != 0 { t.edit_offset } else { -(o.pre_skip as i64) * ts / 48_000 }; - (off, (crate::audio::OPUS_PRE_ROLL as u64 * ts as u64).div_ceil(48_000) as i64) + // (Opus plays at 48 kHz: the pre-roll is in its sample frames) + (off, crate::audio::OPUS_PRE_ROLL as i64) } _ => (t.edit_offset, 0), } @@ -393,14 +416,12 @@ impl MediaSource for Mp4Source { let offset = i128::from(self.audio_offset) * i128::from(src_rate) / i128::from(track.timescale.max(1)); let s0 = i64::try_from(i128::from(p0) - offset).unwrap_or(if offset > 0 { i64::MIN } else { i64::MAX }); let need = (frames as f64 * ratio).ceil() as i64 + 2; - // Samples-per-unit: the track timescale is usually the sample rate for audio. - let unit = src_rate as f64 / track.timescale.max(1) as f64; let mut src: Vec> = vec![vec![0.0; need.max(0) as usize]; ch]; let mut st = self.audio.lock().unwrap_or_else(|e| e.into_inner()); - let first = self.audio_starts.partition_point(|&x| (x as f64 * unit) as i64 <= s0.max(0)).saturating_sub(1); + let first = self.audio_starts.partition_point(|&x| x <= s0.max(0)).saturating_sub(1); let mut i = first; while i < track.samples.len() { - let pk_start = (self.audio_starts[i] as f64 * unit) as i64; + let pk_start = self.audio_starts[i]; if pk_start >= s0.saturating_add(need) { break; } diff --git a/crates/codecs/src/video.rs b/crates/codecs/src/video.rs index 8aada02e..86b21b7d 100644 --- a/crates/codecs/src/video.rs +++ b/crates/codecs/src/video.rs @@ -45,8 +45,19 @@ pub trait VideoDecoder: Send { /// shortcuts that only change pictures nothing references, which come out flagged /// [`DecodedFrame::draft`]. Decoders without such a mode ignore it. fn set_draft(&mut self, _on: bool) {} + /// Made with fewer worker threads than a decoder for playback gets, because it was made for + /// background work ([`filmcraft_media::cancel::with_background`]). A request for a monitor + /// replaces such a decoder. + fn thread_limited(&self) -> bool { + false + } } +/// Worker threads of the H.264 decoders made for background work (thumbnails): a frame-threaded +/// 4K decoder holds `threads + 2` pictures in flight (~0.5 GB with 16 threads), and the frame +/// workers already decode different files in parallel. +const BACKGROUND_H264_THREADS: usize = 2; + /// NAL unit headers of a length-prefixed (avcC / hvcC) sample: the first two bytes of each unit. fn nal_headers(sample: &[u8], length_size: usize) -> impl Iterator + '_ { let mut pos = 0usize; @@ -200,19 +211,36 @@ pub struct H264Decoder { dec: filmcraft_h264::Decoder, length_size: usize, draft: bool, + /// Worker threads (a background decoder gets fewer). + threads: usize, } impl H264Decoder { + /// The worker threads for a decoder made now. + fn threads_now() -> usize { + if filmcraft_media::cancel::background() { BACKGROUND_H264_THREADS.min(h264_threads()) } else { h264_threads() } + } + + fn make_inner(avcc: &[u8], threads: usize) -> Result { + let mut dec = filmcraft_h264::Decoder::with_threads(threads); + if !avcc.is_empty() { + dec.configure_avcc(avcc).map_err(|e| CodecError::Decode(e.to_string()))?; + } + Ok(dec) + } + pub fn new(avcc: Vec) -> Result { recycle_h264_planes(); - let dec = filmcraft_h264::Decoder::from_avcc(&avcc).map_err(|e| CodecError::Decode(e.to_string()))?; + let threads = Self::threads_now(); + let dec = Self::make_inner(&avcc, threads)?; let length_size = avcc_length_size(&avcc); - Ok(Self { avcc, dec, length_size, draft: false }) + Ok(Self { avcc, dec, length_size, draft: false, threads }) } /// A decoder for Annex B byte-stream samples (start codes, in-band parameter sets: MXF, TS). pub fn annexb() -> Self { recycle_h264_planes(); - Self { avcc: Vec::new(), dec: filmcraft_h264::Decoder::new(), length_size: 0, draft: false } + let threads = Self::threads_now(); + Self { avcc: Vec::new(), dec: filmcraft_h264::Decoder::with_threads(threads), length_size: 0, draft: false, threads } } fn convert(p: filmcraft_h264::Picture) -> DecodedFrame { use std::sync::Arc; @@ -245,9 +273,7 @@ impl VideoDecoder for H264Decoder { self.dec.flush().into_iter().map(Self::convert).collect() } fn reset(&mut self) { - if self.avcc.is_empty() { - self.dec = filmcraft_h264::Decoder::new(); - } else if let Ok(d) = filmcraft_h264::Decoder::from_avcc(&self.avcc) { + if let Ok(d) = Self::make_inner(&self.avcc, self.threads) { self.dec = d; } self.dec.set_draft(self.draft); @@ -255,6 +281,9 @@ impl VideoDecoder for H264Decoder { fn name(&self) -> &str { "FilmCraft H.264" } + fn thread_limited(&self) -> bool { + self.threads < h264_threads() + } fn set_draft(&mut self, on: bool) { self.draft = on; self.dec.set_draft(on); diff --git a/crates/codecs/tests/matroska_audio_oracle.rs b/crates/codecs/tests/matroska_audio_oracle.rs new file mode 100644 index 00000000..c1726fd8 --- /dev/null +++ b/crates/codecs/tests/matroska_audio_oracle.rs @@ -0,0 +1,50 @@ +//! Matroska audio made by ffmpeg, against ffmpeg's decode (#236). ffmpeg stamps blocks to the +//! millisecond, so at 44.1 kHz AAC (1024-sample), MP3 (1152) and AC-3 (1536) packets are each +//! stamped up to half a millisecond off their exact start. Read back block by block, as playback +//! reads it, a tone must stay as smooth as ffmpeg's decode of it: no holes or overlaps between +//! packets. + +mod common; + +use common::*; + +/// Largest |second difference| of `x` from `from` on: a click shows up as a spike. +fn roughness(x: &[f32], from: usize) -> f32 { + x.windows(3).skip(from).map(|w| (w[0] - 2.0 * w[1] + w[2]).abs()).fold(0.0, f32::max) +} + +#[test] +fn matroska_audio_plays_without_holes() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let cases: [(&str, u32, &[&str]); 5] = [ + ("tone_aac_48k.mkv", 48_000, &["-c:a", "aac", "-b:a", "160k"]), + ("tone_aac_44k.mkv", 44_100, &["-c:a", "aac", "-b:a", "160k"]), + ("tone_mp3_44k.mkv", 44_100, &["-c:a", "libmp3lame", "-b:a", "192k"]), + ("tone_mp2_44k.mkv", 44_100, &["-c:a", "mp2", "-b:a", "192k"]), + ("tone_ac3_44k.mkv", 44_100, &["-c:a", "ac3", "-b:a", "192k"]), + ]; + for (name, rate, codec) in cases { + let input = format!("sine=frequency=440:sample_rate={rate}:duration=3"); + let args: Vec<&str> = ["-f", "lavfi", "-i", input.as_str(), "-af", "volume=4"].iter().copied().chain(codec.iter().copied()).collect(); + let Some(path) = fixture(&ff, name, &args) else { + eprintln!("SKIPPED ({name}): this ffmpeg can't encode it"); + continue; + }; + let src = filmcraft_codecs::open_bytes(name, bytes(&path)).unwrap(); + assert_eq!(src.info().audio.as_ref().map(|a| a.sample_rate), Some(rate), "{name}"); + let frames = rate as usize * 29 / 10; + let mut ours = Vec::with_capacity(frames); + while ours.len() < frames { + let n = 512.min(frames - ours.len()); + ours.extend_from_slice(&src.audio(ours.len() as i64, n, rate).unwrap().channels[0]); + } + let theirs = ffmpeg_audio_f32(&ff, &path, &[]); + // past the codecs' start-up (priming, MP3's first frame) + let skip = rate as usize / 10; + let (r_ours, r_theirs) = (roughness(&ours, skip), roughness(&theirs[..theirs.len().min(frames)], skip)); + let peak = ours.iter().skip(skip).fold(0f32, |m, x| m.max(x.abs())); + println!("{name}: peak {peak:.3}, max |second difference| ours {r_ours:.5}, ffmpeg {r_theirs:.5}"); + assert!(peak > 0.4, "{name}: the tone is missing (peak {peak})"); + assert!(r_ours < 2.0 * r_theirs + 0.005, "{name}: clicks between packets (|Δ²| {r_ours} vs ffmpeg's {r_theirs})"); + } +} diff --git a/crates/edit/Cargo.toml b/crates/edit/Cargo.toml index fcecc05a..bcf35476 100644 --- a/crates/edit/Cargo.toml +++ b/crates/edit/Cargo.toml @@ -16,5 +16,6 @@ thiserror = { workspace = true } [dev-dependencies] proptest = { workspace = true } +filmcraft-geom = { workspace = true } filmcraft-media = { workspace = true } serde_json = { workspace = true } diff --git a/crates/edit/src/frame_size.rs b/crates/edit/src/frame_size.rs new file mode 100644 index 00000000..607cfe40 --- /dev/null +++ b/crates/edit/src/frame_size.rs @@ -0,0 +1,212 @@ +//! Sequence Settings ▸ "Scale motion effects proportionally when changing frame size". +//! +//! Motion's Position is in sequence pixels and its Scale is relative to the source, so a new frame +//! size leaves every clip where it was in pixels: smaller and off-centre in a larger frame, cropped +//! in a smaller one. With the option on, the whole picture scales instead, by one factor about the +//! frame centre. On an aspect change (16:9 → 9:16) that factor is the one that fits the old frame +//! inside the new one (`min(W1/W0, H1/H0)`), so nothing is cropped and the old composition sits +//! centred, letterboxed or pillarboxed, rather than filling the new frame. + +use filmcraft_project::{ParamValue, Sequence, TrackKind}; + +/// Motion Scale's upper bound (the parameter's range, in %). +const MAX_SCALE: f64 = 10_000.0; + +/// The uniform factor `k` and the old and new frame centres for `from` → `to`, or `None` for an +/// empty frame on either side. +fn fit(from: (u32, u32), to: (u32, u32)) -> Option<(f64, (f64, f64), (f64, f64))> { + if from.0 == 0 || from.1 == 0 || to.0 == 0 || to.1 == 0 { + return None; + } + let (w0, h0, w1, h1) = (f64::from(from.0), f64::from(from.1), f64::from(to.0), f64::from(to.1)); + let k = (w1 / w0).min(h1 / h0); + (k.is_finite() && k > 0.0).then_some((k, (w0 / 2.0, h0 / 2.0), (w1 / 2.0, h1 / 2.0))) +} + +/// Rescale the Motion of every video clip in `seq` for a frame-size change from `from` to `to`: +/// static values and every keyframe. Returns how many clips were rescaled (0 when the size does not +/// change). +/// +/// - Position maps through `p' = c1 + k·(p − c0)`. An automatic (NaN) position stays automatic: it +/// follows the frame centre on its own. +/// - Scale and Scale Width are multiplied by `k`, up to the parameter's 10 000 %. A clip with Scale to +/// Frame Size keeps its Scale, because the renderer refits it to the new frame anyway. +/// - Anchor Point (source pixels) and Rotation do not depend on the frame size and stay as they are. +/// - Only the intrinsic Motion effect changes, as the option's name says: the points of standard +/// effects (a Mirror centre, the Transform effect's Position) keep their pixel values. +pub fn scale_motion(seq: &mut Sequence, from: (u32, u32), to: (u32, u32)) -> usize { + if from == to { + return 0; + } + let Some((k, c0, c1)) = fit(from, to) else { return 0 }; + let mut n = 0; + for track in seq.all_tracks_mut().filter(|t| t.kind == TrackKind::Video) { + for item in &mut track.items { + let refits = item.scale_to_frame; + let Some(motion) = item.effect_mut("motion") else { continue }; + if let Some(p) = motion.param_mut("position") { + p.map_values(|v| match v { + ParamValue::Vec2(mut v) => { + v.x = c1.0 + k * (v.x - c0.0); + v.y = c1.1 + k * (v.y - c0.1); + ParamValue::Vec2(v) + } + other => other, + }); + } + if !refits { + for id in ["scale", "scale_width"] { + if let Some(p) = motion.param_mut(id) { + p.map_values(|v| match v { + ParamValue::Float(s) => ParamValue::Float((s * k).clamp(0.0, MAX_SCALE)), + other => other, + }); + } + } + } + n += 1; + } + } + n +} + +#[cfg(test)] +mod tests { + use super::*; + use filmcraft_project::{ClipId, ItemId, Keyframe, Label, Project, SequenceSettings, TrackItem, find_effect}; + use filmcraft_time::Tick; + + fn seq_with(items: Vec) -> Sequence { + let mut p = Project::new("t"); + let s = p.new_sequence("s", SequenceSettings::default(), 2, 1, None); + let mut seq = p.sequence(s).unwrap().clone(); + seq.video_tracks[0].items = items; + seq + } + + fn clip(id: u64, pos: (f64, f64), scale: f64) -> TrackItem { + let mut motion = find_effect("motion").unwrap().instance(); + motion.param_mut("position").unwrap().value = ParamValue::Vec2(filmcraft_geom::Vec2::new(pos.0, pos.1)); + motion.param_mut("scale").unwrap().value = ParamValue::Float(scale); + TrackItem { + id: ClipId(id), + item: ItemId(1), + name: format!("c{id}"), + label: Label::Iris, + start: Tick(id as i64 * 1_000_000_000_000), + duration: Tick(1_000_000_000_000), + source_in: Tick::ZERO, + speed: 1.0, + reverse: false, + enabled: true, + link: None, + group: None, + effects: vec![motion], + markers: vec![], + gain_db: 0.0, + frame_hold: None, + scale_to_frame: false, + essential: None, + multicam: None, + time_interpolation: Default::default(), + hold_filters: false, + field_options: None, + source_channels: Vec::new(), + graphic: None, + } + } + + fn pos(seq: &Sequence, i: usize) -> (f64, f64) { + match seq.video_tracks[0].items[i].effect("motion").unwrap().param("position").unwrap().value { + ParamValue::Vec2(v) => (v.x, v.y), + _ => panic!("position is a point"), + } + } + + fn scale(seq: &Sequence, i: usize) -> f64 { + match seq.video_tracks[0].items[i].effect("motion").unwrap().param("scale").unwrap().value { + ParamValue::Float(s) => s, + _ => panic!("scale is a number"), + } + } + + fn close(a: (f64, f64), b: (f64, f64)) -> bool { + (a.0 - b.0).abs() < 1e-9 && (a.1 - b.1).abs() < 1e-9 + } + + #[test] + fn same_aspect_scales_position_and_scale_together() { + let mut seq = seq_with(vec![clip(1, (960.0, 540.0), 100.0), clip(2, (300.0, 90.0), 50.0)]); + assert_eq!(scale_motion(&mut seq, (1920, 1080), (1280, 720)), 2); + assert!(close(pos(&seq, 0), (640.0, 360.0)), "{:?}", pos(&seq, 0)); + assert!(close(pos(&seq, 1), (200.0, 60.0)), "{:?}", pos(&seq, 1)); + assert!((scale(&seq, 0) - 100.0 * 2.0 / 3.0).abs() < 1e-9); + assert!((scale(&seq, 1) - 50.0 * 2.0 / 3.0).abs() < 1e-9); + } + + #[test] + fn aspect_change_fits_the_old_frame_inside_the_new_one() { + // 1920x1080 -> 1080x1920: k = min(1080/1920, 1920/1080) = 0.5625, centred + let mut seq = seq_with(vec![clip(1, (960.0, 540.0), 100.0), clip(2, (0.0, 0.0), 100.0)]); + scale_motion(&mut seq, (1920, 1080), (1080, 1920)); + assert!(close(pos(&seq, 0), (540.0, 960.0)), "{:?}", pos(&seq, 0)); + // the old top-left corner lands on the new frame's left edge, letterboxed vertically + assert!(close(pos(&seq, 1), (0.0, 960.0 - 0.5625 * 540.0)), "{:?}", pos(&seq, 1)); + assert!((scale(&seq, 0) - 56.25).abs() < 1e-9); + } + + #[test] + fn keyframes_scale_too() { + let mut c = clip(1, (960.0, 540.0), 100.0); + let m = c.effect_mut("motion").unwrap(); + m.param_mut("position").unwrap().keyframes = vec![ + Keyframe::new(Tick::ZERO, ParamValue::Vec2(filmcraft_geom::Vec2::new(0.0, 0.0))), + Keyframe::new(Tick(10), ParamValue::Vec2(filmcraft_geom::Vec2::new(1920.0, 1080.0))), + ]; + m.param_mut("scale").unwrap().keyframes = vec![Keyframe::new(Tick::ZERO, ParamValue::Float(100.0)), Keyframe::new(Tick(10), ParamValue::Float(200.0))]; + let mut seq = seq_with(vec![c]); + scale_motion(&mut seq, (1920, 1080), (3840, 2160)); + let m = seq.video_tracks[0].items[0].effect("motion").unwrap(); + let kp: Vec<_> = m.param("position").unwrap().keyframes.iter().map(|k| k.value.clone()).collect(); + assert_eq!(kp, vec![ParamValue::Vec2(filmcraft_geom::Vec2::new(0.0, 0.0)), ParamValue::Vec2(filmcraft_geom::Vec2::new(3840.0, 2160.0))]); + let ks: Vec<_> = m.param("scale").unwrap().keyframes.iter().map(|k| k.value.clone()).collect(); + assert_eq!(ks, vec![ParamValue::Float(200.0), ParamValue::Float(400.0)]); + } + + #[test] + fn automatic_position_stays_automatic() { + let mut seq = seq_with(vec![clip(1, (f64::NAN, f64::NAN), 100.0)]); + scale_motion(&mut seq, (1920, 1080), (1280, 720)); + let (x, y) = pos(&seq, 0); + assert!(x.is_nan() && y.is_nan()); + assert!((scale(&seq, 0) - 100.0 * 2.0 / 3.0).abs() < 1e-9); + } + + #[test] + fn scale_to_frame_clips_keep_their_scale_but_move() { + let mut c = clip(1, (480.0, 270.0), 100.0); + c.scale_to_frame = true; + let mut seq = seq_with(vec![c]); + scale_motion(&mut seq, (1920, 1080), (3840, 2160)); + assert!(close(pos(&seq, 0), (960.0, 540.0)), "{:?}", pos(&seq, 0)); + assert_eq!(scale(&seq, 0), 100.0); + } + + #[test] + fn scale_is_capped_at_the_parameter_range() { + let mut seq = seq_with(vec![clip(1, (1.0, 1.0), 5000.0)]); + scale_motion(&mut seq, (2, 2), (8, 8)); + assert_eq!(scale(&seq, 0), MAX_SCALE); + } + + #[test] + fn same_size_or_empty_frame_changes_nothing() { + let mut seq = seq_with(vec![clip(1, (100.0, 100.0), 100.0)]); + let before = seq.clone(); + assert_eq!(scale_motion(&mut seq, (1920, 1080), (1920, 1080)), 0); + assert_eq!(scale_motion(&mut seq, (0, 1080), (1280, 720)), 0); + assert_eq!(scale_motion(&mut seq, (1920, 1080), (1280, 0)), 0); + // Debug, not ==: the automatic anchor is NaN, which never equals itself + assert_eq!(format!("{:?}", seq.video_tracks[0].items[0].effects), format!("{:?}", before.video_tracks[0].items[0].effects)); + } +} diff --git a/crates/edit/src/lib.rs b/crates/edit/src/lib.rs index 67937f85..ed66f951 100644 --- a/crates/edit/src/lib.rs +++ b/crates/edit/src/lib.rs @@ -18,6 +18,7 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] pub mod captions; +pub mod frame_size; pub mod multicam; pub mod through; pub mod transcript; diff --git a/crates/engine/src/autosave.rs b/crates/engine/src/autosave.rs index 5b87a958..73338487 100644 --- a/crates/engine/src/autosave.rs +++ b/crates/engine/src/autosave.rs @@ -22,12 +22,13 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel}; use std::thread::JoinHandle; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use filmcraft_format::autosave as names; use filmcraft_project::Project; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use web_time::{Instant, SystemTime, UNIX_EPOCH}; use crate::settings; @@ -470,7 +471,17 @@ impl Preferences { } } -/// The per-user data directory (`FILMCRAFT_DATA_DIR` overrides): +/// A file with this name next to the executable makes the install portable: its data directory is +/// `/data` instead of the per-user one (the Windows portable zip ships it). +pub const PORTABLE_MARKER: &str = "portable.txt"; + +/// `/data` when `exe_dir` holds [`PORTABLE_MARKER`], else `None`. +pub fn portable_data_dir(exe_dir: &Path) -> Option { + exe_dir.join(PORTABLE_MARKER).is_file().then(|| exe_dir.join("data")) +} + +/// The data directory. `FILMCRAFT_DATA_DIR` overrides; a portable install ([`PORTABLE_MARKER`] next +/// to the executable) uses `/data`; otherwise the per-user directory: /// macOS `~/Library/Application Support/FilmCraft`, Windows `%APPDATA%\FilmCraft`, /// elsewhere `$XDG_DATA_HOME/filmcraft` or `~/.local/share/filmcraft`. pub fn default_data_dir() -> Option { @@ -478,6 +489,9 @@ pub fn default_data_dir() -> Option { if let Some(d) = env("FILMCRAFT_DATA_DIR") { return Some(d); } + if let Some(d) = std::env::current_exe().ok().and_then(|exe| exe.parent().and_then(portable_data_dir)) { + return Some(d); + } if cfg!(target_os = "macos") { return env("HOME").map(|h| h.join("Library/Application Support/FilmCraft")); } diff --git a/crates/engine/src/autosave_tests.rs b/crates/engine/src/autosave_tests.rs index 0e113e41..5ac8281e 100644 --- a/crates/engine/src/autosave_tests.rs +++ b/crates/engine/src/autosave_tests.rs @@ -176,6 +176,19 @@ fn auto_save_ring_rotates_and_can_save_the_project() { let _ = std::fs::remove_dir_all(&d); } +#[test] +fn portable_marker_keeps_data_next_to_the_executable() { + // #139: a portable install must not write to %APPDATA%. + let d = temp_dir("portable"); + assert_eq!(autosave::portable_data_dir(&d), None); + std::fs::create_dir_all(d.join(autosave::PORTABLE_MARKER)).unwrap(); + assert_eq!(autosave::portable_data_dir(&d), None, "a directory is not the marker"); + std::fs::remove_dir_all(d.join(autosave::PORTABLE_MARKER)).unwrap(); + std::fs::write(d.join(autosave::PORTABLE_MARKER), "").unwrap(); + assert_eq!(autosave::portable_data_dir(&d), Some(d.join("data"))); + let _ = std::fs::remove_dir_all(&d); +} + #[test] fn journal_costs_the_ui_thread_only_a_handoff() { // Per change the UI thread pays an Arc clone and a channel send; encoding happens on the worker. diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index acdccda7..28abaf25 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -131,9 +131,16 @@ pub(crate) fn named_clips(s: &Session, spec: &CommandSpec, p: &Value) -> Option< let clips: Vec = named_ids(spec, p, "clips", "clip")?.into_iter().map(ClipId).filter(|c| seq.find_item(*c).is_some()).collect(); (!clips.is_empty()).then_some(clips) } -/// Project items named explicitly in `items` / `item`, likewise. +/// Project items named explicitly in `items` / `item`, likewise. The Project panel's selection +/// holds bins beside items (one id space, see `project.select`), so a bin named here counts too; +/// the project's own top bin does not, since no command acts on it (#244). pub(crate) fn named_items(s: &Session, spec: &CommandSpec, p: &Value) -> Option> { - let items: Vec = named_ids(spec, p, "items", "item")?.into_iter().map(ItemId).filter(|i| s.project.item(*i).is_some()).collect(); + let root = s.project.root.id; + let exists = |i: &ItemId| { + let b = filmcraft_project::BinId(i.0); + s.project.item(*i).is_some() || (b != root && s.project.root.find_bin(b).is_some()) + }; + let items: Vec = named_ids(spec, p, "items", "item")?.into_iter().map(ItemId).filter(exists).collect(); (!items.is_empty()).then_some(items) } @@ -1071,7 +1078,7 @@ fn build() -> Vec { "Media…", ["File", "Export"], None, - r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass"?,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#, + r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass"?,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#, has_seq, crate::export_tools::export_media ), @@ -1513,41 +1520,70 @@ fn build() -> Vec { "Sequence Settings…", ["Sequence"], None, - r#"{"width":u32?,"height":u32?,"fps":f64?,"name":str?,"sampleRate":u32?,"mix":"Stereo|Mono|5.1|Adaptive"?}"#, + r#"{"width":u32?,"height":u32?,"fps":f64?,"name":str?,"sampleRate":u32?,"mix":"Stereo|Mono|5.1|Adaptive"?,"dropFrame":bool?,"maxRenderQuality":bool?,"workingSpace":"rec709"|"rec2100-pq"|"rec2100-hlg"?,"wideGamut":bool?,"autoToneMap":bool?,"scaleMotion":bool=false}"#, has_seq, |s, p| { + const CMD: &str = "sequence.settings"; let id = s.state.active_sequence.ok_or(EngineError::NoSequence)?; - let fps = fps_p(p, "sequence.settings")?; + let fps = fps_p(p, CMD)?; let mix = match str_p(p, "mix") { - Some(m) => Some( - crate::mixer::channels_from(m).ok_or_else(|| bad("sequence.settings", format!("unknown mix `{m}` (Stereo, Mono, 5.1, Adaptive)")))?, - ), + Some(m) => Some(crate::mixer::channels_from(m).ok_or_else(|| bad(CMD, format!("unknown mix `{m}` (Stereo, Mono, 5.1, Adaptive)")))?), None => None, }; + let working = match str_p(p, "workingSpace") { + Some(w) => Some(filmcraft_color::WorkingSpace::parse(w).ok_or_else(|| bad(CMD, format!("unknown working space `{w}`")))?), + None => None, + }; + let drop_frame = bool_p(p, "dropFrame"); + let rate = fps.or_else(|| s.active_sequence().map(|q| q.settings.frame_rate)).unwrap_or(FrameRate::FPS_23_976); + if drop_frame == Some(true) && !rate.supports_drop_frame() { + return Err(bad(CMD, format!("drop-frame timecode needs a 29.97, 59.94 or 119.88 fps timebase, not {rate}"))); + } + let scale_motion = bool_p(p, "scaleMotion").unwrap_or(false); let p = p.clone(); - s.edit("Sequence Settings", |pr, _| { + let scaled = s.edit("Sequence Settings", |pr, _| { if let Some(n) = str_p(&p, "name") { pr.item_mut(id).ok_or(EngineError::NoSequence)?.name = n.to_string(); } let q = pr.sequence_mut(id).ok_or(EngineError::NoSequence)?; - if let Some(w) = checked_u32_p(&p, "width", "sequence.settings")? { + let old_size = (q.settings.width, q.settings.height); + if let Some(w) = checked_u32_p(&p, "width", CMD)? { q.settings.width = w; } - if let Some(h) = checked_u32_p(&p, "height", "sequence.settings")? { + if let Some(h) = checked_u32_p(&p, "height", CMD)? { q.settings.height = h; } if let Some(f) = fps { q.settings.frame_rate = f; } - if let Some(sr) = checked_u32_p(&p, "sampleRate", "sequence.settings")? { + if let Some(sr) = checked_u32_p(&p, "sampleRate", CMD)? { q.settings.sample_rate = sr; } if let Some(m) = mix { q.settings.audio_master = m; } - q.settings.validate().map_err(|e| bad("sequence.settings", e)) + if let Some(df) = drop_frame { + q.settings.drop_frame = df; + } + if let Some(v) = bool_p(&p, "maxRenderQuality") { + q.settings.max_render_quality = v; + } + let c = &mut q.settings.color; + if let Some(w) = working { + c.working = w; + } + if let Some(v) = bool_p(&p, "wideGamut") { + c.wide_gamut = v; + } + if let Some(v) = bool_p(&p, "autoToneMap") { + c.auto_tone_map = v; + } + q.settings.working_space = q.settings.color.working.label().into(); + q.settings.validate().map_err(|e| bad(CMD, e))?; + let new_size = (q.settings.width, q.settings.height); + Ok(if scale_motion { edit::frame_size::scale_motion(q, old_size, new_size) } else { 0 }) })?; - Ok(Value::Null) + Ok(json!({"scaledClips": scaled})) } ), cmd!("sequence.renderEffectsInToOut", "Render Effects In to Out", ["Sequence"], Some("Enter"), r#"{"wait":bool=false}"#, has_seq, |s, p| { @@ -1609,14 +1645,15 @@ fn build() -> Vec { })?; Ok(Value::Null) }), + // With no In / Out mark, like Premiere: the start / end of the sequence (#208). cmd!("markers.goToIn", "Go to In", ["Markers"], Some("Shift+I"), "{}", has_seq, |s, _| { - if let Some(i) = s.active_sequence().and_then(|q| q.mark_in) { + if let Some(i) = s.active_sequence().map(|q| q.mark_in.unwrap_or(Tick::ZERO)) { s.set_playhead(i); } Ok(Value::Null) }), cmd!("markers.goToOut", "Go to Out", ["Markers"], Some("Shift+O"), "{}", has_seq, |s, _| { - if let Some(o) = s.active_sequence().and_then(|q| q.mark_out) { + if let Some(o) = s.active_sequence().map(|q| q.mark_out.unwrap_or_else(|| q.duration())) { s.set_playhead(o); } Ok(Value::Null) diff --git a/crates/engine/src/essential_sound.rs b/crates/engine/src/essential_sound.rs index cd05fca8..58d7556a 100644 --- a/crates/engine/src/essential_sound.rs +++ b/crates/engine/src/essential_sound.rs @@ -12,7 +12,7 @@ //! | `essentialSound.setType` / `clearType` | assign / clear the audio type (clearing removes its effects and gain) | //! | `essentialSound.set` | one setting by dotted key (`repair.noise.amount`), or several (`values`) | //! | `essentialSound.applyPreset` / `savePreset` / `deletePreset` | built-in and user presets | -//! | `essentialSound.autoMatch` | measure integrated loudness (BS.1770) and set the match gain to the target | +//! | `essentialSound.autoMatch` | measure integrated loudness (BS.1770, mono clips as one channel) and set the match gain to the target | //! | `essentialSound.generateDucking` | Volume keyframes that duck Music / Ambience under the trigger types | use serde_json::{Value, json}; @@ -262,15 +262,26 @@ fn delete_preset(s: &mut Session, p: &Value) -> Result { /// Integrated loudness (LUFS) of a clip's own signal (clip gain + clip effects) over its whole /// timeline range, or `None` when it is silent / has no audio. +/// +/// The clip is measured with the channels it has: a stereo (or wider) source as the two channels +/// it plays, a mono source or a mono pick as one channel (its two identical sides folded to one), +/// the way BS.1770 and ffmpeg's `ebur128` measure a mono programme. Metering the dual-mono pair the +/// clip plays would read 3 LU louder than the file and match it 3 dB too quietly (#296). pub fn clip_loudness(item: &TrackItem, sr: u32, sources: &dyn SourceProvider) -> Option { let a0 = item.start.to_units_floor(sr as i64); let a1 = item.end().to_units_floor(sr as i64); - let mut meter = LoudnessMeter::new(sr as f64, 2); + let mono = filmcraft_render::audio::clip_is_mono(item, sources)?; + let mut meter = LoudnessMeter::new(sr as f64, if mono { 1 } else { 2 }); let mut pos = a0; while pos < a1 { let n = (a1 - pos).min(sr as i64) as usize; let [l, r] = filmcraft_render::audio::clip_signal(item, pos, n, sr, sources)?; - meter.process(&[&l, &r]); + if mono { + let m: Vec = l.iter().zip(&r).map(|(a, b)| 0.5 * (a + b)).collect(); + meter.process(&[&m]); + } else { + meter.process(&[&l, &r]); + } pos += n as i64; } Some(meter.integrated()).filter(|v| v.is_finite()) diff --git a/crates/engine/src/essential_sound_tests.rs b/crates/engine/src/essential_sound_tests.rs index 2474972a..e7b5374e 100644 --- a/crates/engine/src/essential_sound_tests.rs +++ b/crates/engine/src/essential_sound_tests.rs @@ -265,6 +265,41 @@ fn auto_match_hits_the_target_loudness() { } } +/// #296: a mono file measured ~3 LU too loud. The same speech imported as a 1-channel WAV and as +/// a dual-mono stereo WAV must measure 3.01 LU apart (the stereo pair carries twice the energy), +/// the mono clip must agree with a one-channel BS.1770 meter over the file (what ffmpeg's +/// `ebur128` reports), and a mono pick (Modify ▸ Audio Channels) of the stereo file counts as mono. +#[test] +fn auto_match_measures_a_mono_clip_as_one_channel() { + let sig = speech(12.0, 0.3, |t| (t % 3.0) < 2.2); + let (mut s, c) = session_with(&[(sig.clone(), 0.0)]); + let stereo = c[0]; + let mono_wav: Arc<[u8]> = filmcraft_media::wav::write_wav16(&sig, 1, SR).into(); + let mono_item = crate::commands::import_bytes(&mut s, "/mono.wav", mono_wav, None).unwrap(); + let r = s.execute("timeline.place", json!({"item": mono_item.0, "audioTrack": "A2", "seconds": 0.0})).unwrap(); + let mono = ClipId(r["clips"][0].as_u64().unwrap()); + for c in [stereo, mono] { + s.execute("essentialSound.setType", json!({"clips": [c.0], "type": "dialogue"})).unwrap(); + } + let measured = |s: &mut Session, c: ClipId| { + let r = s.execute("essentialSound.autoMatch", json!({"clips": [c.0], "target": -23.0})).unwrap(); + r["clips"][0]["measuredLufs"].as_f64().unwrap() + }; + let (two, one) = (measured(&mut s, stereo), measured(&mut s, mono)); + let mut file = LoudnessMeter::new(SR as f64, 1); + file.process(&[&sig]); + let want = file.integrated(); + println!("stereo (dual mono) {two:.3} LUFS, mono {one:.3} LUFS, mono file {want:.3} LUFS"); + assert!((one - want).abs() <= 0.1, "mono clip {one} LUFS, the file reads {want} LUFS"); + assert!((two - one - 3.0103).abs() <= 0.1, "dual mono {two} LUFS should read 3.01 LU above mono {one} LUFS"); + let st = item(&s, mono).essential.unwrap(); + assert!((st.loudness.gain_db - (-23.0 - want)).abs() <= 0.1, "match gain {} dB for {want} LUFS", st.loudness.gain_db); + // one picked channel of the stereo file plays on both sides: measured as mono too + s.execute("clip.audioChannels", json!({"clips": [stereo.0], "channels": [0]})).unwrap(); + let picked = measured(&mut s, stereo); + assert!((picked - want).abs() <= 0.1, "mono pick {picked} LUFS, the file reads {want} LUFS"); +} + #[test] fn ducking_generates_volume_keyframes_under_dialogue() { // dialogue in [2, 4] and [6, 7.5] s; music for 10 s diff --git a/crates/engine/src/explicit_targets_tests.rs b/crates/engine/src/explicit_targets_tests.rs index 164d0398..1f2eca77 100644 --- a/crates/engine/src/explicit_targets_tests.rs +++ b/crates/engine/src/explicit_targets_tests.rs @@ -175,3 +175,39 @@ fn naming_targets_lifts_only_the_selection_condition() { let _ = s.execute("edit.clear", json!({"clips": [a.id.0]})); assert!(clip(&s, a.id).is_some(), "a clip on a locked track is not removed"); } + +/// `project.delete` with explicit ids needs no Project-panel selection, for bins as much as for +/// items: the selection can hold both, so a named bin stands in for it (#244). +#[test] +fn project_delete_takes_explicit_bins_and_items_without_a_selection() { + let mut s = demo(); + let before = s.project.items.len(); + let empty = s.execute("file.newBin", json!({"name": "Empty"})).unwrap()["bin"].as_u64().unwrap(); + let outer = s.execute("file.newBin", json!({"name": "Old footage"})).unwrap()["bin"].as_u64().unwrap(); + let ocean = item_named(&s, "Ocean_Sunset.mp4"); + s.execute("project.moveToBin", json!({"items": [ocean.0], "bin": outer})).unwrap(); + s.state.project_selection.clear(); + assert!(!s.is_enabled("project.delete")); + assert_eq!(disabled(s.execute("project.delete", json!({}))), "select an item in the Project panel"); + // an empty bin, by its id alone + let r = s.execute("project.delete", json!({"items": [empty]})).unwrap(); + assert_eq!(r, json!({"items": 0, "bins": 1})); + assert!(s.project.root.find_bin(filmcraft_project::BinId(empty)).is_none()); + // a bin with a clip in it, and an item, each by id alone + let r = s.execute("project.delete", json!({"items": [outer]})).unwrap(); + assert_eq!(r, json!({"items": 1, "bins": 1})); + assert!(s.project.item(ocean).is_none()); + let dunes = item_named(&s, "Desert_Dunes.mp4"); + let r = s.execute("project.delete", json!({"items": [dunes.0]})).unwrap(); + assert_eq!(r, json!({"items": 1, "bins": 0})); + assert_eq!(s.project.items.len(), before - 2); + assert!(s.state.project_selection.is_empty(), "the selection is left as it was"); + assert!(!s.is_enabled("project.delete"), "menu enablement still follows the selection"); + // ids of nothing, or of the project's own top bin, do not stand in for a selection + let root = s.project.root.id.0; + assert_eq!(disabled(s.execute("project.delete", json!({"items": [root]}))), "select an item in the Project panel"); + assert_eq!(disabled(s.execute("project.delete", json!({"items": [999_999]}))), "select an item in the Project panel"); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.project.items.len(), before); +} diff --git a/crates/engine/src/export_tests.rs b/crates/engine/src/export_tests.rs index 474b466a..a934969d 100644 --- a/crates/engine/src/export_tests.rs +++ b/crates/engine/src/export_tests.rs @@ -484,6 +484,23 @@ fn queue_exports_several_sequences_and_ranges() { assert_eq!(len("parts_2.wav"), 44 + 12_000 * 4, "0.25 s of 16-bit stereo"); } +#[test] +fn queue_folder_paths_expand_home_and_take_either_separator() { + let mut s = demo(); + let first = s.state.active_sequence.unwrap(); + let second = s.execute("file.newSequence", json!({"name": "Second Cut", "width": 640, "height": 360, "fps": 25})).unwrap()["sequence"].as_u64().unwrap(); + let preset = "Waveform Audio 48 kHz 16-bit"; + // a folder that does not exist yet, named by a trailing `\`: each sequence keeps its own name + s.execute("export.queue.add", json!({"preset": preset, "path": "not-yet-made\\", "sequences": [first.0, second]})).unwrap(); + let paths: Vec = queue(&mut s).iter().map(|i| i["path"].as_str().unwrap().to_string()).collect(); + assert!(paths.iter().any(|p| p.ends_with("Second Cut.wav")), "{paths:?}"); + if let Some(home) = crate::media_browser::std_home_dir() { + s.execute("export.queue.add", json!({"preset": preset, "path": "~/Exports/", "sequence": first.0})).unwrap(); + let last = queue(&mut s).last().unwrap()["path"].as_str().unwrap().to_string(); + assert!(last.starts_with(&home), "{last}"); + } +} + #[test] fn quick_export_uses_and_remembers_a_preset() { let mut s = demo(); diff --git a/crates/engine/src/export_tools.rs b/crates/engine/src/export_tools.rs index b36566c7..7d3073f2 100644 --- a/crates/engine/src/export_tools.rs +++ b/crates/engine/src/export_tools.rs @@ -230,6 +230,9 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option if let Some(v) = bool_p(p, "sdr") { settings.sdr = v; } + if let Some(v) = bool_p(p, "alpha") { + settings.alpha = v; + } match (crate::commands::checked_u32_p(p, "width", cmd)?, crate::commands::checked_u32_p(p, "height", cmd)?) { (Some(w), Some(h)) => settings.frame_size = Some((w, h)), (None, None) => {} @@ -304,7 +307,7 @@ pub fn default_export_dir(s: &Session) -> PathBuf { // the web build's virtual file table: written files are offered as downloads return PathBuf::from("/exports"); } - if let Ok(h) = std::env::var("HOME").or_else(|_| std::env::var("USERPROFILE")) { + if let Some(h) = crate::media_browser::std_home_dir() { let movies = Path::new(&h).join("Movies"); return if movies.is_dir() { movies } else { PathBuf::from(h) }; } @@ -327,11 +330,19 @@ fn with_extension(path: &str, settings: &ExportSettings) -> String { } } -fn expand_home(p: &str) -> String { - match (p.strip_prefix("~/"), std::env::var("HOME")) { - (Some(rest), Ok(h)) => format!("{h}/{rest}"), +/// `~/` expanded to the home directory (`USERPROFILE` on Windows). On Windows `/` becomes `\`: +/// after a `\\?\` prefix (what `canonicalize` returns) `/` is not a separator. +pub fn expand_home(p: &str) -> String { + let p = match (p.strip_prefix("~/"), crate::media_browser::std_home_dir()) { + (Some(rest), Some(h)) => format!("{h}/{rest}"), _ => p.to_string(), - } + }; + if cfg!(windows) { p.replace('/', "\\") } else { p } +} + +/// `x` names a folder: it ends in a separator or already exists as one. +fn is_folder(x: &str) -> bool { + x.ends_with(['/', '\\']) || Path::new(x).is_dir() } /// Output path for `seq`: the `path` param (a file, or a directory ending in `/` or existing), else @@ -339,7 +350,7 @@ fn expand_home(p: &str) -> String { fn output_path(s: &Session, p: &Value, seq: ItemId, settings: &ExportSettings, suffix: Option) -> String { let name = s.project.item(seq).map(|i| i.name.clone()).unwrap_or_else(|| "Sequence".into()); let base = match str_p(p, "path").map(expand_home) { - Some(x) if x.ends_with('/') || x.ends_with('\\') || Path::new(&x).is_dir() => Path::new(&x).join(file_safe(&name)).to_string_lossy().to_string(), + Some(x) if is_folder(&x) => Path::new(&x).join(file_safe(&name)).to_string_lossy().to_string(), Some(x) => x, None => default_export_dir(s).join(file_safe(&name)).to_string_lossy().to_string(), }; @@ -633,11 +644,7 @@ fn queue_add(s: &mut Session, p: &Value) -> Result { }; st.range = range_param(s, &project, seq, *r, times, cmd)?; // several ranges of one sequence get numbered names; several sequences their own names - let suffix = if many && (ranges.len() > 1 || str_p(p, "path").is_some_and(|x| !x.ends_with('/') && !Path::new(&expand_home(x)).is_dir())) { - Some(n) - } else { - None - }; + let suffix = if many && (ranges.len() > 1 || str_p(p, "path").is_some_and(|x| !is_folder(&expand_home(x)))) { Some(n) } else { None }; st.path = output_path(s, p, seq, &st, suffix); n += 1; items.push((seq, st)); diff --git a/crates/engine/src/graphics.rs b/crates/engine/src/graphics.rs index 2e19a349..fea4922a 100644 --- a/crates/engine/src/graphics.rs +++ b/crates/engine/src/graphics.rs @@ -98,7 +98,12 @@ pub(crate) fn graphic_source(p: &mut filmcraft_project::Project, w: u32, h: u32, /// topmost clip at the playhead that is free for the duration (a track is added if needed). fn new_graphic_clip(s: &mut Session, layer: filmcraft_project::EffectInstance, name: &str, p: &Value) -> Result { let seconds = f64_p(p, "seconds").unwrap_or(5.0).max(0.01); - place_video_clip(s, name, p, "New Graphic", vec![layer], move |pr, (w, h, rate)| { + // Here `seconds` is duration, not the generic time parser's placement alias. + let mut placement = p.clone(); + if let Some(params) = placement.as_object_mut() { + params.remove("seconds"); + } + place_video_clip(s, name, &placement, "New Graphic", vec![layer], move |pr, (w, h, rate)| { let src = graphic_source(pr, w, h, rate); (src, rate.snap_nearest(Tick::from_seconds_f64(seconds)).max(rate.frame_duration())) }) diff --git a/crates/engine/src/graphics_tests.rs b/crates/engine/src/graphics_tests.rs index 803cb7b3..0c7cc4cb 100644 --- a/crates/engine/src/graphics_tests.rs +++ b/crates/engine/src/graphics_tests.rs @@ -24,6 +24,47 @@ fn text_of(l: &filmcraft_project::graphic::LayerSpec) -> String { } } +#[test] +fn graphic_duration_does_not_set_its_timeline_position() { + for command in ["graphics.newText", "graphics.newShape"] { + for playhead_frame in [0, 72] { + for seconds in [1, 2] { + let mut s = Session::default(); + s.execute("file.newSequence", json!({"name": "Placement", "fps": 24, "width": 128, "height": 128, "video": 1, "audio": 1})).unwrap(); + s.execute("playhead.set", json!({"frame": playhead_frame})).unwrap(); + let start = s.playhead(); + let r = s.execute(command, json!({"seconds": seconds})).unwrap(); + let clip = ClipId(r["clip"].as_u64().unwrap()); + let q = s.active_sequence().unwrap(); + let (_, it) = q.find_item(clip).unwrap(); + assert_eq!(it.start, start, "{command}, duration {seconds}, playhead {playhead_frame}"); + assert_eq!(it.duration, Tick::from_seconds_f64(f64::from(seconds))); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.active_sequence().unwrap().find_item(clip).is_none()); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(s.active_sequence().unwrap().find_item(clip).unwrap().1.start, start); + } + } + } +} + +#[test] +fn graphic_duration_keeps_explicit_time_frame_and_timecode_placement() { + for command in ["graphics.newText", "graphics.newShape"] { + for mut params in [json!({"time": 0}), json!({"frame": 0}), json!({"timecode": "00:00:00:00"})] { + let mut s = Session::default(); + s.execute("file.newSequence", json!({"name": "Placement", "fps": 24, "width": 128, "height": 128, "video": 1, "audio": 1})).unwrap(); + s.execute("playhead.set", json!({"frame": 72})).unwrap(); + params["seconds"] = json!(2); + let r = s.execute(command, params).unwrap(); + let clip = ClipId(r["clip"].as_u64().unwrap()); + let (_, it) = s.active_sequence().unwrap().find_item(clip).unwrap(); + assert_eq!(it.start, Tick::ZERO, "{command}"); + assert_eq!(it.duration, Tick::from_seconds_f64(2.0)); + } + } +} + #[test] fn new_text_makes_a_graphic_clip_above_the_footage() { let mut s = demo(); diff --git a/crates/engine/src/keyboard.rs b/crates/engine/src/keyboard.rs index d6af66dc..98858e6a 100644 --- a/crates/engine/src/keyboard.rs +++ b/crates/engine/src/keyboard.rs @@ -559,7 +559,7 @@ fn export_frame(s: &mut Session, p: &Value) -> Result { }; let img = s.render_program(1.0).ok_or(EngineError::NoSequence)?; let (w, h) = (img.w as u32, img.h as u32); - let bytes = filmcraft_export::encode_still(format, img.to_rgba8(), w, h).map_err(|e| EngineError::Other(e.to_string()))?; + let bytes = filmcraft_export::encode_still(format, img.to_rgba8(), w, h, false).map_err(|e| EngineError::Other(e.to_string()))?; s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; let mut out = json!({"path": path, "width": w, "height": h}); if p.get("import").and_then(Value::as_bool).unwrap_or(false) { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 922ebdd4..443fa567 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -553,10 +553,10 @@ impl Session { let Some(p) = self.persistence.as_mut() else { return }; for ev in p.drain_events() { match ev { - autosave::WorkerEvent::SavedProject { path, revision } => { - if self.path.as_deref() == Some(path.as_str()) && revision > self.saved_revision && revision <= self.revision { - self.saved_revision = revision; - } + autosave::WorkerEvent::SavedProject { path, revision } + if self.path.as_deref() == Some(path.as_str()) && revision > self.saved_revision && revision <= self.revision => + { + self.saved_revision = revision; } autosave::WorkerEvent::Error(m) => { self.log.push(panels::Level::Error, "autosave", m.clone()); @@ -1105,3 +1105,5 @@ mod trim_tests; mod vfx_tests; #[cfg(test)] mod voiceover_tests; +#[cfg(test)] +mod wasm_clock_tests; diff --git a/crates/engine/src/masks.rs b/crates/engine/src/masks.rs index 40ab2e45..cc268ab2 100644 --- a/crates/engine/src/masks.rs +++ b/crates/engine/src/masks.rs @@ -574,7 +574,7 @@ fn track(s: &mut Session, p: &Value) -> Result { let start_path = mask.path_at(mt0); let run = move || { use std::sync::atomic::Ordering; - let t0 = std::time::Instant::now(); + let t0 = web_time::Instant::now(); let gray = |t: Tick| -> Option<(filmcraft_render::track::Prepared, f64)> { let f = src.video_frame(filmcraft_media::FrameRequest { time: t, scale }).ok()?; let g = filmcraft_render::track::Gray::from_rgba8(f.width as usize, f.height as usize, &f.to_rgba8()); diff --git a/crates/engine/src/media_browser.rs b/crates/engine/src/media_browser.rs index 60cc6c24..b7922fd3 100644 --- a/crates/engine/src/media_browser.rs +++ b/crates/engine/src/media_browser.rs @@ -663,7 +663,9 @@ pub fn std_volumes() -> Vec { v } -/// The desktop user's home directory. +/// The desktop user's home directory. Windows prefers `USERPROFILE`: shells like Git Bash set +/// `HOME` to a Unix-style path (`/c/Users/…`) that native file APIs cannot open. pub fn std_home_dir() -> Option { - std::env::var("HOME").ok().or_else(|| std::env::var("USERPROFILE").ok()).filter(|h| !h.is_empty()) + let vars = if cfg!(windows) { ["USERPROFILE", "HOME"] } else { ["HOME", "USERPROFILE"] }; + vars.iter().find_map(|v| std::env::var(v).ok().filter(|h| !h.is_empty())) } diff --git a/crates/engine/src/mixer.rs b/crates/engine/src/mixer.rs index 83e48aff..75c8042a 100644 --- a/crates/engine/src/mixer.rs +++ b/crates/engine/src/mixer.rs @@ -1129,19 +1129,19 @@ pub fn audio_gain(s: &mut Session, p: &Value) -> Result { } m => return Err(bad("clip.audioGain", format!("unknown mode `{m}` (set, adjust, normalizeMax, normalizeAll)"))), }; - let n = new_gain.len(); - let applied = new_gain.clone(); - s.edit_sequence("Audio Gain", move |q, _, _| { + let applied = s.edit_sequence("Audio Gain", move |q, _, _| { + let mut applied = Vec::new(); for t in q.audio_tracks.iter_mut() { for i in t.items.iter_mut() { if let Some((_, g)) = new_gain.iter().find(|(c, _)| *c == i.id) { i.gain_db = g.clamp(-96.0, 96.0); + applied.push((i.id, i.gain_db)); } } } - Ok(()) + Ok(applied) })?; - Ok(json!({"mode": mode, "clips": n, "gainDb": applied.iter().map(|(c, g)| json!({"clip": c.0, "gainDb": g})).collect::>()})) + Ok(json!({"mode": mode, "clips": applied.len(), "gainDb": applied.iter().map(|(c, g)| json!({"clip": c.0, "gainDb": g})).collect::>()})) } /// Effects panel ▸ Set Selected as Default Transition (video or audio, from the effect's kind). diff --git a/crates/engine/src/mixer_tests.rs b/crates/engine/src/mixer_tests.rs index 3843d018..faf9243a 100644 --- a/crates/engine/src/mixer_tests.rs +++ b/crates/engine/src/mixer_tests.rs @@ -312,6 +312,45 @@ fn audio_gain_modes_and_peak() { assert!(s.execute("clip.audioGain", json!({"mode": "loud"})).is_err()); } +#[test] +fn audio_gain_reports_only_audio_items_it_updates() { + for mode in ["set", "adjust"] { + for target in [29, 30] { + for linked in [true, false] { + let mut s = demo(); + s.execute("sequence.linkedSelection", json!({"on": linked})).unwrap(); + let r = s.execute("clip.audioGain", json!({"clips": [target], "mode": mode, "db": -6})).unwrap(); + let applied = linked || target == 30; + assert_eq!(r["clips"], usize::from(applied), "{mode}, target {target}, linked {linked}: {r}"); + assert_eq!(r["gainDb"], if applied { json!([{"clip": 30, "gainDb": -6.0}]) } else { json!([]) }); + let q = s.active_sequence().unwrap(); + assert_eq!(q.find_item(ClipId(29)).unwrap().1.gain_db, 0.0); + assert_eq!(q.find_item(ClipId(30)).unwrap().1.gain_db, if applied { -6.0 } else { 0.0 }); + if applied { + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.active_sequence().unwrap().find_item(ClipId(30)).unwrap().1.gain_db, 0.0); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(s.active_sequence().unwrap().find_item(ClipId(30)).unwrap().1.gain_db, -6.0); + } + } + } + } +} + +#[test] +fn audio_gain_reports_the_clamped_stored_gain() { + for mode in ["set", "adjust"] { + for db in [-200.0_f64, 200.0] { + let mut s = demo(); + s.execute("sequence.linkedSelection", json!({"on": false})).unwrap(); + let r = s.execute("clip.audioGain", json!({"clips": [30], "mode": mode, "db": db})).unwrap(); + let stored = s.active_sequence().unwrap().find_item(ClipId(30)).unwrap().1.gain_db; + assert_eq!(stored, db.clamp(-96.0, 96.0)); + assert_eq!(r["gainDb"], json!([{"clip": 30, "gainDb": stored}])); + } + } +} + #[test] fn default_audio_transition_is_used_by_apply() { let mut s = demo(); diff --git a/crates/engine/src/presets.rs b/crates/engine/src/presets.rs index 07de7812..75f71144 100644 --- a/crates/engine/src/presets.rs +++ b/crates/engine/src/presets.rs @@ -550,13 +550,14 @@ fn export(s: &mut Session, p: &Value) -> Result { if chosen.is_empty() { return Err(bad("presets.export", "no presets to export")); } - filmcraft_format::atomic_write(Path::new(path), &file_bytes(&chosen)).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; + // through the host: atomic on the desktop, a download on the web (no filesystem there) + s.services.write_file(path, &file_bytes(&chosen)).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; Ok(json!({"path": path, "count": chosen.len()})) } fn import(s: &mut Session, p: &Value) -> Result { let path = str_p(p, "path").ok_or_else(|| bad("presets.import", "need `path`"))?; - let bytes = std::fs::read(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; let presets = parse_file(&bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; let names: Vec = presets.iter().map(|x| x.name.clone()).collect(); for pr in presets { diff --git a/crates/engine/src/presets_tests.rs b/crates/engine/src/presets_tests.rs index 57c642e7..db8065b3 100644 --- a/crates/engine/src/presets_tests.rs +++ b/crates/engine/src/presets_tests.rs @@ -1,7 +1,10 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + use serde_json::json; -use crate::Session; use crate::presets::{KeyframeMode, retime}; +use crate::{Services, Session}; use filmcraft_project::{ClipId, ParamValue, TrackItem}; use filmcraft_time::{TICKS_PER_SECOND, Tick}; @@ -203,3 +206,35 @@ fn mask_geometry_scales_to_the_target_frame_size() { assert!((c.x - 480.0).abs() < 1e-6 && (c.y - 270.0).abs() < 1e-6, "{c:?}"); assert_eq!(m.feather.value, ParamValue::Float(80.0)); } + +/// A host without a filesystem (the web): files exist only through `Services`. +#[derive(Default)] +struct MemFs(Mutex>>); + +impl Services for MemFs { + fn read_file(&self, path: &str) -> std::io::Result> { + self.0.lock().unwrap().get(path).cloned().ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, path.to_string())) + } + fn write_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> { + self.0.lock().unwrap().insert(path.into(), data.to_vec()); + Ok(()) + } +} + +/// #379: export and import go through the host's services, not `std::fs` (which traps on the web). +#[test] +fn export_and_import_go_through_host_services() { + let fs = Arc::new(MemFs::default()); + let mut s = Session::new(fs.clone()); + let path = "/exports/owned-preset.prfpset"; + let r = s.execute("presets.export", json!({"path": path, "names": ["Fade In"]})).unwrap(); + assert_eq!(r["count"], 1); + let bytes = fs.read_file(path).unwrap(); + assert!(String::from_utf8_lossy(&bytes).contains("filmcraft.effect-presets")); + // a second session on the same host imports it from there + let mut s2 = Session::new(fs); + let r = s2.execute("presets.import", json!({"path": path})).unwrap(); + assert_eq!(r["imported"], json!(["Fade In"])); + assert_eq!(s2.presets.user.len(), 1); + assert!(s2.execute("presets.import", json!({"path": "/exports/missing.prfpset"})).is_err()); +} diff --git a/crates/engine/src/previews.rs b/crates/engine/src/previews.rs index bb1bd78d..8533f655 100644 --- a/crates/engine/src/previews.rs +++ b/crates/engine/src/previews.rs @@ -480,7 +480,7 @@ pub fn render(s: &mut Session, mode: RenderMode, p: &Value) -> Result { let nseg = todo.len(); let pool = s.media.clone(); let run = move || { - let t0 = std::time::Instant::now(); + let t0 = web_time::Instant::now(); let mut bytes = 0u64; let mut done_frames = 0u64; let mut outcome: std::result::Result<(), String> = Ok(()); @@ -581,7 +581,7 @@ pub fn render_audio(s: &mut Session, p: &Value) -> Result { let (prog, res) = (job.progress.clone(), job.result.clone()); let nseg = todo.len(); let run = move || { - let t0 = std::time::Instant::now(); + let t0 = web_time::Instant::now(); let mut bytes = 0u64; let mut outcome: std::result::Result<(), String> = Ok(()); let Some(q) = project.sequence(seq) else { return }; diff --git a/crates/engine/src/project_manager.rs b/crates/engine/src/project_manager.rs index 756c3b4e..21f3b122 100644 --- a/crates/engine/src/project_manager.rs +++ b/crates/engine/src/project_manager.rs @@ -208,7 +208,7 @@ pub fn run(s: &mut Session, p: &Value) -> Result { let prev_dst = crate::previews::dir_for_project(&project_path); let services = s.services.clone(); let run = move || { - let t0 = std::time::Instant::now(); + let t0 = web_time::Instant::now(); let r = execute(proj, work, sources, previews.map(|d| (d, prev_dst)), &project_path, pr, &prog, &*services); let secs = t0.elapsed().as_secs_f64(); let r = r.map(|bytes| filmcraft_export::Report { diff --git a/crates/engine/src/proxies.rs b/crates/engine/src/proxies.rs index 55d131ee..4786f17b 100644 --- a/crates/engine/src/proxies.rs +++ b/crates/engine/src/proxies.rs @@ -200,7 +200,7 @@ pub fn start_job(s: &mut Session, label: String, work: Vec<(ItemId, String, &'st let outputs: Arc>> = Arc::default(); let (prog, res, outs) = (job.progress.clone(), job.result.clone(), outputs.clone()); let run = move || { - let t0 = std::time::Instant::now(); + let t0 = web_time::Instant::now(); let mut bytes = 0; let mut err = None; let n = tasks.len(); diff --git a/crates/engine/src/sequence_tools_tests.rs b/crates/engine/src/sequence_tools_tests.rs index d9df881f..aa81758c 100644 --- a/crates/engine/src/sequence_tools_tests.rs +++ b/crates/engine/src/sequence_tools_tests.rs @@ -483,3 +483,24 @@ fn selection_follows_playhead() { s.execute("playhead.set", json!({"frame": 10})).unwrap(); assert_eq!(s.state.selection, sel, "off again"); } + +/// #208: with no In / Out mark, Go to In / Go to Out move to the start / end of the sequence +/// (they did nothing); with marks they go to them. +#[test] +fn go_to_in_out_without_marks_go_to_the_sequence_ends() { + let mut s = demo(); + let end = s.active_sequence().unwrap().duration(); + let mid = filmcraft_time::Tick(end.0 / 2); + s.execute("playhead.set", json!({"time": mid.0})).unwrap(); + s.execute("markers.goToIn", json!({})).unwrap(); + assert_eq!(s.playhead(), filmcraft_time::Tick::ZERO, "the start"); + s.execute("markers.goToOut", json!({})).unwrap(); + assert_eq!(s.playhead(), end, "the end"); + // with marks, the marks + let at = filmcraft_time::Tick(end.0 / 4); + s.execute("playhead.set", json!({"time": at.0})).unwrap(); + s.execute("markers.markIn", json!({})).unwrap(); + s.execute("playhead.set", json!({"time": mid.0})).unwrap(); + s.execute("markers.goToIn", json!({})).unwrap(); + assert_eq!(s.playhead(), s.active_sequence().unwrap().mark_in.unwrap()); +} diff --git a/crates/engine/src/settings.rs b/crates/engine/src/settings.rs index f4216067..fbc07fb3 100644 --- a/crates/engine/src/settings.rs +++ b/crates/engine/src/settings.rs @@ -955,7 +955,7 @@ static CATEGORIES: &[Category] = &[ b("playback.draftDecode", "Draft decoding at reduced playback resolution (H.264: faster, some frames less filtered)", true), f("playback.hardwareDecoding", "Hardware decoding", Kind::Choice(HW_DECODE), true), Row::Note( - "Hardware decoding: Auto uses the system's video decoder (VideoToolbox on macOS) for the H.264 and HEVC streams it supports, and FilmCraft's own decoder for everything else or if the hardware fails. Media that is already open keeps its decoder until it is reopened.", + "Hardware decoding: Auto uses the system's video decoder (VideoToolbox on macOS, Media Foundation on Windows, VA-API for H.264 and HEVC on Linux) for the streams it supports, and FilmCraft's own decoder for everything else or if the hardware fails. Media that is already open keeps its decoder until it is reopened.", ), ], }, @@ -1084,10 +1084,8 @@ pub fn sanitize(v: &mut Value, defaults: &Value) { let Some(slot) = path.iter().try_fold(&mut *v, |v, k| v.get_mut(*k)) else { continue }; let def = path.iter().try_fold(defaults, |v, k| v.get(*k)).cloned().unwrap_or(Value::Null); match f.kind { - Kind::Choice(_) | Kind::Color => { - if validate(f.key, slot).is_err() { - *slot = def; - } + Kind::Choice(_) | Kind::Color if validate(f.key, slot).is_err() => { + *slot = def; } Kind::Int { min, max, .. } => match slot.as_f64().filter(|x| x.is_finite()) { Some(x) => *slot = json!(x.round().clamp(min, max) as u64), diff --git a/crates/engine/src/shortcut_presets.rs b/crates/engine/src/shortcut_presets.rs index 118f2991..7e105e50 100644 --- a/crates/engine/src/shortcut_presets.rs +++ b/crates/engine/src/shortcut_presets.rs @@ -11,6 +11,9 @@ pub type Entry = (&'static str, &'static str, &'static str); /// FilmCraft additions to the per-command defaults: panel-specific shortcuts. pub const FILMCRAFT_PANEL: &[Entry] = &[ ("project.delete", "Backspace", "Project"), + // the forward-delete key (Delete on Windows and Linux keyboards) clears timeline clips too; + // application-wide it is Project ▸ Clear (#243) + ("edit.clear", "Delete", "Timeline"), ("file.newBin", "Cmd+B", "Project"), ("edit.undo", "Left", "History"), ("edit.redo", "Right", "History"), diff --git a/crates/engine/src/shortcuts_tests.rs b/crates/engine/src/shortcuts_tests.rs index 520a73e5..1f98b8ec 100644 --- a/crates/engine/src/shortcuts_tests.rs +++ b/crates/engine/src/shortcuts_tests.rs @@ -132,6 +132,19 @@ fn panel_shortcuts_override_application_ones_with_focus() { assert!(k.as_array().unwrap().iter().any(|b| b["command"] == "edit.copy" && b["keys"] == "Ctrl+C"), "{k}"); } +#[test] +fn delete_clears_timeline_clips_and_project_items() { + // #243: with the Timeline focused, Delete (the forward-delete key, labelled Delete on Windows + // and Linux keyboards) resolved to Project ▸ Clear and left the selected clips in place. + let s = Session::default(); + let del = Chord::parse("Delete").unwrap(); + for p in [Platform::Mac, Platform::Windows] { + assert_eq!(s.shortcuts.resolve(&del, Some("Timeline"), p).unwrap().command, "edit.clear", "{p:?}"); + assert_eq!(s.shortcuts.resolve(&del, Some("Project"), p).unwrap().command, "project.delete", "{p:?}"); + assert_eq!(s.shortcuts.resolve(&del, None, p).unwrap().command, "project.delete", "{p:?}"); + } +} + #[test] fn compat_presets_map_other_editors() { let mut s = Session::default(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 4193afe9..01d5b813 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -144,6 +144,73 @@ fn sequence_parameters_are_bounded_and_failed_changes_are_atomic() { assert_eq!(s.active_sequence().unwrap().settings.height, 8192, "a 16384x8192 panorama is accepted"); } +/// The first video clip's Motion Position and Scale (static values). +fn first_motion(s: &Session) -> (u64, (f64, f64), f64) { + let q = s.active_sequence().unwrap(); + let it = q.video_tracks.iter().flat_map(|t| &t.items).find(|i| i.effect("motion").is_some()).unwrap(); + let m = it.effect("motion").unwrap(); + let p = match m.param("position").unwrap().value { + filmcraft_project::ParamValue::Vec2(v) => (v.x, v.y), + _ => panic!("position is a point"), + }; + let sc = match m.param("scale").unwrap().value { + filmcraft_project::ParamValue::Float(v) => v, + _ => panic!("scale is a number"), + }; + (it.id.0, p, sc) +} + +#[test] +fn sequence_settings_scales_motion_with_the_frame_size_in_one_undo_step() { + let mut s = demo(); + let before = (*s.project).clone(); + let (id, p0, s0) = first_motion(&s); + assert_eq!((s.active_sequence().unwrap().settings.width, s.active_sequence().unwrap().settings.height), (1920, 1080)); + let r = s.execute("sequence.settings", json!({"width":1280,"height":720,"scaleMotion":true})).unwrap(); + assert!(r["scaledClips"].as_u64().unwrap() >= 1, "{r}"); + let (id1, p1, s1) = first_motion(&s); + assert_eq!(id1, id); + assert!((p1.0 - p0.0 * 2.0 / 3.0).abs() < 1e-9 && (p1.1 - p0.1 * 2.0 / 3.0).abs() < 1e-9, "{p0:?} -> {p1:?}"); + assert!((s1 - s0 * 2.0 / 3.0).abs() < 1e-9, "{s0} -> {s1}"); + let after = (*s.project).clone(); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(*s.project, before, "size and Motion come back in one undo step"); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(*s.project, after); + // without the option the frame size changes alone (the command's default, as before) + let r = s.execute("sequence.settings", json!({"width":3840,"height":2160})).unwrap(); + assert_eq!(r["scaledClips"], json!(0)); + assert_eq!(first_motion(&s), (id, p1, s1)); + assert_eq!(s.active_sequence().unwrap().settings.width, 3840); +} + +#[test] +fn sequence_settings_timecode_render_quality_and_colour() { + let mut s = demo(); + s.execute("sequence.settings", json!({"fps":29.97,"dropFrame":true,"maxRenderQuality":true})).unwrap(); + let st = s.active_sequence().unwrap().settings.clone(); + assert_eq!(st.frame_rate, FrameRate::FPS_29_97); + assert!(st.drop_frame && st.max_render_quality); + s.execute("sequence.settings", json!({"dropFrame":false})).unwrap(); + assert!(!s.active_sequence().unwrap().settings.drop_frame); + // drop-frame only exists for the NTSC rates; a refused change leaves everything as it was + let before = (*s.project).clone(); + let history = s.history.undo.len(); + for params in [json!({"fps":25,"dropFrame":true}), json!({"workingSpace":"rec2020-log"}), json!({"fps":25,"dropFrame":true,"width":1280})] { + assert!(s.execute("sequence.settings", params.clone()).is_err(), "{params}"); + assert_eq!(*s.project, before, "{params}"); + assert_eq!(s.history.undo.len(), history); + } + s.execute("sequence.settings", json!({"workingSpace":"rec2100-pq","wideGamut":true,"autoToneMap":true})).unwrap(); + let st = s.active_sequence().unwrap().settings.clone(); + assert_eq!(st.color.working, filmcraft_color::WorkingSpace::Rec2100Pq); + assert!(st.color.wide_gamut && st.color.auto_tone_map); + assert_eq!(st.working_space, "Rec. 2100 PQ"); + // disabled without a sequence + let mut empty = Session::default(); + assert!(empty.execute("sequence.settings", json!({"width":1280})).is_err()); +} + #[test] fn zero_fps_is_refused() { let mut s = demo(); @@ -170,6 +237,51 @@ fn effects_and_keyframes() { assert!(it.effect("motion").unwrap().params["scale"].is_animated()); } +/// `effects.addKeyframe` is the keyframe diamond of Effect Controls and Properties: it adds a +/// keyframe at the playhead or removes the one there, and the value then follows the keyframes +/// that are left (Premiere's Add/Remove Keyframe), unlike the stopwatch, which drops them all. +#[test] +fn add_keyframe_toggles_the_keyframe_at_the_playhead() { + let mut s = demo(); + let c = s.active_sequence().unwrap().video_tracks[0].items[0].id.0; + let scale = |s: &Session| { + let it = s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1; + let p = &it.effect("motion").unwrap().params["scale"]; + (p.keyframes.len(), p.f64_at(it.source_time_at(s.playhead()))) + }; + let key = json!({"clip": c, "effect": "motion", "param": "scale"}); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + s.execute("effects.addKeyframe", key.clone()).unwrap(); + assert_eq!(scale(&s), (1, 100.0), "the first keyframe turns animation on"); + s.execute("playhead.set", json!({"seconds": 2.5})).unwrap(); + s.execute("effects.addKeyframe", key.clone()).unwrap(); + s.execute("effects.setParam", json!({"clip": c, "effect": "motion", "param": "scale", "value": 50.0})).unwrap(); + assert_eq!(scale(&s), (2, 50.0)); + // on a keyframe: only that keyframe goes, and Scale is the other keyframe's 100 again + s.execute("effects.addKeyframe", key.clone()).unwrap(); + assert_eq!(scale(&s), (1, 100.0)); + s.undo(); + assert_eq!(scale(&s), (2, 50.0)); + s.redo(); + assert_eq!(scale(&s), (1, 100.0)); + // parameters that name nothing are errors and change nothing + for bad in [ + json!({}), + json!({"clip": c}), + json!({"clip": u64::MAX, "param": "scale"}), + json!({"clip": c, "effect": "nope", "param": "scale"}), + json!({"clip": c, "effect": u64::MAX, "param": "scale"}), + json!({"clip": c, "effect": "motion", "param": "nope"}), + json!({"clip": c, "effect": "motion", "param": "scale", "mask": 7}), + ] { + assert!(s.execute("effects.addKeyframe", bad.clone()).is_err(), "{bad}"); + } + assert_eq!(scale(&s), (1, 100.0)); + // the stopwatch ends the animation and keeps the value at the playhead + s.execute("effects.toggleAnimation", key).unwrap(); + assert_eq!(scale(&s), (0, 100.0)); +} + #[test] fn transitions_and_markers() { let mut s = demo(); diff --git a/crates/engine/src/wasm_clock_tests.rs b/crates/engine/src/wasm_clock_tests.rs new file mode 100644 index 00000000..6a2a13fc --- /dev/null +++ b/crates/engine/src/wasm_clock_tests.rs @@ -0,0 +1,53 @@ +//! The web build runs every job synchronously (`wait || cfg!(target_arch = "wasm32")`), so a job +//! body executes on wasm32 — where `std::time::Instant::now()` panics with "time not implemented +//! on this platform" and takes the whole app down (#230: Create Proxies and mask tracking both +//! did). `web_time::Instant` is `std::time::Instant` on native targets and `performance.now()` +//! on the web, so there is never a reason to name the std one in this crate. + +use std::path::Path; + +/// Every non-test source file of the crate, as (path, contents). +fn sources() -> Vec<(String, String)> { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut out = Vec::new(); + let mut stack = vec![dir]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(&d).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + continue; + } + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default(); + if !name.ends_with(".rs") || name.ends_with("_tests.rs") || name == "media_test_util.rs" { + continue; + } + out.push((p.display().to_string(), std::fs::read_to_string(&p).unwrap())); + } + } + assert!(out.len() > 20, "found {} source files", out.len()); + out +} + +#[test] +fn no_std_instant_in_code_the_web_build_runs() { + let mut hits = Vec::new(); + for (path, text) in sources() { + for (i, line) in text.lines().enumerate() { + // `use std::time::{.., Instant, ..}` or a fully qualified `std::time::Instant` + let names_std_instant = line.contains("std::time::Instant") || (line.contains("use std::time::") && line.contains("Instant")); + if names_std_instant && !line.contains("web_time") && !line.trim_start().starts_with("//") { + hits.push(format!("{path}:{}: {}", i + 1, line.trim())); + } + } + } + assert!(hits.is_empty(), "std::time::Instant panics on wasm32; use web_time::Instant:\n{}", hits.join("\n")); +} + +/// The clock the jobs use works here as well: a job's "Done in 0.0s" report needs an elapsed time. +#[test] +fn web_time_instant_measures_elapsed_time() { + let t0 = web_time::Instant::now(); + let secs = t0.elapsed().as_secs_f64(); + assert!((0.0..60.0).contains(&secs), "{secs}"); +} diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 419a7368..008c3703 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -253,6 +253,10 @@ pub struct ExportSettings { /// sequence's HDR space and signal it (VUI / `colr` / `mdcv` / `clli` / SEI). #[serde(default)] pub sdr: bool, + /// Keep the alpha channel in PNG and TIFF sequences (straight alpha, Premiere's "Include + /// Alpha Channel"). Off, every frame is flattened over black (#160). Other formats ignore it. + #[serde(default)] + pub alpha: bool, /// Output frame size (None = Match Source: the sequence size times `scale`). pub frame_size: Option<(u32, u32)>, /// Output frame rate (None = Match Source). @@ -445,6 +449,7 @@ impl Default for ExportSettings { apv_profile: String::new(), mxf_video_codec: MxfVideoCodec::default(), sdr: false, + alpha: false, frame_size: None, frame_rate: None, scaling: Scaling::default(), @@ -1266,11 +1271,12 @@ impl Write for SharedBuf { /// Encode straight sRGB RGBA8 pixels as a PNG file (thumbnails, previews). pub fn encode_png(rgba: Vec, w: u32, h: u32) -> Result> { - encode_still(Format::PngSequence, rgba, w, h) + encode_still(Format::PngSequence, rgba, w, h, false) } -/// Encode one still of an image sequence (also Export Frame). -pub fn encode_still(format: Format, rgba: Vec, w: u32, h: u32) -> Result> { +/// Encode one still of an image sequence (also Export Frame). PNG always carries the alpha +/// channel it is given; `alpha` keeps it in TIFF too, which is otherwise written as RGB. +pub fn encode_still(format: Format, rgba: Vec, w: u32, h: u32, alpha: bool) -> Result> { let enc = |e: image::ImageError| ExportError::Encode(e.to_string()); let mut out = std::io::Cursor::new(Vec::new()); match format { @@ -1279,9 +1285,12 @@ pub fn encode_still(format: Format, rgba: Vec, w: u32, h: u32) -> Result { let img = image::RgbaImage::from_raw(w, h, rgba).ok_or_else(|| ExportError::Encode("frame size".into()))?; - let rgb = image::DynamicImage::ImageRgba8(img).to_rgb8(); let f = if format == Format::TiffSequence { image::ImageFormat::Tiff } else { image::ImageFormat::Bmp }; - rgb.write_to(&mut out, f).map_err(enc)? + if alpha && format == Format::TiffSequence { + img.write_to(&mut out, f).map_err(enc)? + } else { + image::DynamicImage::ImageRgba8(img).to_rgb8().write_to(&mut out, f).map_err(enc)? + } } _ => return Err(ExportError::Unsupported(format!("{} is not an image sequence", format.label()))), } @@ -1358,7 +1367,7 @@ pub fn export(project: &Arc, seq: ItemId, settings: &ExportSettings, so let written: Vec> = (f..end) .into_par_iter() .map(|fi| { - let data = encode_still(settings.format, pipe.frame(fi, sources).0, w, h)?; + let data = encode_still(settings.format, pipe.frame(fi, sources).0, w, h, settings.alpha)?; write_output(settings, &pcm::image_sequence_path(&settings.path, (fi - f0) as u64, count), data) }) .collect(); diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index fde70f83..c41f28e1 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -12,7 +12,7 @@ use filmcraft_time::FrameRate; use rayon::prelude::*; use crate::settings::{ExportEffects, Placement, Scaling, TextOverlay}; -use crate::{ExportError, ExportSettings, Result}; +use crate::{ExportError, ExportSettings, Format, Result}; /// Where the rendered picture lands in the output frame. #[derive(Clone, Copy, Debug, PartialEq)] @@ -61,6 +61,8 @@ pub(crate) struct Pipeline { geom: Geometry, opts: RenderOptions, pub hdr_out: bool, + /// Keep straight alpha in the RGBA8 output instead of flattening over black. + alpha: bool, out_tf: Option, effects: ExportEffects, overlay: Option, @@ -106,6 +108,7 @@ impl Pipeline { geom, opts, hdr_out, + alpha: settings.alpha && matches!(settings.format, Format::PngSequence | Format::TiffSequence), out_tf, effects, overlay, @@ -134,7 +137,7 @@ impl Pipeline { filmcraft_frame::pool::recycle_f32(img.px); return (Vec::new(), out); } - let mut rgba = img.over_black_rgba8(); + let mut rgba = if self.alpha { img.to_rgba8() } else { img.over_black_rgba8() }; // the float image is not needed any more: its buffer serves the next frame's layers filmcraft_frame::pool::recycle_f32(img.px); if lim.enabled { diff --git a/crates/export/src/settings_tests.rs b/crates/export/src/settings_tests.rs index b4fcbc57..f0841484 100644 --- a/crates/export/src/settings_tests.rs +++ b/crates/export/src/settings_tests.rs @@ -109,6 +109,28 @@ fn image_sequences_are_numbered_like_premiere() { } } +/// `alpha` keeps straight alpha in PNG and TIFF sequences; off (the default) flattens over black (#160). +#[test] +fn image_sequences_keep_alpha_when_asked() { + let (p, seq, m) = matte([0.0, 0.0, 1.0, 0.5], 64, 36, None); + for (fmt, ext) in [(Format::PngSequence, "png"), (Format::TiffSequence, "tif")] { + for alpha in [false, true] { + let dir = Scratch::new(&format!("alpha-{ext}-{alpha}")); + let mut s = ExportSettings { format: fmt, path: dir.path(&format!("a.{ext}")), alpha, ..Default::default() }; + s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(2))); + export(&p, seq, &s, &m, &Progress::default()).unwrap(); + let img = image::open(dir.0.join(format!("a000.{ext}"))).unwrap(); + let px = img.to_rgba8().get_pixel(32, 18).0; + if alpha { + assert!(img.color().has_alpha(), "{fmt:?} must be written with an alpha channel"); + assert!((100..=160).contains(&px[3]) && px[2] > 200 && px[0] < 15, "{fmt:?} straight alpha: {px:?}"); + } else { + assert!(px[3] == 255 && px[2] > 100 && px[2] < 240 && px[0] < 15, "{fmt:?} flattened over black: {px:?}"); + } + } + } +} + #[test] fn wav_and_aiff_audio_only() { let (p, seq, m) = matte([0.0, 0.0, 0.0, 1.0], 64, 36, Some(-6.0)); diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index f71db2b6..a704a817 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -505,15 +505,28 @@ impl GpuCompositor { /// The source description of an uploaded frame. fn src_info(&self, f: &VideoFrame, key: (usize, u32, u32)) -> Option { let up = self.uploads.get(&key)?; - Some(SrcInfo { kind: up.kind, code_scale: up.code_scale, alpha: up.alpha_scale, chroma: up.chroma, size: (f.width, f.height), color: f.color }) + let code_levels = match &f.data { + PixelData::Yuv8 { .. } => 256.0, + PixelData::Yuv16 { bits, .. } => (*bits as f32).exp2(), + _ => 1.0, + }; + Some(SrcInfo { + kind: up.kind, + code_scale: up.code_scale, + code_levels, + alpha: up.alpha_scale, + chroma: up.chroma, + size: (f.width, f.height), + color: f.color, + }) } fn uniforms(src: &SrcInfo, m: &filmcraft_geom::Affine, opacity: f32, blend: Blend, out: (u32, u32)) -> [f32; 28] { let (kr, kb) = src.color.matrix.kr_kb(); - let bits_scale = src.code_scale / 255.0; // code units relative to 8-bit + let bits_scale = src.code_levels / 256.0; // 2^(bits - 8), independent of texture encoding let (yo, ys, co, cs) = match (src.color.range, src.kind) { (Range::Limited, 2) => (16.0 * bits_scale, 219.0 * bits_scale, 128.0 * bits_scale, 224.0 * bits_scale), - (Range::Full, 2) => (0.0, src.code_scale - 1.0, src.code_scale / 2.0, src.code_scale - 1.0), + (Range::Full, 2) => (0.0, src.code_levels - 1.0, src.code_levels / 2.0, src.code_levels - 1.0), _ => (0.0, 1.0, 0.0, 1.0), }; let transfer = match src.color.transfer { @@ -645,6 +658,7 @@ impl GpuCompositor { let src = self.src_info(&l.frame, *k).unwrap_or(SrcInfo { kind: 1, code_scale: 1.0, + code_levels: 1.0, alpha: 0.0, chroma: (1, 1), size: (l.frame.width, l.frame.height), @@ -678,7 +692,7 @@ impl GpuCompositor { // what the layer draws: the effect result (linear premultiplied RGBA) or the frame let (u, tex) = match &job { Some(j) => { - let s = SrcInfo { kind: 1, code_scale: 1.0, alpha: 0.0, chroma: l.size(), size: l.size(), color: l.frame.color }; + let s = SrcInfo { kind: 1, code_scale: 1.0, code_levels: 1.0, alpha: 0.0, chroma: l.size(), size: l.size(), color: l.frame.color }; (Self::uniforms(&s, &l.matrix, l.opacity, l.blend, (w, h)), [j.result.clone(), self.dummy.clone(), self.dummy.clone(), self.dummy.clone()]) } None => { @@ -884,11 +898,13 @@ impl GpuCompositor { } } -/// What a layer's shader samples: the texture kind (0 RGBA8 sRGB, 1 linear premultiplied RGBA -/// float, 2 YUV planes), code scale, chroma plane size, picture size and colour description. +/// Shader source: kind (0 RGBA8 sRGB, 1 linear premultiplied RGBA, 2 YUV), sampling scales, dimensions, color. struct SrcInfo { kind: u32, + /// Texture sample -> code units: 255 for R8Unorm, 2^bits for R16Float. code_scale: f32, + /// YUV quantization levels: 2^bits; R8Unorm's decoding scale is one less. + code_levels: f32, /// Alpha texture sample → alpha (0: the frame has no alpha plane). alpha: f32, chroma: (u32, u32), diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index a11cef8b..982c7b62 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -9,6 +9,83 @@ fn device() -> Option<(wgpu::Device, wgpu::Queue)> { pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok() } +#[test] +fn yuv_code_range_uniforms() { + let mut mismatches = Vec::new(); + for bits in [8, 10, 12, 16] { + let levels = (1u32 << bits) as f32; + for range in [Range::Full, Range::Limited] { + let src = SrcInfo { + kind: 2, + code_scale: if bits == 8 { 255.0 } else { levels }, + code_levels: levels, + alpha: 0.0, + chroma: (1, 1), + size: (1, 1), + color: filmcraft_color::ColorInfo { range, ..filmcraft_color::ColorInfo::REC709 }, + }; + let u = GpuCompositor::uniforms(&src, &Affine::IDENTITY, 1.0, Blend::Normal, (1, 1)); + // Full: 0..2^bits-1, neutral 2^(bits-1). Limited: BT.709 8-bit codes * 2^(bits-8). + let expected = match range { + Range::Full => [0.0, levels - 1.0, levels / 2.0, levels - 1.0], + Range::Limited => [16.0, 219.0, 128.0, 224.0].map(|v| v * (1u32 << (bits - 8)) as f32), + }; + if u[16..20] != expected { + mismatches.push(format!("{bits}-bit {range:?}: {:?} != {expected:?}", &u[16..20])); + } + } + } + assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); +} + +#[test] +fn yuv_code_range_gpu_matches_cpu() { + let Some((dev, q)) = device() else { + eprintln!("no GPU adapter; skipping"); + return; + }; + let mut c = GpuCompositor::new(&dev, &q); + let mut prepared_compositor = GpuCompositor::new(&dev, &q); + for bits in [8, 10, 12, 16] { + let scale = 1u32 << (bits - 8); + for range in [Range::Full, Range::Limited] { + // Constant chroma isolates range normalization from CPU/GPU chroma reconstruction. + for (u, v) in [(128, 128), (64, 192), (192, 64)] { + let y = (0..=255).map(|code| (code * scale) as u16).collect::>(); + let planes = [Arc::new(y), Arc::new(vec![(u * scale) as u16; 256]), Arc::new(vec![(v * scale) as u16; 256])]; + let data = if bits == 8 { + PixelData::Yuv8 { planes: planes.map(|p| Arc::new(p.iter().map(|v| *v as u8).collect())), chroma: Chroma::C444, alpha: None } + } else { + PixelData::Yuv16 { planes, chroma: Chroma::C444, bits, alpha: None } + }; + let frame = Arc::new(VideoFrame { + width: 256, + height: 1, + data, + color: filmcraft_color::ColorInfo { range, ..filmcraft_color::ColorInfo::REC709 }, + par: (1, 1), + pts: Default::default(), + }); + let cpu = frame.to_rgba8(); + let plan = FramePlan::Layers { width: 256, height: 1, layers: vec![PlanLayer::new(frame, Affine::IDENTITY, 1.0, Blend::Normal)] }; + c.composite(&plan); + let (w, h, gpu) = c.read_output().expect("readback"); + assert_eq!((w, h, gpu.len()), (256, 1, cpu.len())); + let error = cpu.iter().zip(&gpu).map(|(a, b)| a.abs_diff(*b)).max().expect("pixels"); + // One output-code tolerance: CPU f32 vs GPU f16 accumulator quantization. + assert!(error <= 1, "{bits}-bit {range:?} U={u} V={v}: max error {error}"); + if u == 128 && v == 128 { + assert!(gpu.as_chunks::<4>().0.iter().all(|p| p[0] == p[1] && p[1] == p[2]), "neutral tint: {bits}-bit {range:?}"); + } + let prepared = prepare(&plan); + prepared_compositor.composite_prepared(&plan, Some(&prepared)); + assert_eq!(prepared_compositor.read_output().expect("prepared readback").2, gpu, "prepared: {bits}-bit {range:?}"); + eprintln!("range gate: {bits}-bit {range:?} U={u} V={v}, max error={error}, prepared=inline"); + } + } + } +} + fn yuv_frame(w: u32, h: u32) -> Arc { let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); let y: Vec = (0..w * h).map(|i| (16 + ((i % w) * 219 / w)) as u8).collect(); diff --git a/crates/h264/src/decoder.rs b/crates/h264/src/decoder.rs index 5e2f810d..59070d35 100644 --- a/crates/h264/src/decoder.rs +++ b/crates/h264/src/decoder.rs @@ -6,11 +6,11 @@ //! jobs of later pictures block per row on the reference data they need, so several pictures decode //! concurrently (frame-level parallelism). -use crate::dpb::{Dpb, Output, OutputMeta}; +use crate::dpb::{Dpb, Output, OutputMeta, RefMark}; use crate::error::{Error, Result, ensure, invalid, unsupported}; use crate::params::{Pps, Sps}; use crate::picture::{Frame, FrameRef, MbKind, MbState, Planes, RefPic}; -use crate::slice::{NalHeader, Poc, PocState, SliceHeader, SliceType, nal_type}; +use crate::slice::{NalHeader, Poc, PocState, SliceHeader, SliceType, is_new_picture, nal_type}; use crate::slicedec::{PicState, SliceDecoder}; use crate::transform::LevelScale; use crate::{ColorInfo, Picture}; @@ -81,7 +81,7 @@ pub struct Decoder { spss: Vec>>, ppss: Vec>>, nal_length_size: Option, - dpb: Dpb, + dpb: Dpb, poc_state: PocState, prev_ref_frame_num: u32, pending: Option, @@ -89,7 +89,7 @@ pub struct Decoder { active_sps: Option>, ls_cache: Vec<(Arc, Arc)>, /// Pictures leaving the DPB, converted once their decoding job has finished. - out_queue: VecDeque, + out_queue: VecDeque>, shared: Arc, #[cfg(feature = "threads")] pool: Option, @@ -106,27 +106,6 @@ impl Default for Decoder { } } -/// Is `sh` the first slice of a new primary coded picture relative to `prev` (7.4.1.2.4)? -fn is_new_picture(prev: &SliceHeader, sh: &SliceHeader, sps: &Sps) -> bool { - if sh.frame_num != prev.frame_num - || sh.pps_id != prev.pps_id - || sh.field_pic != prev.field_pic - || sh.bottom_field != prev.bottom_field - || (sh.nal_ref_idc == 0) != (prev.nal_ref_idc == 0) - || sh.idr != prev.idr - || (sh.idr && prev.idr && sh.idr_pic_id != prev.idr_pic_id) - { - return true; - } - if sps.pic_order_cnt_type == 0 && (sh.pic_order_cnt_lsb != prev.pic_order_cnt_lsb || sh.delta_pic_order_cnt_bottom != prev.delta_pic_order_cnt_bottom) { - return true; - } - if sps.pic_order_cnt_type == 1 && sh.delta_pic_order_cnt != prev.delta_pic_order_cnt { - return true; - } - false -} - /// The worker threads [`Decoder::new`] uses: the available cores (at most 16) with the `threads` /// feature on native targets, 1 otherwise. pub fn default_threads() -> usize { @@ -384,7 +363,9 @@ impl Decoder { let Some(pending) = self.pending.as_mut() else { return invalid("slice without a started picture"); }; - let refs = self.dpb.build_ref_lists(&sh, pending.poc.frame(), sps.max_frame_num())?; + let refs = self + .dpb + .build_ref_lists(&sh, pending.poc.frame(), sps.max_frame_num(), |e| RefPic { frame: e.frame.clone(), long_term: e.mark == RefMark::Long })?; if !sh.slice_type.is_intra() { ensure!(!refs[0].is_empty(), "no reference pictures available for inter slice"); if sh.slice_type.is_b() { @@ -464,7 +445,7 @@ impl Decoder { } _ => Planes::gray(mb_w * 16, mb_h * 16), }; - (Arc::new(Frame::from_planes(id, 0, &planes)), meta.clone()) + (Arc::new(Frame::from_planes(id, 0, &planes)), 0, meta.clone()) }; let mut upd = |fnum: u32| poc_state.update_gap_frame(fnum, sps); self.shared.stats.lock().map(|mut s| s.frame_num_gaps += 1).ok(); @@ -516,7 +497,7 @@ impl Decoder { run_job(frame, slices, &shared, draft); } - fn emit(&mut self, outs: Vec) { + fn emit(&mut self, outs: Vec>) { self.out_queue.extend(outs); } } @@ -613,7 +594,7 @@ fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta { } } -fn make_picture(o: &Output) -> Picture { +fn make_picture(o: &Output) -> Picture { let (cx, cy, cw, ch) = o.meta.crop; let (cx, cy, cw, ch) = (cx as usize, cy as usize, cw as usize, ch as usize); let (y, u, v) = o.frame.copy_cropped((cx, cy, cw, ch)); diff --git a/crates/h264/src/dpb.rs b/crates/h264/src/dpb.rs index 691c62b1..657aa411 100644 --- a/crates/h264/src/dpb.rs +++ b/crates/h264/src/dpb.rs @@ -1,8 +1,11 @@ //! Decoded picture buffer: reference marking (8.2.5), reference list construction (8.2.4) and output //! ordering / bumping (C.4). +//! +//! The buffer only does bookkeeping on picture headers; what a picture is (`F`: the software +//! decoder's [`crate::picture::FrameRef`], a hardware decoder's surface) is up to the decoder, so +//! hardware decoders driven by the same headers (VA-API) share this logic and output order. use crate::error::{Result, ensure}; -use crate::picture::{FrameRef, RefPic}; use crate::slice::{Mmco, SliceHeader, SliceType}; use std::sync::Arc; @@ -28,8 +31,8 @@ pub struct OutputMeta { pub draft: bool, } -pub struct DpbEntry { - pub frame: FrameRef, +pub struct DpbEntry { + pub frame: F, pub poc: i32, pub frame_num: u32, pub long_term_frame_idx: u32, @@ -39,8 +42,8 @@ pub struct DpbEntry { pub meta: Arc, } -pub struct Dpb { - pub entries: Vec, +pub struct Dpb { + pub entries: Vec>, /// None = "no long-term frame indices". pub max_long_term_frame_idx: Option, pub capacity: usize, @@ -48,18 +51,18 @@ pub struct Dpb { } /// A picture leaving the DPB for output. -pub struct Output { - pub frame: FrameRef, +pub struct Output { + pub frame: F, pub poc: i32, pub meta: Arc, } -impl Dpb { +impl Dpb { pub fn new() -> Self { Dpb { entries: Vec::new(), max_long_term_frame_idx: None, capacity: 16, max_reorder: 16 } } - fn frame_num_wrap(e: &DpbEntry, cur_frame_num: u32, max_frame_num: u32) -> i32 { + fn frame_num_wrap(e: &DpbEntry, cur_frame_num: u32, max_frame_num: u32) -> i32 { if e.frame_num > cur_frame_num { e.frame_num as i32 - max_frame_num as i32 } else { e.frame_num as i32 } } @@ -73,7 +76,7 @@ impl Dpb { } /// Output the picture with the smallest POC (C.4.5.3 "bumping"). Returns false if none waits. - fn bump(&mut self, out: &mut Vec) -> bool { + fn bump(&mut self, out: &mut Vec>) -> bool { let Some(i) = self.entries.iter().enumerate().filter(|(_, e)| e.needed_for_output).min_by_key(|(_, e)| e.poc).map(|(i, _)| i) else { return false; }; @@ -85,13 +88,13 @@ impl Dpb { } /// Output everything in POC order and clear non-reference entries. - pub fn flush(&mut self, out: &mut Vec) { + pub fn flush(&mut self, out: &mut Vec>) { while self.bump(out) {} self.prune(); } /// IDR: mark all references unused; output (or drop) prior pictures. - pub fn idr(&mut self, no_output_of_prior_pics: bool, out: &mut Vec) { + pub fn idr(&mut self, no_output_of_prior_pics: bool, out: &mut Vec>) { for e in &mut self.entries { e.mark = RefMark::Unused; } @@ -122,7 +125,8 @@ impl Dpb { self.prune(); } - /// Insert "non-existing" frames for a frame_num gap (8.2.5.2). + /// Insert "non-existing" frames for a frame_num gap (8.2.5.2). `make_frame` gives each one's + /// picture and POC. #[allow(clippy::too_many_arguments)] pub fn fill_frame_num_gap( &mut self, @@ -130,17 +134,17 @@ impl Dpb { frame_num: u32, max_frame_num: u32, max_num_ref_frames: usize, - make_frame: &mut dyn FnMut(u32) -> (FrameRef, Arc), + make_frame: &mut dyn FnMut(u32) -> (F, i32, Arc), poc_state: &mut dyn FnMut(u32), ) { let mut unused = (prev_ref_frame_num + 1) % max_frame_num; let mut guard = 0; while unused != frame_num && guard < max_frame_num { self.sliding_window(unused, max_frame_num, max_num_ref_frames); - let (frame, meta) = make_frame(unused); + let (frame, poc, meta) = make_frame(unused); poc_state(unused); self.entries.push(DpbEntry { - poc: frame.poc, + poc, frame, frame_num: unused, long_term_frame_idx: 0, @@ -154,26 +158,26 @@ impl Dpb { } } - /// Initial reference picture lists (8.2.4.2) + modification (8.2.4.3) for a frame slice. - pub fn build_ref_lists(&self, sh: &SliceHeader, cur_poc: i32, max_frame_num: u32) -> Result<[Vec; 2]> { - let mut lists: [Vec; 2] = [Vec::new(), Vec::new()]; + /// Initial reference picture lists (8.2.4.2) + modification (8.2.4.3) for a frame slice, each + /// entry made by `to_ref` from the DPB entry it refers to. + pub fn build_ref_lists(&self, sh: &SliceHeader, cur_poc: i32, max_frame_num: u32, to_ref: impl Fn(&DpbEntry) -> R) -> Result<[Vec; 2]> { + let mut lists: [Vec; 2] = [Vec::new(), Vec::new()]; if sh.slice_type.is_intra() { return Ok(lists); } let cur_fn = sh.frame_num; - let pic_num = |e: &DpbEntry| Self::frame_num_wrap(e, cur_fn, max_frame_num); - let short: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Short).collect(); - let mut long: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Long).collect(); + let pic_num = |e: &DpbEntry| Self::frame_num_wrap(e, cur_fn, max_frame_num); + let short: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Short).collect(); + let mut long: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Long).collect(); long.sort_by_key(|e| e.long_term_frame_idx); - let to_ref = |e: &&DpbEntry| RefPic { frame: e.frame.clone(), long_term: e.mark == RefMark::Long }; - let mut entry_lists: [Vec<&DpbEntry>; 2] = [Vec::new(), Vec::new()]; + let mut entry_lists: [Vec<&DpbEntry>; 2] = [Vec::new(), Vec::new()]; if sh.slice_type != SliceType::B { let mut s = short.clone(); s.sort_by_key(|e| std::cmp::Reverse(pic_num(e))); entry_lists[0] = s.into_iter().chain(long.iter().copied()).collect(); } else { - let mut before: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc < cur_poc).collect(); - let mut after: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc > cur_poc).collect(); + let mut before: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc < cur_poc).collect(); + let mut after: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc > cur_poc).collect(); before.sort_by_key(|e| std::cmp::Reverse(e.poc)); after.sort_by_key(|e| e.poc); entry_lists[0] = before.iter().chain(after.iter()).chain(long.iter()).copied().collect(); @@ -188,7 +192,7 @@ impl Dpb { let nlists = if sh.slice_type == SliceType::B { 2 } else { 1 }; for l in 0..nlists { let n = sh.num_ref_idx_active[l] as usize; - let mut list: Vec> = entry_lists[l].iter().copied().map(Some).collect(); + let mut list: Vec>> = entry_lists[l].iter().copied().map(Some).collect(); list.truncate(n); list.resize(n, None); if !sh.ref_pic_list_mod[l].is_empty() { @@ -199,7 +203,7 @@ impl Dpb { for m in &sh.ref_pic_list_mod[l] { ensure!(ref_idx < n, "too many ref_pic_list_modification entries"); list.push(None); // temporarily one longer - let (target, is_long): (Option<&DpbEntry>, bool) = match m.idc { + let (target, is_long): (Option<&DpbEntry>, bool) = match m.idc { 0 | 1 => { let d = m.value as i32 + 1; let no_wrap = if m.idc == 0 { @@ -249,7 +253,7 @@ impl Dpb { } // Fill missing entries (should not happen in conforming streams) with the first valid entry. let fallback = list.iter().flatten().next().copied().or_else(|| self.entries.last()); - let out: Vec = list.iter().filter_map(|e| e.or(fallback)).map(|e| to_ref(&e)).collect(); + let out: Vec = list.iter().filter_map(|e| e.or(fallback)).map(&to_ref).collect(); lists[l] = out; } Ok(lists) @@ -327,12 +331,12 @@ impl Dpb { pub fn store_picture( &mut self, sh: &SliceHeader, - frame: FrameRef, + frame: F, poc: i32, max_frame_num: u32, max_num_ref_frames: usize, meta: Arc, - out: &mut Vec, + out: &mut Vec>, ) { let mut mark = RefMark::Unused; let mut long_idx = 0; @@ -404,7 +408,7 @@ impl Dpb { } } -impl Default for Dpb { +impl Default for Dpb { fn default() -> Self { Self::new() } diff --git a/crates/h264/src/lib.rs b/crates/h264/src/lib.rs index 6cc05a0c..c70e8b0f 100644 --- a/crates/h264/src/lib.rs +++ b/crates/h264/src/lib.rs @@ -28,7 +28,7 @@ mod cavlc; mod cavlc_tables; mod deblock; mod decoder; -mod dpb; +pub mod dpb; mod error; mod inter; mod intra; diff --git a/crates/h264/src/slice.rs b/crates/h264/src/slice.rs index b92bdf69..f832bc9d 100644 --- a/crates/h264/src/slice.rs +++ b/crates/h264/src/slice.rs @@ -387,6 +387,27 @@ pub struct PocState { prev_frame_num: u32, } +/// Is `sh` the first slice of a new primary coded picture relative to `prev` (7.4.1.2.4)? +pub fn is_new_picture(prev: &SliceHeader, sh: &SliceHeader, sps: &Sps) -> bool { + if sh.frame_num != prev.frame_num + || sh.pps_id != prev.pps_id + || sh.field_pic != prev.field_pic + || sh.bottom_field != prev.bottom_field + || (sh.nal_ref_idc == 0) != (prev.nal_ref_idc == 0) + || sh.idr != prev.idr + || (sh.idr && prev.idr && sh.idr_pic_id != prev.idr_pic_id) + { + return true; + } + if sps.pic_order_cnt_type == 0 && (sh.pic_order_cnt_lsb != prev.pic_order_cnt_lsb || sh.delta_pic_order_cnt_bottom != prev.delta_pic_order_cnt_bottom) { + return true; + } + if sps.pic_order_cnt_type == 1 && sh.delta_pic_order_cnt != prev.delta_pic_order_cnt { + return true; + } + false +} + /// Output of POC computation. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Poc { diff --git a/crates/hevc/src/decoder.rs b/crates/hevc/src/decoder.rs index 4b87295f..48f50066 100644 --- a/crates/hevc/src/decoder.rs +++ b/crates/hevc/src/decoder.rs @@ -25,7 +25,7 @@ struct PendingPic { /// Header of the latest independent slice segment (for dependent slice segments). last_sh: SliceHeader, first_sh: SliceHeader, - rps: RefPicSet, + rps: RefPicSet, poc: i32, output: bool, meta: Arc, @@ -72,7 +72,7 @@ pub struct Decoder { spss: Vec>>, ppss: Vec>>, nal_length_size: Option, - dpb: Dpb, + dpb: Dpb, poc_state: PocState, pending: Option, next_id: u32, @@ -82,7 +82,7 @@ pub struct Decoder { /// RASL pictures associated with the last IRAP are skipped (it had NoRaslOutputFlag = 1). skip_rasl: bool, layout_cache: Vec<(Arc, Arc, Arc, Option>>>)>, - out_queue: VecDeque, + out_queue: VecDeque>, shared: Arc, #[cfg(feature = "threads")] pool: Option, @@ -578,7 +578,7 @@ fn output_meta(sps: &Sps, pts: i64, key: bool, draft: bool) -> OutputMeta { } } -fn make_picture(o: &Output) -> Picture { +fn make_picture(o: &Output) -> Picture { let (cx, cy, cw, ch) = o.meta.crop; let (cx, cy, cw, ch) = (cx as usize, cy as usize, cw as usize, ch as usize); let (y, u, v) = o.frame.copy_cropped((cx, cy, cw, ch)); diff --git a/crates/hevc/src/dpb.rs b/crates/hevc/src/dpb.rs index 6bb4e177..d6f029a9 100644 --- a/crates/hevc/src/dpb.rs +++ b/crates/hevc/src/dpb.rs @@ -1,8 +1,11 @@ //! Decoded picture buffer: POC derivation (8.3.1), reference picture set (8.3.2), missing //! reference generation (8.3.3), reference picture lists (8.3.4) and output / bumping (C.5.2). +//! +//! The buffer only does bookkeeping on picture headers; what a picture is (`F`: the software +//! decoder's [`crate::picture::FrameRef`], a hardware decoder's surface) is up to the decoder, so +//! hardware decoders driven by the same headers (VA-API) share this logic and output order. use crate::error::{Result, ensure}; -use crate::picture::{FrameRef, RefPic}; use crate::slice::SliceHeader; use std::sync::Arc; @@ -29,8 +32,8 @@ pub enum Marking { Long, } -pub struct Entry { - pub frame: FrameRef, +pub struct Entry { + pub frame: F, pub poc: i32, pub marking: Marking, pub needed_for_output: bool, @@ -39,31 +42,49 @@ pub struct Entry { } /// A picture leaving the DPB for output. -pub struct Output { - pub frame: FrameRef, +pub struct Output { + pub frame: F, pub poc: i32, pub meta: Arc, } -/// Reference picture set of the current picture (8.3.2) as DPB indices / generated frames. -#[derive(Default)] -pub struct RefPicSet { - pub st_curr_before: Vec, - pub st_curr_after: Vec, - pub lt_curr: Vec, +/// A reference picture of the current picture. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Ref { + pub frame: F, + pub poc: i32, + pub long_term: bool, } -#[derive(Default)] -pub struct Dpb { - pub entries: Vec, +/// Reference picture set of the current picture (8.3.2) as DPB indices / generated frames. +pub struct RefPicSet { + pub st_curr_before: Vec>, + pub st_curr_after: Vec>, + pub lt_curr: Vec>, +} + +impl Default for RefPicSet { + fn default() -> Self { + Self { st_curr_before: Vec::new(), st_curr_after: Vec::new(), lt_curr: Vec::new() } + } +} + +pub struct Dpb { + pub entries: Vec>, pub max_dec_pic_buffering: usize, pub max_num_reorder: usize, /// SpsMaxLatencyPictures (0 = no limit). pub max_latency: u32, } -impl Dpb { - fn bump(&mut self, outs: &mut Vec) -> bool { +impl Default for Dpb { + fn default() -> Self { + Self { entries: Vec::new(), max_dec_pic_buffering: 0, max_num_reorder: 0, max_latency: 0 } + } +} + +impl Dpb { + fn bump(&mut self, outs: &mut Vec>) -> bool { let Some(i) = (0..self.entries.len()).filter(|&i| self.entries[i].needed_for_output).min_by_key(|&i| self.entries[i].poc) else { return false }; let e = &mut self.entries[i]; e.needed_for_output = false; @@ -85,7 +106,7 @@ impl Dpb { } /// Output everything (end of stream or IRAP without no_output_of_prior_pics). - pub fn flush(&mut self, outs: &mut Vec) { + pub fn flush(&mut self, outs: &mut Vec>) { while self.bump(outs) {} for e in &mut self.entries { e.marking = Marking::Unused; @@ -100,7 +121,7 @@ impl Dpb { /// C.5.2.2: removal of pictures before decoding the current (non-IRAP-reset) picture, after the RPS /// has been applied. - pub fn bump_before_decode(&mut self, outs: &mut Vec) { + pub fn bump_before_decode(&mut self, outs: &mut Vec>) { self.remove_unused(); loop { let full = self.entries.len() >= self.max_dec_pic_buffering.max(1); @@ -116,7 +137,7 @@ impl Dpb { } /// C.5.2.3: insert the current picture and apply "additional bumping". - pub fn insert(&mut self, frame: FrameRef, poc: i32, output: bool, meta: Arc, outs: &mut Vec) { + pub fn insert(&mut self, frame: F, poc: i32, output: bool, meta: Arc, outs: &mut Vec>) { for e in &mut self.entries { if e.needed_for_output { e.latency += 1; @@ -138,8 +159,8 @@ impl Dpb { poc: i32, max_poc_lsb: i32, irap_no_rasl: bool, - make_missing: &mut dyn FnMut(i32) -> FrameRef, - ) -> Result { + make_missing: &mut dyn FnMut(i32) -> F, + ) -> Result> { let mut set = RefPicSet::default(); if sh.nal.is_irap() && irap_no_rasl { for e in &mut self.entries { @@ -196,13 +217,13 @@ impl Dpb { self.entries[i].marking = Marking::Long; } } - let mut make = |found: Option, p: i32, lt: bool, entries: &mut Vec| -> RefPic { + let mut make = |found: Option, p: i32, lt: bool, entries: &mut Vec>| -> Ref { match found { - Some(i) => RefPic { frame: entries[i].frame.clone(), poc: entries[i].poc, long_term: lt }, + Some(i) => Ref { frame: entries[i].frame.clone(), poc: entries[i].poc, long_term: lt }, None => { // 8.3.3: generate an unavailable reference picture let frame = make_missing(p); - RefPic { frame, poc: p, long_term: lt } + Ref { frame, poc: p, long_term: lt } } } }; @@ -228,8 +249,8 @@ impl Dpb { } /// Build RefPicList0/1 (8.3.4). -pub fn build_ref_lists(sh: &SliceHeader, rps: &RefPicSet) -> Result<[Vec; 2]> { - let mut lists: [Vec; 2] = [Vec::new(), Vec::new()]; +pub fn build_ref_lists(sh: &SliceHeader, rps: &RefPicSet) -> Result<[Vec>; 2]> { + let mut lists: [Vec>; 2] = [Vec::new(), Vec::new()]; if sh.is_intra() { return Ok(lists); } diff --git a/crates/hevc/src/lib.rs b/crates/hevc/src/lib.rs index a9fa837f..c6a6484c 100644 --- a/crates/hevc/src/lib.rs +++ b/crates/hevc/src/lib.rs @@ -23,7 +23,7 @@ mod cabac; mod decoder; -mod dpb; +pub mod dpb; mod error; mod filter; mod inter; diff --git a/crates/hevc/src/picture.rs b/crates/hevc/src/picture.rs index 5aadbf6e..55d2730f 100644 --- a/crates/hevc/src/picture.rs +++ b/crates/hevc/src/picture.rs @@ -207,9 +207,4 @@ fn copy_line(line: &[u16], x0: i32, out: &mut [i16]) { pub type FrameRef = Arc; /// An entry of RefPicList0/1 of the current slice. -#[derive(Clone)] -pub struct RefPic { - pub frame: FrameRef, - pub poc: i32, - pub long_term: bool, -} +pub type RefPic = crate::dpb::Ref; diff --git a/crates/interchange/src/fcp7.rs b/crates/interchange/src/fcp7.rs index 047b013f..0fdddd59 100644 --- a/crates/interchange/src/fcp7.rs +++ b/crates/interchange/src/fcp7.rs @@ -932,6 +932,10 @@ impl Exp<'_, '_> { let hold = c.frame_hold; let src_in = hold.unwrap_or(c.source_in); let src_out = if hold.is_some() { src_in + c.duration } else { c.source_out() }; + let (start_f, end_f, in_f) = (frames_round(rate, c.start), frames_round(rate, c.end()), frames_round(rate, src_in)); + // At normal speed (and on a freeze frame) the source span is the record span. Rounded on its own + // it can come out a frame shorter or longer than `end - start` (#340), so derive it from them. + let out_f = if hold.is_some() || c.speed.abs() == 1.0 { in_f.saturating_add(end_f.saturating_sub(start_f)) } else { frames_round(rate, src_out) }; self.w.open(tag, &[("id", xid)]); if generator.is_none() && !is_seq { let mid = self.master_id(base); @@ -940,12 +944,12 @@ impl Exp<'_, '_> { self.w.text("name", &c.name); self.w.text("enabled", bool_str(c.enabled)); let media_dur = self.p.item(base).map(|i| i.duration()).unwrap_or(Tick::ZERO).max(src_out); - self.w.text("duration", frames_round(rate, media_dur)); + self.w.text("duration", frames_round(rate, media_dur).max(out_f)); write_rate(&mut self.w, rate); - self.w.text("start", frames_round(rate, c.start)); - self.w.text("end", frames_round(rate, c.end())); - self.w.text("in", frames_round(rate, src_in)); - self.w.text("out", frames_round(rate, src_out)); + self.w.text("start", start_f); + self.w.text("end", end_f); + self.w.text("in", in_f); + self.w.text("out", out_f); self.w.text("pproTicksIn", src_in.0); self.w.text("pproTicksOut", src_out.0); if let Some(g) = &generator { diff --git a/crates/interchange/tests/fcp7.rs b/crates/interchange/tests/fcp7.rs index 2b72d1e4..4d3677ef 100644 --- a/crates/interchange/tests/fcp7.rs +++ b/crates/interchange/tests/fcp7.rs @@ -483,3 +483,47 @@ fn a_master_clip_used_by_an_earlier_sequence_still_lands_in_its_bin() { assert_ne!(bin, p.root.id, "the take was left loose at the top"); assert_eq!(p.root.find_bin(bin).unwrap().name, "Comedy"); } + +/// `start`/`end`/`in`/`out` of every sequence clip item in an exported document. +fn clip_spans(xml: &str) -> Vec<(i64, i64, i64, i64)> { + let tag = |block: &str, name: &str| -> i64 { + let open = format!("<{name}>"); + let at = block.find(&open).unwrap_or_else(|| panic!("no <{name}> in {block}")) + open.len(); + block[at..].split('<').next().unwrap().trim().parse().unwrap() + }; + xml.split("")).map(|b| (tag(b, "start"), tag(b, "end"), tag(b, "in"), tag(b, "out"))).collect() +} + +/// #340: a clip placed by seconds in a 23.976 sequence has sub-frame start, duration and source in. +/// Its record span and source span were rounded to frames independently, so `out - in` could come +/// out a frame shorter than `end - start` (Premiere then reads a different source range). +#[test] +fn sub_frame_clips_keep_equal_source_and_record_spans() { + const SECOND: i64 = 254_016_000_000; + let r = FrameRate::FPS_23_976; + let mut p = Project::new("P"); + let a = media(&mut p, "/m/a.mov", true, true, r); + let s = sequence(&mut p, "t", r, false); + let c = clip(&mut p, s, TrackKind::Video, 0, a, 0, 24, 0); + // the issue's case first (start 0, source in 20 s, 3 s long), then starts, ins and lengths in tenths of a second + let mut cases = vec![(0, 200, 30)]; + for start in [0, 1, 5, 17, 33] { + for src in [0, 3, 7, 200, 413] { + for dur in [1, 4, 30, 71, 125] { + cases.push((start, src, dur)); + } + } + } + for (start, src, dur) in cases { + let (_, it) = p.sequence_mut(s).unwrap().find_item_mut(c).unwrap(); + it.start = Tick(start * SECOND / 10); + it.source_in = Tick(src * SECOND / 10); + it.duration = Tick(dur * SECOND / 10); + let (bytes, _) = export(&p, s, Format::Fcp7Xml, &ExportOptions::default()).expect("export"); + let xml = String::from_utf8(bytes).unwrap(); + let spans = clip_spans(&xml); + assert_eq!(spans.len(), 1, "{xml}"); + let (start_f, end_f, in_f, out_f) = spans[0]; + assert_eq!(out_f - in_f, end_f - start_f, "start {start}, in {src}, duration {dur} (tenths of a second): {spans:?}"); + } +} diff --git a/crates/isobmff/src/codec.rs b/crates/isobmff/src/codec.rs index 117b6725..ee926cb5 100644 --- a/crates/isobmff/src/codec.rs +++ b/crates/isobmff/src/codec.rs @@ -1505,10 +1505,8 @@ impl SampleEntry { CodecConfig::Aac(c) => { b.leaf(b"esds", &esds_payload(0x40, &c.asc, c.max_bitrate, c.avg_bitrate, c.buffer_size)); } - CodecConfig::Mp3 => { - if format == *b"mp4a" { - b.leaf(b"esds", &esds_payload(0x6B, &[], 0, 0, 0)); - } + CodecConfig::Mp3 if format == *b"mp4a" => { + b.leaf(b"esds", &esds_payload(0x6B, &[], 0, 0, 0)); } CodecConfig::Alac { cookie } => { let m = b.start_full(b"alac", 0, 0); diff --git a/crates/matroska/src/demux.rs b/crates/matroska/src/demux.rs index 808b45b7..9344059c 100644 --- a/crates/matroska/src/demux.rs +++ b/crates/matroska/src/demux.rs @@ -713,10 +713,8 @@ impl Walker<'_> { match h.id { SEEK_HEAD => file.seek_head.extend(parse_seekhead(&data)), INFO => file.info = parse_info(&data), - TRACKS => { - if file.tracks.is_empty() { - file.tracks = parse_tracks(&data, file.info.timestamp_scale); - } + TRACKS if file.tracks.is_empty() => { + file.tracks = parse_tracks(&data, file.info.timestamp_scale); } CUES => parse_cues(&data, &mut file.cues), CHAPTERS => parse_chapters(&data, &mut file.chapters), diff --git a/crates/media/src/cancel.rs b/crates/media/src/cancel.rs index 7848a845..bfb391dd 100644 --- a/crates/media/src/cancel.rs +++ b/crates/media/src/cancel.rs @@ -13,6 +13,10 @@ //! [`with_draft`] marks reduced-resolution playback (opt-in): decoders may take spec-safe //! shortcuts that only change pictures nothing references (H.264: no deblocking of non-reference //! pictures). Never set for exports, renders or a frame shown while paused. +//! +//! [`with_background`] marks work nobody waits on interactively (thumbnails of the Project panel, +//! the Media Browser and the timeline): sources should not keep expensive state, such as a +//! frame-threaded 4K decoder, for it once it is done. use std::cell::{Cell, RefCell}; use std::sync::Arc; @@ -24,6 +28,28 @@ thread_local! { static CURRENT: RefCell>> = const { RefCell::new(None) }; static CATCH_UP: Cell> = const { Cell::new(None) }; static DRAFT: Cell = const { Cell::new(false) }; + static BACKGROUND: Cell = const { Cell::new(false) }; +} + +/// Run `f` as background work when `on` (restoring the previous hint after): a thumbnail, not a +/// frame a monitor or playback waits for. +pub fn with_background(on: bool, f: impl FnOnce() -> R) -> R { + /// Restores the previous hint however `f` ends (a panic caught further up must not leave the + /// worker thread marked as background). + struct Restore(bool); + impl Drop for Restore { + fn drop(&mut self) { + BACKGROUND.with(|c| c.set(self.0)); + } + } + let _restore = Restore(BACKGROUND.with(|c| c.replace(on))); + f() +} + +/// Whether the work running on this thread is background work (off unless inside +/// [`with_background`]`(true, …)`). +pub fn background() -> bool { + BACKGROUND.with(Cell::get) } /// Run `f` with the draft-decoding hint `on` (restoring the previous one after). Frames decoded @@ -97,6 +123,24 @@ mod tests { assert!(!draft()); } + #[test] + fn background_is_off_by_default_and_scoped() { + assert!(!background()); + with_background(true, || { + assert!(background()); + with_background(false, || assert!(!background())); + assert!(background()); + }); + assert!(!background()); + } + + #[test] + fn a_panic_inside_background_work_does_not_leave_the_thread_marked() { + let r = std::panic::catch_unwind(|| with_background(true, || panic!("boom"))); + assert!(r.is_err()); + assert!(!background()); + } + #[test] fn catch_up_is_scoped_to_the_closure() { assert_eq!(catch_up(), None); diff --git a/crates/platform/Cargo.toml b/crates/platform/Cargo.toml index f52c1290..55e3be06 100644 --- a/crates/platform/Cargo.toml +++ b/crates/platform/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "filmcraft-platform" -description = "OS media integration: hardware video decoding through the operating system's codecs (VideoToolbox on macOS), with transparent fallback to the pure-Rust decoders" +description = "OS media integration: hardware video decoding through the operating system's codecs (VideoToolbox on macOS, Media Foundation on Windows, VA-API on Linux), with transparent fallback to the pure-Rust decoders" version.workspace = true edition.workspace = true license.workspace = true @@ -21,10 +21,13 @@ type_complexity = "allow" undocumented_unsafe_blocks = "deny" [dependencies] +filmcraft-bitstream = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-color = { workspace = true } filmcraft-export = { workspace = true } filmcraft-frame = { workspace = true } +filmcraft-h264 = { workspace = true } +filmcraft-hevc = { workspace = true } filmcraft-isobmff = { workspace = true } filmcraft-time = { workspace = true } log = { workspace = true } @@ -49,6 +52,11 @@ windows = { version = "0.62", features = [ "Win32_System_LibraryLoader", ] } +# Linux: libva (VA-API) is loaded at run time with libloading (ISC; already in the lockfile through +# wgpu), so building needs no libva headers and a system without libva simply decodes in software. +[target.'cfg(target_os = "linux")'.dependencies] +libloading = "0.8" + [dev-dependencies] filmcraft-testkit = { workspace = true } filmcraft-media = { workspace = true } diff --git a/crates/platform/README.md b/crates/platform/README.md index eb361484..01f5faa2 100644 --- a/crates/platform/README.md +++ b/crates/platform/README.md @@ -8,7 +8,7 @@ It is the one crate of the workspace allowed to contain `unsafe`, under the rule ```rust // at startup (the desktop app, filmcraft-cli, the bench) -let availability = filmcraft_platform::register(); // Available("VideoToolbox") on macOS, Available("Media Foundation") on Windows +let availability = filmcraft_platform::register(); // Available("VideoToolbox") on macOS, Available("Media Foundation") on Windows, Available("VA-API") on Linux with a driver ``` ## What it does @@ -70,6 +70,37 @@ let availability = filmcraft_platform::register(); // Available("VideoToolbox") It declines two-pass VBR, HDR, MXF, interlaced output, sizes outside NVENC's limits and systems without an NVIDIA GPU or driver, and the software encoder runs instead. A failure during an export ends it with an error, since a hardware stream cannot be finished in software. +- **Linux: VA-API, H.264 (`avcC`) and HEVC (`hvcC`)**: H.264 8-bit 4:2:0 progressive, Constrained + Baseline / Main / High; HEVC Main, Main 10 and Main Still Picture, 4:2:0 8- and 10-bit (`vaapi/`), on Intel (iHD, i965), AMD and other Mesa drivers. `libva.so.2` and `libva-drm.so.2` + are loaded at run time (`vaapi/va.rs`; no libva headers needed to build), and each decoder opens + its own display on the first DRM render node (`/dev/dri/renderD128`…) with a working driver. + VA-API decoding is *stateless*: the host parses the stream and keeps the decoded picture buffer, + and the GPU decodes one picture's slice data at a time into a surface. That host side + (`vaapi/h264.rs`, `vaapi/hevc.rs`, safe code) is the software decoders' own: their parameter-set + and slice-header parsers, POC computation and DPBs (`filmcraft_h264::dpb`, `filmcraft_hevc::dpb`, + generic over what a picture is: H.264 reference marking including MMCOs and reference lists with + modifications; HEVC reference picture sets and lists, RASL pictures of a CRA the run starts at + left out; output order). The two decoders therefore make + the same decisions and output the same pictures in the same order, and the parity tests are + bit-exact. Each picture is sent as one picture-parameter buffer, the scaling matrices, and a slice + parameter + slice data buffer per slice (the NAL unit as stored, emulation prevention included). + HEVC pictures are sent the same way per slice segment (scaling lists only when enabled, converted + to raster order). Pictures the DPB outputs are read back right away (`vaGetImage` into an NV12 or + P010 image, then `biplanar.rs`), **the one GPU to CPU copy**. Surfaces: the stream's DPB size plus two. + - *frame_num gaps* (and every start at a non-IDR picture: an open-GOP seek) get "non-existing" + frames like the software decoder's: copies of the latest decoded frame, mid-gray when there is + none (`vaPutImage`). Pictures predicted from them (the leading pictures of an open GOP after a + seek) are concealment in both decoders and can differ: the hardware also reads motion data the + stand-ins do not have. Everything from the seek point on is bit-exact. + - *Declined up front* (our decoder is used): VP9 and AV1 (not through VA-API yet), H.264 profiles + other than Baseline / Main / High, 10-bit H.264, HEVC range extensions and profiles other than + Main / Main 10 / Main Still Picture, 4:2:2 / 4:4:4, field / MBAFF coding, no libva, no render + node or driver, a profile or size the driver does not decode. *Mid-stream errors* + (`HybridDecoder` continues in software): FMO, SP / SI slices, data partitioning, a size or DPB + change, a missing reference picture (HEVC: one the RPS names that was never decoded), more than + 15 HEVC reference frames, any driver error. + - An HEVC `flush` ends the stream as in software (references are dropped and the next CRA + leaves out its RASL pictures); the GOP cache always seeks after a flush. - **Other systems:** `register()` does nothing and returns `Availability::Unavailable`. - **`HybridDecoder`** (`hybrid.rs`, safe code): the hardware decoder plus the means to build our software decoder for the same `SampleEntry` (`filmcraft_codecs::software_video_decoder`). On a @@ -114,8 +145,10 @@ built-in encoder, which changes three things from H.264: - **Choosing the format is the opt-in.** No `hardware_encoding` setting: Export ▸ Format ▸ H.265 (HEVC), or `"format": "hevc"` in `file.exportMedia` (`h265` is accepted too). The format is listed as available only on a machine with a hardware HEVC encoder: `register()` hands - `hardware_encode::hevc_available` (one small hardware session, created on the first question) to - `filmcraft_export::register_format_probe`. + `hardware_encode::hevc_available` (one small hardware session, about 0.1 s) to + `filmcraft_export::register_format_probe`, and asks it right away on a thread of its own + (`warm_hevc_probe`), so the first draw of the format list does not create the session on the UI + thread. - **There is no fallback encoder.** What the hardware path does not take is an error, not a different encoder: two-pass VBR is refused up front (`ExportSettings::validate`), HDR sequences are exported as SDR (the H.265 path is 8-bit), and odd sizes, non-square pixels or a machine @@ -140,13 +173,14 @@ level and flags are the encoder's own. If it is missing the export stops with th - **Never undecodable:** the factory declines (returns `None`, so the software decoder is used) when Settings ▸ Playback ▸ Hardware decoding is Off, for formats it does not take (field-coded H.264, bit depths other than 8 / 10, 4:4:4 or monochrome, luma / chroma depth mismatch, larger - than 8192×8192; on Windows also 4:2:2 and the profiles listed above) and when the OS cannot - create a hardware session (VideoToolbox) or a GPU-backed decoder (Media Foundation). + than 8192×8192; on Windows also 4:2:2 and the profiles listed above; on Linux everything but + 8-bit 4:2:0 progressive H.264 and 4:2:0 HEVC Main / Main 10) and when the OS cannot create a hardware session (VideoToolbox), a + GPU-backed decoder (Media Foundation) or a VA-API configuration and context (Linux). - **Interchangeable:** colour, pixel aspect, pts, presentation order, `is_random_access` and `is_disposable` come from the software decoders' own helpers (`filmcraft_codecs::hw`, `video::vui_color`, `sar_par`). -- **Never crash:** no `unwrap` / `expect` / `panic!` outside tests; the output callback runs under - `catch_unwind`; every `unsafe` block has a `// SAFETY:` comment; the public API is safe. +- **Never crash:** no `unwrap` / `expect` / `panic!` outside tests; the output callback (and + libva's error-message callback) runs under `catch_unwind`; every `unsafe` block has a `// SAFETY:` comment; the public API is safe. - **Counted:** `perf.stats` `decode.hardware` (frames, software frames, sessions, declined, fallbacks; `filmcraft_codecs::hw::hw_stats`) and `backend` (the registered backend's name, `filmcraft_codecs::hw::hw_backend`). `export.hardware` counts the NVENC encoder's frames, sessions and declines. @@ -162,6 +196,9 @@ level and flags are the encoder's own. If it is missing the export stops with th | `tests/nvenc.rs` (Windows, NVIDIA) | H.264 from NVENC (1280×720, 6 Mbps, 72 frames) decodes with our decoder at worst 46.9 dB luma PSNR; IDR at 0, 24 and 48; dts / pts right | | `tests/nvenc_export.rs` (Windows, NVIDIA) | Export with hardware encoding against the software encoder through the export pipeline: the two decoded files at worst 54.8 dB luma PSNR; ffmpeg decodes the file without errors; declined cases go to the software encoder; the counters | | `src/nvenc/abi_tests.rs` (Windows) | FFI structs' sizes, alignments, field offsets, constants and GUIDs against a C compiler's view of NVIDIA's `nvEncodeAPI.h` (12.1) | +| `tests/vaapi.rs` (Linux, VA-API) | HEVC Main (open GOP: CRA + RASL), Main 10, scaling lists, 3 slices with WPP (10-bit), weighted prediction with 4 references, transform skip + AMP + B-pyramid (10-bit, open GOP), Main / Main 10 at 1080p and 2160p (compared picture by picture, little memory); H.264 High (B-pyramid), Constrained Baseline with 3 slices per picture, Main with explicit weighted prediction (P and B) and 4 references, temporal direct, custom scaling matrices (JVT) with the 8x8 transform and 2 slices, an open-GOP stream, and 1080p / 2160p, 640×360 (coded 368: cropping): every picture **bit-exact** with our software decoder, same pts order, count, colour and aspect, also after `reset` + reseek to every later sync sample (after an open-GOP I picture, from the seek point on), a mid-stream `flush` with decoding carrying on, and a full pass after resets; forced mid-stream failures (`VaDecoder::fail_after`) at five points continue with the software decoder's exact output; interlaced and 10-bit H.264, 4:2:2 HEVC and the Off setting are declined; an HEVC mid-GOP flush is not checked (it ends the stream, as in software); seeded mutation of samples (through the hybrid and straight into the hardware decoder) and of `avcC` records never panics or hangs; 40 decoders created and dropped; concurrent decoders; a decoder moving between threads | +| `src/vaapi/tests.rs` (every OS) | The H.264 and HEVC front ends with a recording stand-in for the hardware, on x264 streams (B-pyramid, weighted prediction, 4 slices, open GOP with temporal direct) and x265 streams (open GOP with RASL, B-pyramid, Main 10 with 2 slices and weighted prediction): output order and pts are the software decoder's, also after seeks and a mid-stream flush; every buffer is consistent (current surface free, references decoded and listed once, reference lists of the active length, slice data offsets inside the slice); damaged samples are errors | +| `src/vaapi/abi_tests.rs` | FFI structs' sizes, alignments, field offsets, constants and bit-field positions against a C compiler's view of libva's `va.h` and `va_dec_hevc.h` (VA-API 1.23) | Fixtures are made with ffmpeg into `target/fixtures/platform/` (generator only, never linked); tests skip without ffmpeg or without a hardware decoder. @@ -185,9 +222,14 @@ M4 Pro, load 150–190 (`cargo xtask bench --hw off|auto`): CPU per decoded fram playback with no dropped frames at Full, 1/2 and 1/4. Details in [docs/performance.md](../../docs/performance.md). +Hardware decoding, Linux (Intel Iris Xe, iHD driver; `cargo xtask bench --hw off|auto`): CPU per +decoded frame H.264 1080p 104 → 2.3 ms, 2160p 417 → 11.4 ms; HEVC 1080p 46 → 2.3 ms, 2160p +203 → 10.1 ms, Main 10 2160p 212 → 18.4 ms. +Details in [docs/performance.md](../../docs/performance.md). + ## Not yet Zero-copy upload of decoded pictures into wgpu textures (`CVPixelBuffer`s on macOS, Direct3D 11 -textures on Windows); B-frames and 10-bit / HDR HEVC (Main 10) in hardware encoding; VA-API (Linux) -decoders; field-coded H.264; HEVC, 10-bit and HDR encoding with NVENC, and encoders from other +textures on Windows); B-frames and 10-bit / HDR HEVC (Main 10) in hardware encoding; HEVC, VP9 and +VP9 and AV1 through VA-API (Linux), and VA-API encoding; field-coded H.264; HEVC, 10-bit and HDR encoding with NVENC, and encoders from other vendors on Windows (through Media Foundation); VP9 / AV1 4:4:4 and 12-bit on Windows. diff --git a/crates/platform/src/hardware_encode.rs b/crates/platform/src/hardware_encode.rs index 498b68ea..742c656b 100644 --- a/crates/platform/src/hardware_encode.rs +++ b/crates/platform/src/hardware_encode.rs @@ -13,7 +13,7 @@ //! configurations are declined, and a declined export fails with "encoder not available" instead of //! falling back; [`hevc_available`] tells the format list whether this machine can do it at all. -use std::sync::OnceLock; +use std::sync::{Once, OnceLock}; use filmcraft_export::{BitrateMode, EncodedPacket, EncoderFrame, ExportError, ExportSettings, Format, H264Pass, H264Profile, HardwareEncoding, VideoEncoder}; use filmcraft_isobmff::{AvcConfig, HevcConfig, SampleEntry}; @@ -66,11 +66,13 @@ pub fn config_for(w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Result }) } -/// Whether this machine has a hardware HEVC encoder (asked once, by creating a small session): what -/// makes `Format::Hevc` available. +static HEVC_PROBE: OnceLock = OnceLock::new(); + +/// Whether this machine has a hardware HEVC encoder (asked once, by creating a small session, which +/// takes about 0.1 s): what makes `Format::Hevc` available. [`warm_hevc_probe`] asks it ahead of the +/// first caller. pub fn hevc_available() -> bool { - static AVAILABLE: OnceLock = OnceLock::new(); - *AVAILABLE.get_or_init(|| { + *HEVC_PROBE.get_or_init(|| { let probe = VtConfig { width: 640, height: 360, @@ -88,6 +90,27 @@ pub fn hevc_available() -> bool { }) } +/// Whether [`hevc_available`] has its answer yet (diagnostics and tests). +pub fn hevc_probed() -> bool { + HEVC_PROBE.get().is_some() +} + +/// Ask [`hevc_available`] now, on a thread of its own, so that the first draw of the format list does +/// not spend its 0.1 s creating a hardware session on the UI thread. Whoever asks while the thread +/// runs waits for the same answer (no longer than asking first); if the thread cannot start, or the +/// probe panics, the first caller asks, as before. Only the first call starts a thread. +pub fn warm_hevc_probe() { + static STARTED: Once = Once::new(); + STARTED.call_once(|| { + let spawned = std::thread::Builder::new().name("filmcraft-hevc-probe".into()).spawn(|| { + let _ = std::panic::catch_unwind(hevc_available); + }); + if let Err(why) = spawned { + log::info!("HEVC probe not started ahead of time: {why}"); + } + }); +} + /// The encoder factory `register` puts in front of the built-in ones: a hardware encoder when the /// export asks for one (H.264 with `hardware_encoding`; HEVC always) and the OS takes the /// configuration, `None` (the built-in encoder, if there is one) otherwise. diff --git a/crates/platform/src/lib.rs b/crates/platform/src/lib.rs index 44875796..54e4468a 100644 --- a/crates/platform/src/lib.rs +++ b/crates/platform/src/lib.rs @@ -3,8 +3,9 @@ //! //! [`register`] puts the platform's hardware decoder factory in front of FilmCraft's own decoders //! (`filmcraft_codecs::register_video_decoder`). Today that is VideoToolbox on macOS for H.264 -//! (`avcC`) and HEVC (`hvcC`) streams, 8- and 10-bit, 4:2:0 and 4:2:2; on other systems -//! registration does nothing and reports [`Availability::Unavailable`]. It also registers a +//! (`avcC`) and HEVC (`hvcC`) streams, 8- and 10-bit, 4:2:0 and 4:2:2; Media Foundation / DXVA on +//! Windows; VA-API on Linux for H.264 ([`vaapi`]); on other systems (and on Linux without a VA-API +//! driver) registration does nothing and reports [`Availability::Unavailable`]. It also registers a //! hardware H.264 encoder factory (`filmcraft_export::register_encoder`) that only acts when an //! export asks for it (`ExportSettings::hardware_encoding` = `Auto`), see [`hardware_encode`]. //! @@ -25,10 +26,11 @@ #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] -// Used by the Windows decoder only; compiled everywhere so their tests run on every system. +// Used by the Windows (and, for `biplanar`, Linux) decoders; compiled everywhere so their tests run +// on every system. #[cfg_attr(not(target_os = "windows"), allow(dead_code))] mod annexb; -#[cfg_attr(not(target_os = "windows"), allow(dead_code))] +#[cfg_attr(not(any(target_os = "windows", target_os = "linux")), allow(dead_code))] mod biplanar; #[cfg(target_os = "macos")] pub mod hardware_encode; @@ -37,6 +39,8 @@ pub mod hybrid; pub mod media_foundation; #[cfg(target_os = "windows")] pub mod nvenc; +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub mod vaapi; #[cfg(target_os = "macos")] #[allow(unsafe_code)] pub mod videotoolbox; @@ -64,6 +68,9 @@ pub fn register() -> Availability { filmcraft_codecs::register_video_decoder(videotoolbox_factory); filmcraft_export::register_encoder(hardware_encode::videotoolbox_encoder_factory); filmcraft_export::register_format_probe(filmcraft_export::Format::Hevc, hardware_encode::hevc_available); + // the probe creates a hardware session (about 0.1 s): answer it now, off the UI thread, + // before the first draw of the format list asks + hardware_encode::warm_hevc_probe(); filmcraft_codecs::hw::set_hw_backend("VideoToolbox"); Availability::Available("VideoToolbox") } @@ -77,7 +84,23 @@ pub fn register() -> Availability { filmcraft_codecs::hw::set_hw_backend("Media Foundation"); Availability::Available("Media Foundation") } - #[cfg(not(any(target_os = "macos", target_os = "windows")))] + #[cfg(target_os = "linux")] + { + match vaapi::va::probe() { + Ok(driver) => { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| log::info!("hardware decoding through VA-API: {driver}")); + filmcraft_codecs::register_video_decoder(vaapi_factory); + filmcraft_codecs::hw::set_hw_backend("VA-API"); + Availability::Available("VA-API") + } + Err(why) => { + log::info!("no VA-API hardware decoding: {why}"); + Availability::Unavailable("no VA-API driver (libva and a DRM render node)") + } + } + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] { Availability::Unavailable("no hardware video decoder for this system yet") } @@ -93,7 +116,11 @@ pub fn registered() -> bool { { filmcraft_codecs::video_decoder_registered(media_foundation_factory) } - #[cfg(not(any(target_os = "macos", target_os = "windows")))] + #[cfg(target_os = "linux")] + { + filmcraft_codecs::video_decoder_registered(vaapi_factory) + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] { false } @@ -110,7 +137,11 @@ pub fn hardware_decoder_for(entry: &filmcraft_isobmff::SampleEntry) -> bool { { media_foundation::stream_info(entry).is_some_and(|info| media_foundation::MfDecoder::new(info).is_ok()) } - #[cfg(not(any(target_os = "macos", target_os = "windows")))] + #[cfg(target_os = "linux")] + { + filmcraft_codecs::hw::NalStreamInfo::from_entry(entry).and_then(|r| r.ok()).is_some_and(|info| vaapi::VaDecoder::new(info).is_ok()) + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] { let _ = entry; false @@ -153,3 +184,22 @@ pub fn media_foundation_factory(entry: &filmcraft_isobmff::SampleEntry) -> Optio } } } + +/// The VA-API factory: a [`HybridDecoder`] around [`vaapi::VaDecoder`] for H.264 and HEVC +/// streams this system's VA-API driver decodes, `None` otherwise (other codecs keep the software +/// decoders). +#[cfg(target_os = "linux")] +pub fn vaapi_factory(entry: &filmcraft_isobmff::SampleEntry) -> Option>> { + if !filmcraft_codecs::hw::hardware_decoding() { + return None; + } + let info = filmcraft_codecs::hw::NalStreamInfo::from_entry(entry)?.ok()?; + match vaapi::VaDecoder::new(info.clone()) { + Ok(va) => Some(Ok(Box::new(HybridDecoder::new(Box::new(va), entry.clone(), info)))), + Err(why) => { + log::info!("hardware decoding declined for {} video: {why}", entry.codec.name()); + filmcraft_codecs::hw::note_hw_declined(); + None + } + } +} diff --git a/crates/platform/src/vaapi/abi_tests.rs b/crates/platform/src/vaapi/abi_tests.rs new file mode 100644 index 00000000..69e5ffc0 --- /dev/null +++ b/crates/platform/src/vaapi/abi_tests.rs @@ -0,0 +1,365 @@ +//! The VA-API FFI structures against the C compiler's view of libva's `va.h` and `va_dec_hevc.h` (VA-API 1.23, libva +//! 2.23): sizes, alignments, field offsets, constants and bit-field positions. Generated by running a +//! C program (built with GCC against the libva-dev headers, x86-64 Linux) that prints them; the +//! header itself is not in this repository. To regenerate, compile a program printing `sizeof` / +//! `offsetof` of the structures of `src/vaapi/ffi.rs`, the constants it uses, and the `value` word of +//! each `seq_fields` / `pic_fields` bit-field set alone, against libva's MIT-licensed headers. +#![allow(clippy::unreadable_literal)] + +use super::ffi::*; +use std::mem::{align_of, offset_of, size_of}; + +#[test] +fn layouts_match_the_header() { + assert_eq!((size_of::(), align_of::()), (8, 4), "VAConfigAttrib"); + assert_eq!(offset_of!(VAConfigAttrib, type_), 0, "VAConfigAttrib.type"); + assert_eq!(offset_of!(VAConfigAttrib, value), 4, "VAConfigAttrib.value"); + assert_eq!((size_of::(), align_of::()), (36, 4), "VAPictureH264"); + assert_eq!(offset_of!(VAPictureH264, picture_id), 0, "VAPictureH264.picture_id"); + assert_eq!(offset_of!(VAPictureH264, frame_idx), 4, "VAPictureH264.frame_idx"); + assert_eq!(offset_of!(VAPictureH264, flags), 8, "VAPictureH264.flags"); + assert_eq!(offset_of!(VAPictureH264, TopFieldOrderCnt), 12, "VAPictureH264.TopFieldOrderCnt"); + assert_eq!(offset_of!(VAPictureH264, BottomFieldOrderCnt), 16, "VAPictureH264.BottomFieldOrderCnt"); + assert_eq!(offset_of!(VAPictureH264, va_reserved), 20, "VAPictureH264.va_reserved"); + assert_eq!((size_of::(), align_of::()), (672, 4), "VAPictureParameterBufferH264"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, CurrPic), 0, "VAPictureParameterBufferH264.CurrPic"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, ReferenceFrames), 36, "VAPictureParameterBufferH264.ReferenceFrames"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, picture_width_in_mbs_minus1), 612, "VAPictureParameterBufferH264.picture_width_in_mbs_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, picture_height_in_mbs_minus1), 614, "VAPictureParameterBufferH264.picture_height_in_mbs_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, bit_depth_luma_minus8), 616, "VAPictureParameterBufferH264.bit_depth_luma_minus8"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, bit_depth_chroma_minus8), 617, "VAPictureParameterBufferH264.bit_depth_chroma_minus8"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, num_ref_frames), 618, "VAPictureParameterBufferH264.num_ref_frames"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, seq_fields), 620, "VAPictureParameterBufferH264.seq_fields"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, num_slice_groups_minus1), 624, "VAPictureParameterBufferH264.num_slice_groups_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, slice_group_map_type), 625, "VAPictureParameterBufferH264.slice_group_map_type"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, slice_group_change_rate_minus1), 626, "VAPictureParameterBufferH264.slice_group_change_rate_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, pic_init_qp_minus26), 628, "VAPictureParameterBufferH264.pic_init_qp_minus26"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, pic_init_qs_minus26), 629, "VAPictureParameterBufferH264.pic_init_qs_minus26"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, chroma_qp_index_offset), 630, "VAPictureParameterBufferH264.chroma_qp_index_offset"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, second_chroma_qp_index_offset), 631, "VAPictureParameterBufferH264.second_chroma_qp_index_offset"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, pic_fields), 632, "VAPictureParameterBufferH264.pic_fields"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, frame_num), 636, "VAPictureParameterBufferH264.frame_num"); + assert_eq!(offset_of!(VAPictureParameterBufferH264, va_reserved), 640, "VAPictureParameterBufferH264.va_reserved"); + assert_eq!((size_of::(), align_of::()), (240, 4), "VAIQMatrixBufferH264"); + assert_eq!(offset_of!(VAIQMatrixBufferH264, ScalingList4x4), 0, "VAIQMatrixBufferH264.ScalingList4x4"); + assert_eq!(offset_of!(VAIQMatrixBufferH264, ScalingList8x8), 96, "VAIQMatrixBufferH264.ScalingList8x8"); + assert_eq!(offset_of!(VAIQMatrixBufferH264, va_reserved), 224, "VAIQMatrixBufferH264.va_reserved"); + assert_eq!((size_of::(), align_of::()), (3128, 4), "VASliceParameterBufferH264"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_size), 0, "VASliceParameterBufferH264.slice_data_size"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_offset), 4, "VASliceParameterBufferH264.slice_data_offset"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_flag), 8, "VASliceParameterBufferH264.slice_data_flag"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_bit_offset), 12, "VASliceParameterBufferH264.slice_data_bit_offset"); + assert_eq!(offset_of!(VASliceParameterBufferH264, first_mb_in_slice), 14, "VASliceParameterBufferH264.first_mb_in_slice"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_type), 16, "VASliceParameterBufferH264.slice_type"); + assert_eq!(offset_of!(VASliceParameterBufferH264, direct_spatial_mv_pred_flag), 17, "VASliceParameterBufferH264.direct_spatial_mv_pred_flag"); + assert_eq!(offset_of!(VASliceParameterBufferH264, num_ref_idx_l0_active_minus1), 18, "VASliceParameterBufferH264.num_ref_idx_l0_active_minus1"); + assert_eq!(offset_of!(VASliceParameterBufferH264, num_ref_idx_l1_active_minus1), 19, "VASliceParameterBufferH264.num_ref_idx_l1_active_minus1"); + assert_eq!(offset_of!(VASliceParameterBufferH264, cabac_init_idc), 20, "VASliceParameterBufferH264.cabac_init_idc"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_qp_delta), 21, "VASliceParameterBufferH264.slice_qp_delta"); + assert_eq!(offset_of!(VASliceParameterBufferH264, disable_deblocking_filter_idc), 22, "VASliceParameterBufferH264.disable_deblocking_filter_idc"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_alpha_c0_offset_div2), 23, "VASliceParameterBufferH264.slice_alpha_c0_offset_div2"); + assert_eq!(offset_of!(VASliceParameterBufferH264, slice_beta_offset_div2), 24, "VASliceParameterBufferH264.slice_beta_offset_div2"); + assert_eq!(offset_of!(VASliceParameterBufferH264, RefPicList0), 28, "VASliceParameterBufferH264.RefPicList0"); + assert_eq!(offset_of!(VASliceParameterBufferH264, RefPicList1), 1180, "VASliceParameterBufferH264.RefPicList1"); + assert_eq!(offset_of!(VASliceParameterBufferH264, luma_log2_weight_denom), 2332, "VASliceParameterBufferH264.luma_log2_weight_denom"); + assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_log2_weight_denom), 2333, "VASliceParameterBufferH264.chroma_log2_weight_denom"); + assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l0_flag), 2334, "VASliceParameterBufferH264.luma_weight_l0_flag"); + assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l0), 2336, "VASliceParameterBufferH264.luma_weight_l0"); + assert_eq!(offset_of!(VASliceParameterBufferH264, luma_offset_l0), 2400, "VASliceParameterBufferH264.luma_offset_l0"); + assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l0_flag), 2464, "VASliceParameterBufferH264.chroma_weight_l0_flag"); + assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l0), 2466, "VASliceParameterBufferH264.chroma_weight_l0"); + assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_offset_l0), 2594, "VASliceParameterBufferH264.chroma_offset_l0"); + assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l1_flag), 2722, "VASliceParameterBufferH264.luma_weight_l1_flag"); + assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l1), 2724, "VASliceParameterBufferH264.luma_weight_l1"); + assert_eq!(offset_of!(VASliceParameterBufferH264, luma_offset_l1), 2788, "VASliceParameterBufferH264.luma_offset_l1"); + assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l1_flag), 2852, "VASliceParameterBufferH264.chroma_weight_l1_flag"); + assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l1), 2854, "VASliceParameterBufferH264.chroma_weight_l1"); + assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_offset_l1), 2982, "VASliceParameterBufferH264.chroma_offset_l1"); + assert_eq!(offset_of!(VASliceParameterBufferH264, va_reserved), 3112, "VASliceParameterBufferH264.va_reserved"); + assert_eq!((size_of::(), align_of::()), (48, 4), "VAImageFormat"); + assert_eq!(offset_of!(VAImageFormat, fourcc), 0, "VAImageFormat.fourcc"); + assert_eq!(offset_of!(VAImageFormat, byte_order), 4, "VAImageFormat.byte_order"); + assert_eq!(offset_of!(VAImageFormat, bits_per_pixel), 8, "VAImageFormat.bits_per_pixel"); + assert_eq!(offset_of!(VAImageFormat, depth), 12, "VAImageFormat.depth"); + assert_eq!(offset_of!(VAImageFormat, red_mask), 16, "VAImageFormat.red_mask"); + assert_eq!(offset_of!(VAImageFormat, green_mask), 20, "VAImageFormat.green_mask"); + assert_eq!(offset_of!(VAImageFormat, blue_mask), 24, "VAImageFormat.blue_mask"); + assert_eq!(offset_of!(VAImageFormat, alpha_mask), 28, "VAImageFormat.alpha_mask"); + assert_eq!(offset_of!(VAImageFormat, va_reserved), 32, "VAImageFormat.va_reserved"); + assert_eq!((size_of::(), align_of::()), (120, 4), "VAImage"); + assert_eq!(offset_of!(VAImage, image_id), 0, "VAImage.image_id"); + assert_eq!(offset_of!(VAImage, format), 4, "VAImage.format"); + assert_eq!(offset_of!(VAImage, buf), 52, "VAImage.buf"); + assert_eq!(offset_of!(VAImage, width), 56, "VAImage.width"); + assert_eq!(offset_of!(VAImage, height), 58, "VAImage.height"); + assert_eq!(offset_of!(VAImage, data_size), 60, "VAImage.data_size"); + assert_eq!(offset_of!(VAImage, num_planes), 64, "VAImage.num_planes"); + assert_eq!(offset_of!(VAImage, pitches), 68, "VAImage.pitches"); + assert_eq!(offset_of!(VAImage, offsets), 80, "VAImage.offsets"); + assert_eq!(offset_of!(VAImage, num_palette_entries), 92, "VAImage.num_palette_entries"); + assert_eq!(offset_of!(VAImage, entry_bytes), 96, "VAImage.entry_bytes"); + assert_eq!(offset_of!(VAImage, component_order), 100, "VAImage.component_order"); + assert_eq!(offset_of!(VAImage, va_reserved), 104, "VAImage.va_reserved"); + assert_eq!((size_of::(), align_of::()), (28, 4), "VAPictureHEVC"); + assert_eq!(offset_of!(VAPictureHEVC, picture_id), 0, "VAPictureHEVC.picture_id"); + assert_eq!(offset_of!(VAPictureHEVC, pic_order_cnt), 4, "VAPictureHEVC.pic_order_cnt"); + assert_eq!(offset_of!(VAPictureHEVC, flags), 8, "VAPictureHEVC.flags"); + assert_eq!(offset_of!(VAPictureHEVC, va_reserved), 12, "VAPictureHEVC.va_reserved"); + assert_eq!((size_of::(), align_of::()), (604, 4), "VAPictureParameterBufferHEVC"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, CurrPic), 0, "VAPictureParameterBufferHEVC.CurrPic"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, ReferenceFrames), 28, "VAPictureParameterBufferHEVC.ReferenceFrames"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pic_width_in_luma_samples), 448, "VAPictureParameterBufferHEVC.pic_width_in_luma_samples"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pic_height_in_luma_samples), 450, "VAPictureParameterBufferHEVC.pic_height_in_luma_samples"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pic_fields), 452, "VAPictureParameterBufferHEVC.pic_fields"); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, sps_max_dec_pic_buffering_minus1), + 456, + "VAPictureParameterBufferHEVC.sps_max_dec_pic_buffering_minus1" + ); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, bit_depth_luma_minus8), 457, "VAPictureParameterBufferHEVC.bit_depth_luma_minus8"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, bit_depth_chroma_minus8), 458, "VAPictureParameterBufferHEVC.bit_depth_chroma_minus8"); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, pcm_sample_bit_depth_luma_minus1), + 459, + "VAPictureParameterBufferHEVC.pcm_sample_bit_depth_luma_minus1" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, pcm_sample_bit_depth_chroma_minus1), + 460, + "VAPictureParameterBufferHEVC.pcm_sample_bit_depth_chroma_minus1" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_min_luma_coding_block_size_minus3), + 461, + "VAPictureParameterBufferHEVC.log2_min_luma_coding_block_size_minus3" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_diff_max_min_luma_coding_block_size), + 462, + "VAPictureParameterBufferHEVC.log2_diff_max_min_luma_coding_block_size" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_min_transform_block_size_minus2), + 463, + "VAPictureParameterBufferHEVC.log2_min_transform_block_size_minus2" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_diff_max_min_transform_block_size), + 464, + "VAPictureParameterBufferHEVC.log2_diff_max_min_transform_block_size" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_min_pcm_luma_coding_block_size_minus3), + 465, + "VAPictureParameterBufferHEVC.log2_min_pcm_luma_coding_block_size_minus3" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_diff_max_min_pcm_luma_coding_block_size), + 466, + "VAPictureParameterBufferHEVC.log2_diff_max_min_pcm_luma_coding_block_size" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, max_transform_hierarchy_depth_intra), + 467, + "VAPictureParameterBufferHEVC.max_transform_hierarchy_depth_intra" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, max_transform_hierarchy_depth_inter), + 468, + "VAPictureParameterBufferHEVC.max_transform_hierarchy_depth_inter" + ); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, init_qp_minus26), 469, "VAPictureParameterBufferHEVC.init_qp_minus26"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, diff_cu_qp_delta_depth), 470, "VAPictureParameterBufferHEVC.diff_cu_qp_delta_depth"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_cb_qp_offset), 471, "VAPictureParameterBufferHEVC.pps_cb_qp_offset"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_cr_qp_offset), 472, "VAPictureParameterBufferHEVC.pps_cr_qp_offset"); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_parallel_merge_level_minus2), + 473, + "VAPictureParameterBufferHEVC.log2_parallel_merge_level_minus2" + ); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_tile_columns_minus1), 474, "VAPictureParameterBufferHEVC.num_tile_columns_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_tile_rows_minus1), 475, "VAPictureParameterBufferHEVC.num_tile_rows_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, column_width_minus1), 476, "VAPictureParameterBufferHEVC.column_width_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, row_height_minus1), 514, "VAPictureParameterBufferHEVC.row_height_minus1"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, slice_parsing_fields), 556, "VAPictureParameterBufferHEVC.slice_parsing_fields"); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, log2_max_pic_order_cnt_lsb_minus4), + 560, + "VAPictureParameterBufferHEVC.log2_max_pic_order_cnt_lsb_minus4" + ); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_short_term_ref_pic_sets), 561, "VAPictureParameterBufferHEVC.num_short_term_ref_pic_sets"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_long_term_ref_pic_sps), 562, "VAPictureParameterBufferHEVC.num_long_term_ref_pic_sps"); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, num_ref_idx_l0_default_active_minus1), + 563, + "VAPictureParameterBufferHEVC.num_ref_idx_l0_default_active_minus1" + ); + assert_eq!( + offset_of!(VAPictureParameterBufferHEVC, num_ref_idx_l1_default_active_minus1), + 564, + "VAPictureParameterBufferHEVC.num_ref_idx_l1_default_active_minus1" + ); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_beta_offset_div2), 565, "VAPictureParameterBufferHEVC.pps_beta_offset_div2"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_tc_offset_div2), 566, "VAPictureParameterBufferHEVC.pps_tc_offset_div2"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_extra_slice_header_bits), 567, "VAPictureParameterBufferHEVC.num_extra_slice_header_bits"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, st_rps_bits), 568, "VAPictureParameterBufferHEVC.st_rps_bits"); + assert_eq!(offset_of!(VAPictureParameterBufferHEVC, va_reserved), 572, "VAPictureParameterBufferHEVC.va_reserved"); + assert_eq!((size_of::(), align_of::()), (264, 4), "VASliceParameterBufferHEVC"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_size), 0, "VASliceParameterBufferHEVC.slice_data_size"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_offset), 4, "VASliceParameterBufferHEVC.slice_data_offset"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_flag), 8, "VASliceParameterBufferHEVC.slice_data_flag"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_byte_offset), 12, "VASliceParameterBufferHEVC.slice_data_byte_offset"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_segment_address), 16, "VASliceParameterBufferHEVC.slice_segment_address"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, RefPicList), 20, "VASliceParameterBufferHEVC.RefPicList"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, LongSliceFlags), 52, "VASliceParameterBufferHEVC.LongSliceFlags"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, collocated_ref_idx), 56, "VASliceParameterBufferHEVC.collocated_ref_idx"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, num_ref_idx_l0_active_minus1), 57, "VASliceParameterBufferHEVC.num_ref_idx_l0_active_minus1"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, num_ref_idx_l1_active_minus1), 58, "VASliceParameterBufferHEVC.num_ref_idx_l1_active_minus1"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_qp_delta), 59, "VASliceParameterBufferHEVC.slice_qp_delta"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_cb_qp_offset), 60, "VASliceParameterBufferHEVC.slice_cb_qp_offset"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_cr_qp_offset), 61, "VASliceParameterBufferHEVC.slice_cr_qp_offset"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_beta_offset_div2), 62, "VASliceParameterBufferHEVC.slice_beta_offset_div2"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_tc_offset_div2), 63, "VASliceParameterBufferHEVC.slice_tc_offset_div2"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, luma_log2_weight_denom), 64, "VASliceParameterBufferHEVC.luma_log2_weight_denom"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_chroma_log2_weight_denom), 65, "VASliceParameterBufferHEVC.delta_chroma_log2_weight_denom"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_luma_weight_l0), 66, "VASliceParameterBufferHEVC.delta_luma_weight_l0"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, luma_offset_l0), 81, "VASliceParameterBufferHEVC.luma_offset_l0"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_chroma_weight_l0), 96, "VASliceParameterBufferHEVC.delta_chroma_weight_l0"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, ChromaOffsetL0), 126, "VASliceParameterBufferHEVC.ChromaOffsetL0"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_luma_weight_l1), 156, "VASliceParameterBufferHEVC.delta_luma_weight_l1"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, luma_offset_l1), 171, "VASliceParameterBufferHEVC.luma_offset_l1"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_chroma_weight_l1), 186, "VASliceParameterBufferHEVC.delta_chroma_weight_l1"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, ChromaOffsetL1), 216, "VASliceParameterBufferHEVC.ChromaOffsetL1"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, five_minus_max_num_merge_cand), 246, "VASliceParameterBufferHEVC.five_minus_max_num_merge_cand"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, num_entry_point_offsets), 248, "VASliceParameterBufferHEVC.num_entry_point_offsets"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, entry_offset_to_subset_array), 250, "VASliceParameterBufferHEVC.entry_offset_to_subset_array"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_num_emu_prevn_bytes), 252, "VASliceParameterBufferHEVC.slice_data_num_emu_prevn_bytes"); + assert_eq!(offset_of!(VASliceParameterBufferHEVC, va_reserved), 256, "VASliceParameterBufferHEVC.va_reserved"); + assert_eq!((size_of::(), align_of::()), (1016, 4), "VAIQMatrixBufferHEVC"); + assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList4x4), 0, "VAIQMatrixBufferHEVC.ScalingList4x4"); + assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList8x8), 96, "VAIQMatrixBufferHEVC.ScalingList8x8"); + assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList16x16), 480, "VAIQMatrixBufferHEVC.ScalingList16x16"); + assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList32x32), 864, "VAIQMatrixBufferHEVC.ScalingList32x32"); + assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingListDC16x16), 992, "VAIQMatrixBufferHEVC.ScalingListDC16x16"); + assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingListDC32x32), 998, "VAIQMatrixBufferHEVC.ScalingListDC32x32"); + assert_eq!(offset_of!(VAIQMatrixBufferHEVC, va_reserved), 1000, "VAIQMatrixBufferHEVC.va_reserved"); +} + +#[test] +fn constants_match_the_header() { + assert_eq!(VA_STATUS_SUCCESS as i64, 0, "VA_STATUS_SUCCESS"); + assert_eq!(VA_STATUS_ERROR_UNSUPPORTED_PROFILE as i64, 12, "VA_STATUS_ERROR_UNSUPPORTED_PROFILE"); + assert_eq!(VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT as i64, 13, "VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT"); + assert_eq!(VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT as i64, 14, "VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT"); + assert_eq!(VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED as i64, 19, "VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED"); + assert_eq!(VA_STATUS_ERROR_DECODING_ERROR as i64, 23, "VA_STATUS_ERROR_DECODING_ERROR"); + assert_eq!(VA_INVALID_ID as i64, 4294967295, "VA_INVALID_ID"); + assert_eq!(VA_INVALID_SURFACE as i64, 4294967295, "VA_INVALID_SURFACE"); + assert_eq!(VAProfileH264Main as i64, 6, "VAProfileH264Main"); + assert_eq!(VAProfileH264High as i64, 7, "VAProfileH264High"); + assert_eq!(VAProfileH264ConstrainedBaseline as i64, 13, "VAProfileH264ConstrainedBaseline"); + assert_eq!(VAEntrypointVLD as i64, 1, "VAEntrypointVLD"); + assert_eq!(VAConfigAttribRTFormat as i64, 0, "VAConfigAttribRTFormat"); + assert_eq!(VA_RT_FORMAT_YUV420 as i64, 1, "VA_RT_FORMAT_YUV420"); + assert_eq!(VA_PROGRESSIVE as i64, 1, "VA_PROGRESSIVE"); + assert_eq!(VAPictureParameterBufferType as i64, 0, "VAPictureParameterBufferType"); + assert_eq!(VAIQMatrixBufferType as i64, 1, "VAIQMatrixBufferType"); + assert_eq!(VASliceParameterBufferType as i64, 4, "VASliceParameterBufferType"); + assert_eq!(VASliceDataBufferType as i64, 5, "VASliceDataBufferType"); + assert_eq!(VA_SLICE_DATA_FLAG_ALL as i64, 0, "VA_SLICE_DATA_FLAG_ALL"); + assert_eq!(VA_PICTURE_H264_INVALID as i64, 1, "VA_PICTURE_H264_INVALID"); + assert_eq!(VA_PICTURE_H264_SHORT_TERM_REFERENCE as i64, 8, "VA_PICTURE_H264_SHORT_TERM_REFERENCE"); + assert_eq!(VA_PICTURE_H264_LONG_TERM_REFERENCE as i64, 16, "VA_PICTURE_H264_LONG_TERM_REFERENCE"); + assert_eq!(VA_FOURCC_NV12 as i64, 842094158, "VA_FOURCC_NV12"); + assert_eq!(VA_FOURCC_P010 as i64, 808530000, "VA_FOURCC_P010"); + assert_eq!(VAProfileHEVCMain as i64, 17, "VAProfileHEVCMain"); + assert_eq!(VAProfileHEVCMain10 as i64, 18, "VAProfileHEVCMain10"); + assert_eq!(VA_RT_FORMAT_YUV420_10 as i64, 256, "VA_RT_FORMAT_YUV420_10"); + assert_eq!(VA_PICTURE_HEVC_INVALID as i64, 1, "VA_PICTURE_HEVC_INVALID"); + assert_eq!(VA_PICTURE_HEVC_LONG_TERM_REFERENCE as i64, 8, "VA_PICTURE_HEVC_LONG_TERM_REFERENCE"); + assert_eq!(VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE as i64, 16, "VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE"); + assert_eq!(VA_PICTURE_HEVC_RPS_ST_CURR_AFTER as i64, 32, "VA_PICTURE_HEVC_RPS_ST_CURR_AFTER"); + assert_eq!(VA_PICTURE_HEVC_RPS_LT_CURR as i64, 64, "VA_PICTURE_HEVC_RPS_LT_CURR"); + assert_eq!(VA_PADDING_LOW as i64, 4, "VA_PADDING_LOW"); + assert_eq!(VA_PADDING_MEDIUM as i64, 8, "VA_PADDING_MEDIUM"); +} + +/// Each bit-field set alone to its largest value: the word the C compiler makes of it. +#[test] +fn bit_fields_match_the_header() { + assert_eq!(3u32 << seq_bits::CHROMA_FORMAT_IDC, 0x3, "seq_fields.chroma_format_idc"); + assert_eq!(1u32 << seq_bits::RESIDUAL_COLOUR_TRANSFORM, 0x4, "seq_fields.residual_colour_transform_flag"); + assert_eq!(1u32 << seq_bits::GAPS_IN_FRAME_NUM_ALLOWED, 0x8, "seq_fields.gaps_in_frame_num_value_allowed_flag"); + assert_eq!(1u32 << seq_bits::FRAME_MBS_ONLY, 0x10, "seq_fields.frame_mbs_only_flag"); + assert_eq!(1u32 << seq_bits::MB_ADAPTIVE_FRAME_FIELD, 0x20, "seq_fields.mb_adaptive_frame_field_flag"); + assert_eq!(1u32 << seq_bits::DIRECT_8X8_INFERENCE, 0x40, "seq_fields.direct_8x8_inference_flag"); + assert_eq!(1u32 << seq_bits::MIN_LUMA_BI_PRED_SIZE_8X8, 0x80, "seq_fields.MinLumaBiPredSize8x8"); + assert_eq!(15u32 << seq_bits::LOG2_MAX_FRAME_NUM_MINUS4, 0xf00, "seq_fields.log2_max_frame_num_minus4"); + assert_eq!(3u32 << seq_bits::PIC_ORDER_CNT_TYPE, 0x3000, "seq_fields.pic_order_cnt_type"); + assert_eq!(15u32 << seq_bits::LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4, 0x3c000, "seq_fields.log2_max_pic_order_cnt_lsb_minus4"); + assert_eq!(1u32 << seq_bits::DELTA_PIC_ORDER_ALWAYS_ZERO, 0x40000, "seq_fields.delta_pic_order_always_zero_flag"); + assert_eq!(1u32 << pic_bits::ENTROPY_CODING_MODE, 0x1, "pic_fields.entropy_coding_mode_flag"); + assert_eq!(1u32 << pic_bits::WEIGHTED_PRED, 0x2, "pic_fields.weighted_pred_flag"); + assert_eq!(3u32 << pic_bits::WEIGHTED_BIPRED_IDC, 0xc, "pic_fields.weighted_bipred_idc"); + assert_eq!(1u32 << pic_bits::TRANSFORM_8X8_MODE, 0x10, "pic_fields.transform_8x8_mode_flag"); + assert_eq!(1u32 << pic_bits::FIELD_PIC, 0x20, "pic_fields.field_pic_flag"); + assert_eq!(1u32 << pic_bits::CONSTRAINED_INTRA_PRED, 0x40, "pic_fields.constrained_intra_pred_flag"); + assert_eq!(1u32 << pic_bits::PIC_ORDER_PRESENT, 0x80, "pic_fields.pic_order_present_flag"); + assert_eq!(1u32 << pic_bits::DEBLOCKING_FILTER_CONTROL_PRESENT, 0x100, "pic_fields.deblocking_filter_control_present_flag"); + assert_eq!(1u32 << pic_bits::REDUNDANT_PIC_CNT_PRESENT, 0x200, "pic_fields.redundant_pic_cnt_present_flag"); + assert_eq!(1u32 << pic_bits::REFERENCE_PIC, 0x400, "pic_fields.reference_pic_flag"); + assert_eq!(3u32 << hevc_pic_bits::CHROMA_FORMAT_IDC, 0x3, "hevc pic_fields.chroma_format_idc"); + assert_eq!(1u32 << hevc_pic_bits::SEPARATE_COLOUR_PLANE, 0x4, "hevc pic_fields.separate_colour_plane_flag"); + assert_eq!(1u32 << hevc_pic_bits::PCM_ENABLED, 0x8, "hevc pic_fields.pcm_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::SCALING_LIST_ENABLED, 0x10, "hevc pic_fields.scaling_list_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::TRANSFORM_SKIP_ENABLED, 0x20, "hevc pic_fields.transform_skip_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::AMP_ENABLED, 0x40, "hevc pic_fields.amp_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::STRONG_INTRA_SMOOTHING_ENABLED, 0x80, "hevc pic_fields.strong_intra_smoothing_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::SIGN_DATA_HIDING_ENABLED, 0x100, "hevc pic_fields.sign_data_hiding_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::CONSTRAINED_INTRA_PRED, 0x200, "hevc pic_fields.constrained_intra_pred_flag"); + assert_eq!(1u32 << hevc_pic_bits::CU_QP_DELTA_ENABLED, 0x400, "hevc pic_fields.cu_qp_delta_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::WEIGHTED_PRED, 0x800, "hevc pic_fields.weighted_pred_flag"); + assert_eq!(1u32 << hevc_pic_bits::WEIGHTED_BIPRED, 0x1000, "hevc pic_fields.weighted_bipred_flag"); + assert_eq!(1u32 << hevc_pic_bits::TRANSQUANT_BYPASS_ENABLED, 0x2000, "hevc pic_fields.transquant_bypass_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::TILES_ENABLED, 0x4000, "hevc pic_fields.tiles_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::ENTROPY_CODING_SYNC_ENABLED, 0x8000, "hevc pic_fields.entropy_coding_sync_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED, 0x10000, "hevc pic_fields.pps_loop_filter_across_slices_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::LOOP_FILTER_ACROSS_TILES_ENABLED, 0x20000, "hevc pic_fields.loop_filter_across_tiles_enabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::PCM_LOOP_FILTER_DISABLED, 0x40000, "hevc pic_fields.pcm_loop_filter_disabled_flag"); + assert_eq!(1u32 << hevc_pic_bits::NO_PIC_REORDERING, 0x80000, "hevc pic_fields.NoPicReorderingFlag"); + assert_eq!(1u32 << hevc_pic_bits::NO_BI_PRED, 0x100000, "hevc pic_fields.NoBiPredFlag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::LISTS_MODIFICATION_PRESENT, 0x1, "hevc slice_parsing_fields.lists_modification_present_flag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::LONG_TERM_REF_PICS_PRESENT, 0x2, "hevc slice_parsing_fields.long_term_ref_pics_present_flag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::SPS_TEMPORAL_MVP_ENABLED, 0x4, "hevc slice_parsing_fields.sps_temporal_mvp_enabled_flag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::CABAC_INIT_PRESENT, 0x8, "hevc slice_parsing_fields.cabac_init_present_flag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::OUTPUT_FLAG_PRESENT, 0x10, "hevc slice_parsing_fields.output_flag_present_flag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::DEPENDENT_SLICE_SEGMENTS_ENABLED, 0x20, "hevc slice_parsing_fields.dependent_slice_segments_enabled_flag"); + assert_eq!( + 1u32 << hevc_slice_parsing_bits::PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT, + 0x40, + "hevc slice_parsing_fields.pps_slice_chroma_qp_offsets_present_flag" + ); + assert_eq!(1u32 << hevc_slice_parsing_bits::SAMPLE_ADAPTIVE_OFFSET_ENABLED, 0x80, "hevc slice_parsing_fields.sample_adaptive_offset_enabled_flag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::DEBLOCKING_FILTER_OVERRIDE_ENABLED, 0x100, "hevc slice_parsing_fields.deblocking_filter_override_enabled_flag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::PPS_DISABLE_DEBLOCKING_FILTER, 0x200, "hevc slice_parsing_fields.pps_disable_deblocking_filter_flag"); + assert_eq!( + 1u32 << hevc_slice_parsing_bits::SLICE_SEGMENT_HEADER_EXTENSION_PRESENT, + 0x400, + "hevc slice_parsing_fields.slice_segment_header_extension_present_flag" + ); + assert_eq!(1u32 << hevc_slice_parsing_bits::RAP_PIC, 0x800, "hevc slice_parsing_fields.RapPicFlag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::IDR_PIC, 0x1000, "hevc slice_parsing_fields.IdrPicFlag"); + assert_eq!(1u32 << hevc_slice_parsing_bits::INTRA_PIC, 0x2000, "hevc slice_parsing_fields.IntraPicFlag"); + assert_eq!(1u32 << hevc_slice_bits::LAST_SLICE_OF_PIC, 0x1, "LongSliceFlags.LastSliceOfPic"); + assert_eq!(1u32 << hevc_slice_bits::DEPENDENT_SLICE_SEGMENT, 0x2, "LongSliceFlags.dependent_slice_segment_flag"); + assert_eq!(3u32 << hevc_slice_bits::SLICE_TYPE, 0xc, "LongSliceFlags.slice_type"); + assert_eq!(3u32 << hevc_slice_bits::COLOR_PLANE_ID, 0x30, "LongSliceFlags.color_plane_id"); + assert_eq!(1u32 << hevc_slice_bits::SLICE_SAO_LUMA, 0x40, "LongSliceFlags.slice_sao_luma_flag"); + assert_eq!(1u32 << hevc_slice_bits::SLICE_SAO_CHROMA, 0x80, "LongSliceFlags.slice_sao_chroma_flag"); + assert_eq!(1u32 << hevc_slice_bits::MVD_L1_ZERO, 0x100, "LongSliceFlags.mvd_l1_zero_flag"); + assert_eq!(1u32 << hevc_slice_bits::CABAC_INIT, 0x200, "LongSliceFlags.cabac_init_flag"); + assert_eq!(1u32 << hevc_slice_bits::SLICE_TEMPORAL_MVP_ENABLED, 0x400, "LongSliceFlags.slice_temporal_mvp_enabled_flag"); + assert_eq!(1u32 << hevc_slice_bits::SLICE_DEBLOCKING_FILTER_DISABLED, 0x800, "LongSliceFlags.slice_deblocking_filter_disabled_flag"); + assert_eq!(1u32 << hevc_slice_bits::COLLOCATED_FROM_L0, 0x1000, "LongSliceFlags.collocated_from_l0_flag"); + assert_eq!(1u32 << hevc_slice_bits::SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED, 0x2000, "LongSliceFlags.slice_loop_filter_across_slices_enabled_flag"); +} diff --git a/crates/platform/src/vaapi/ffi.rs b/crates/platform/src/vaapi/ffi.rs new file mode 100644 index 00000000..4fc2e28e --- /dev/null +++ b/crates/platform/src/vaapi/ffi.rs @@ -0,0 +1,519 @@ +// The declarations in this file are transcribed from libva's `va/va.h`, `va/va_dec_hevc.h` and `va/va_drm.h` +// (VA-API 1.23, libva 2.23), which carry this notice: +// +// Copyright (c) 2007-2009 Intel Corporation. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sub license, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice (including the +// next paragraph) shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. +// IN NO EVENT SHALL INTEL AND/OR ITS SUPPLIERS BE LIABLE FOR +// ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +//! VA-API (`va.h`, `va_dec_hevc.h`, `va_drm.h`; MIT-licensed headers by Intel) as `libva.so.2` / +//! `libva-drm.so.2` expect it: the H.264 and HEVC decode buffers this backend fills in, the image structures it reads back +//! through, and the entry points it calls. Written from the public header; sizes and field offsets +//! are checked against a C compiler's by `abi_tests.rs`. +//! +//! The structures are plain data and compile on every target, so the code that fills them in +//! (`vaapi::h264`) is tested everywhere; only `vaapi::va` loads the library (Linux). +//! +//! FFI module (docs/adr/0001-platform-ffi.md): plain `repr(C)` data, no logic. + +#![allow(non_snake_case, non_camel_case_types, non_upper_case_globals, dead_code)] + +use std::ffi::{c_char, c_int, c_uint, c_void}; + +pub type VADisplay = *mut c_void; +pub type VAStatus = c_int; +pub type VAGenericID = c_uint; +pub type VAConfigID = VAGenericID; +pub type VAContextID = VAGenericID; +pub type VASurfaceID = VAGenericID; +pub type VABufferID = VAGenericID; +pub type VAImageID = VAGenericID; +/// `VAProfile` (a C enum: `int`). +pub type VAProfile = c_int; +/// `VAEntrypoint` (a C enum: `int`). +pub type VAEntrypoint = c_int; +/// `VABufferType` (a C enum: `int`). +pub type VABufferType = c_int; +/// `VAConfigAttribType` (a C enum: `int`). +pub type VAConfigAttribType = c_int; +pub type VAMessageCallback = Option; + +pub const VA_STATUS_SUCCESS: VAStatus = 0; +pub const VA_STATUS_ERROR_UNSUPPORTED_PROFILE: VAStatus = 0x0c; +pub const VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT: VAStatus = 0x0d; +pub const VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT: VAStatus = 0x0e; +pub const VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED: VAStatus = 0x13; +pub const VA_STATUS_ERROR_DECODING_ERROR: VAStatus = 0x17; + +pub const VA_INVALID_ID: VAGenericID = 0xffff_ffff; +pub const VA_INVALID_SURFACE: VASurfaceID = VA_INVALID_ID; + +pub const VAProfileH264Main: VAProfile = 6; +pub const VAProfileH264High: VAProfile = 7; +pub const VAProfileH264ConstrainedBaseline: VAProfile = 13; +pub const VAProfileHEVCMain: VAProfile = 17; +pub const VAProfileHEVCMain10: VAProfile = 18; +pub const VAEntrypointVLD: VAEntrypoint = 1; + +pub const VAConfigAttribRTFormat: VAConfigAttribType = 0; +pub const VA_RT_FORMAT_YUV420: c_uint = 0x0000_0001; +pub const VA_RT_FORMAT_YUV420_10: c_uint = 0x0000_0100; + +/// `vaCreateContext` flag: progressive pictures only. +pub const VA_PROGRESSIVE: c_int = 0x1; + +pub const VAPictureParameterBufferType: VABufferType = 0; +pub const VAIQMatrixBufferType: VABufferType = 1; +pub const VASliceParameterBufferType: VABufferType = 4; +pub const VASliceDataBufferType: VABufferType = 5; + +/// `VASliceParameterBuffer*::slice_data_flag`: the whole slice is in the buffer. +pub const VA_SLICE_DATA_FLAG_ALL: u32 = 0x00; + +pub const VA_PICTURE_H264_INVALID: u32 = 0x0000_0001; +pub const VA_PICTURE_H264_SHORT_TERM_REFERENCE: u32 = 0x0000_0008; +pub const VA_PICTURE_H264_LONG_TERM_REFERENCE: u32 = 0x0000_0010; + +pub const VA_FOURCC_NV12: u32 = 0x3231_564E; +pub const VA_FOURCC_P010: u32 = 0x3031_3050; + +pub const VA_PICTURE_HEVC_INVALID: u32 = 0x0000_0001; +pub const VA_PICTURE_HEVC_LONG_TERM_REFERENCE: u32 = 0x0000_0008; +pub const VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE: u32 = 0x0000_0010; +pub const VA_PICTURE_HEVC_RPS_ST_CURR_AFTER: u32 = 0x0000_0020; +pub const VA_PICTURE_HEVC_RPS_LT_CURR: u32 = 0x0000_0040; + +pub const VA_PADDING_LOW: usize = 4; +pub const VA_PADDING_MEDIUM: usize = 8; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct VAConfigAttrib { + pub type_: VAConfigAttribType, + pub value: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VAPictureH264 { + pub picture_id: VASurfaceID, + pub frame_idx: u32, + pub flags: u32, + pub TopFieldOrderCnt: i32, + pub BottomFieldOrderCnt: i32, + pub va_reserved: [u32; VA_PADDING_LOW], +} + +impl VAPictureH264 { + /// An unused entry (`VA_INVALID_SURFACE`, `VA_PICTURE_H264_INVALID`). + pub const INVALID: Self = Self { + picture_id: VA_INVALID_SURFACE, + frame_idx: 0, + flags: VA_PICTURE_H264_INVALID, + TopFieldOrderCnt: 0, + BottomFieldOrderCnt: 0, + va_reserved: [0; VA_PADDING_LOW], + }; +} + +/// `VAPictureParameterBufferH264`. `seq_fields` and `pic_fields` are C bit-field unions, stored +/// here as their `value` word (bits allocated from the least significant end, as GCC and Clang do +/// on the little-endian targets libva runs on). +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct VAPictureParameterBufferH264 { + pub CurrPic: VAPictureH264, + pub ReferenceFrames: [VAPictureH264; 16], + pub picture_width_in_mbs_minus1: u16, + pub picture_height_in_mbs_minus1: u16, + pub bit_depth_luma_minus8: u8, + pub bit_depth_chroma_minus8: u8, + pub num_ref_frames: u8, + pub seq_fields: u32, + pub num_slice_groups_minus1: u8, + pub slice_group_map_type: u8, + pub slice_group_change_rate_minus1: u16, + pub pic_init_qp_minus26: i8, + pub pic_init_qs_minus26: i8, + pub chroma_qp_index_offset: i8, + pub second_chroma_qp_index_offset: i8, + pub pic_fields: u32, + pub frame_num: u16, + pub va_reserved: [u32; VA_PADDING_MEDIUM], +} + +/// `seq_fields` bit positions (`chroma_format_idc` 2 bits, the log2 fields 4, `pic_order_cnt_type` 2). +pub mod seq_bits { + pub const CHROMA_FORMAT_IDC: u32 = 0; + pub const RESIDUAL_COLOUR_TRANSFORM: u32 = 2; + pub const GAPS_IN_FRAME_NUM_ALLOWED: u32 = 3; + pub const FRAME_MBS_ONLY: u32 = 4; + pub const MB_ADAPTIVE_FRAME_FIELD: u32 = 5; + pub const DIRECT_8X8_INFERENCE: u32 = 6; + pub const MIN_LUMA_BI_PRED_SIZE_8X8: u32 = 7; + pub const LOG2_MAX_FRAME_NUM_MINUS4: u32 = 8; + pub const PIC_ORDER_CNT_TYPE: u32 = 12; + pub const LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4: u32 = 14; + pub const DELTA_PIC_ORDER_ALWAYS_ZERO: u32 = 18; +} + +/// `pic_fields` bit positions (`weighted_bipred_idc` 2 bits). +pub mod pic_bits { + pub const ENTROPY_CODING_MODE: u32 = 0; + pub const WEIGHTED_PRED: u32 = 1; + pub const WEIGHTED_BIPRED_IDC: u32 = 2; + pub const TRANSFORM_8X8_MODE: u32 = 4; + pub const FIELD_PIC: u32 = 5; + pub const CONSTRAINED_INTRA_PRED: u32 = 6; + pub const PIC_ORDER_PRESENT: u32 = 7; + pub const DEBLOCKING_FILTER_CONTROL_PRESENT: u32 = 8; + pub const REDUNDANT_PIC_CNT_PRESENT: u32 = 9; + pub const REFERENCE_PIC: u32 = 10; +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct VAIQMatrixBufferH264 { + /// 4x4 scaling lists in raster scan order: Intra Y, Cb, Cr, Inter Y, Cb, Cr. + pub ScalingList4x4: [[u8; 16]; 6], + /// 8x8 scaling lists in raster scan order: Intra Y, Inter Y. + pub ScalingList8x8: [[u8; 64]; 2], + pub va_reserved: [u32; VA_PADDING_LOW], +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct VASliceParameterBufferH264 { + pub slice_data_size: u32, + pub slice_data_offset: u32, + pub slice_data_flag: u32, + /// Bits from the start of the NAL unit (its header byte included) to `slice_data()`, counted + /// without emulation prevention bytes; the data buffer itself keeps them. + pub slice_data_bit_offset: u16, + pub first_mb_in_slice: u16, + pub slice_type: u8, + pub direct_spatial_mv_pred_flag: u8, + pub num_ref_idx_l0_active_minus1: u8, + pub num_ref_idx_l1_active_minus1: u8, + pub cabac_init_idc: u8, + pub slice_qp_delta: i8, + pub disable_deblocking_filter_idc: u8, + pub slice_alpha_c0_offset_div2: i8, + pub slice_beta_offset_div2: i8, + pub RefPicList0: [VAPictureH264; 32], + pub RefPicList1: [VAPictureH264; 32], + pub luma_log2_weight_denom: u8, + pub chroma_log2_weight_denom: u8, + pub luma_weight_l0_flag: u8, + pub luma_weight_l0: [i16; 32], + pub luma_offset_l0: [i16; 32], + pub chroma_weight_l0_flag: u8, + pub chroma_weight_l0: [[i16; 2]; 32], + pub chroma_offset_l0: [[i16; 2]; 32], + pub luma_weight_l1_flag: u8, + pub luma_weight_l1: [i16; 32], + pub luma_offset_l1: [i16; 32], + pub chroma_weight_l1_flag: u8, + pub chroma_weight_l1: [[i16; 2]; 32], + pub chroma_offset_l1: [[i16; 2]; 32], + pub va_reserved: [u32; VA_PADDING_LOW], +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VAPictureHEVC { + pub picture_id: VASurfaceID, + pub pic_order_cnt: i32, + pub flags: u32, + pub va_reserved: [u32; VA_PADDING_LOW], +} + +impl VAPictureHEVC { + /// An unused entry (`VA_INVALID_SURFACE`, `VA_PICTURE_HEVC_INVALID`). + pub const INVALID: Self = Self { picture_id: VA_INVALID_SURFACE, pic_order_cnt: 0, flags: VA_PICTURE_HEVC_INVALID, va_reserved: [0; VA_PADDING_LOW] }; +} + +/// `VAPictureParameterBufferHEVC`; `pic_fields` and `slice_parsing_fields` are C bit-field unions +/// stored as their `value` word (see [`VAPictureParameterBufferH264`]). +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct VAPictureParameterBufferHEVC { + pub CurrPic: VAPictureHEVC, + pub ReferenceFrames: [VAPictureHEVC; 15], + pub pic_width_in_luma_samples: u16, + pub pic_height_in_luma_samples: u16, + pub pic_fields: u32, + pub sps_max_dec_pic_buffering_minus1: u8, + pub bit_depth_luma_minus8: u8, + pub bit_depth_chroma_minus8: u8, + pub pcm_sample_bit_depth_luma_minus1: u8, + pub pcm_sample_bit_depth_chroma_minus1: u8, + pub log2_min_luma_coding_block_size_minus3: u8, + pub log2_diff_max_min_luma_coding_block_size: u8, + pub log2_min_transform_block_size_minus2: u8, + pub log2_diff_max_min_transform_block_size: u8, + pub log2_min_pcm_luma_coding_block_size_minus3: u8, + pub log2_diff_max_min_pcm_luma_coding_block_size: u8, + pub max_transform_hierarchy_depth_intra: u8, + pub max_transform_hierarchy_depth_inter: u8, + pub init_qp_minus26: i8, + pub diff_cu_qp_delta_depth: u8, + pub pps_cb_qp_offset: i8, + pub pps_cr_qp_offset: i8, + pub log2_parallel_merge_level_minus2: u8, + pub num_tile_columns_minus1: u8, + pub num_tile_rows_minus1: u8, + pub column_width_minus1: [u16; 19], + pub row_height_minus1: [u16; 21], + pub slice_parsing_fields: u32, + pub log2_max_pic_order_cnt_lsb_minus4: u8, + pub num_short_term_ref_pic_sets: u8, + pub num_long_term_ref_pic_sps: u8, + pub num_ref_idx_l0_default_active_minus1: u8, + pub num_ref_idx_l1_default_active_minus1: u8, + pub pps_beta_offset_div2: i8, + pub pps_tc_offset_div2: i8, + pub num_extra_slice_header_bits: u8, + /// Bits of `short_term_ref_pic_set()` in the slice header (0 when the SPS's set is used), + /// counted without emulation prevention bytes. + pub st_rps_bits: u32, + pub va_reserved: [u32; VA_PADDING_MEDIUM], +} + +/// `VAPictureParameterBufferHEVC::pic_fields` bit positions (`chroma_format_idc` 2 bits). +pub mod hevc_pic_bits { + pub const CHROMA_FORMAT_IDC: u32 = 0; + pub const SEPARATE_COLOUR_PLANE: u32 = 2; + pub const PCM_ENABLED: u32 = 3; + pub const SCALING_LIST_ENABLED: u32 = 4; + pub const TRANSFORM_SKIP_ENABLED: u32 = 5; + pub const AMP_ENABLED: u32 = 6; + pub const STRONG_INTRA_SMOOTHING_ENABLED: u32 = 7; + pub const SIGN_DATA_HIDING_ENABLED: u32 = 8; + pub const CONSTRAINED_INTRA_PRED: u32 = 9; + pub const CU_QP_DELTA_ENABLED: u32 = 10; + pub const WEIGHTED_PRED: u32 = 11; + pub const WEIGHTED_BIPRED: u32 = 12; + pub const TRANSQUANT_BYPASS_ENABLED: u32 = 13; + pub const TILES_ENABLED: u32 = 14; + pub const ENTROPY_CODING_SYNC_ENABLED: u32 = 15; + pub const PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED: u32 = 16; + pub const LOOP_FILTER_ACROSS_TILES_ENABLED: u32 = 17; + pub const PCM_LOOP_FILTER_DISABLED: u32 = 18; + pub const NO_PIC_REORDERING: u32 = 19; + pub const NO_BI_PRED: u32 = 20; +} + +/// `VAPictureParameterBufferHEVC::slice_parsing_fields` bit positions. +pub mod hevc_slice_parsing_bits { + pub const LISTS_MODIFICATION_PRESENT: u32 = 0; + pub const LONG_TERM_REF_PICS_PRESENT: u32 = 1; + pub const SPS_TEMPORAL_MVP_ENABLED: u32 = 2; + pub const CABAC_INIT_PRESENT: u32 = 3; + pub const OUTPUT_FLAG_PRESENT: u32 = 4; + pub const DEPENDENT_SLICE_SEGMENTS_ENABLED: u32 = 5; + pub const PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT: u32 = 6; + pub const SAMPLE_ADAPTIVE_OFFSET_ENABLED: u32 = 7; + pub const DEBLOCKING_FILTER_OVERRIDE_ENABLED: u32 = 8; + pub const PPS_DISABLE_DEBLOCKING_FILTER: u32 = 9; + pub const SLICE_SEGMENT_HEADER_EXTENSION_PRESENT: u32 = 10; + pub const RAP_PIC: u32 = 11; + pub const IDR_PIC: u32 = 12; + pub const INTRA_PIC: u32 = 13; +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct VASliceParameterBufferHEVC { + pub slice_data_size: u32, + pub slice_data_offset: u32, + pub slice_data_flag: u32, + /// Bytes from the start of the NAL unit (its two-byte header included) to + /// `slice_segment_data()`, counted without emulation prevention bytes; the data buffer itself + /// keeps them. + pub slice_data_byte_offset: u32, + pub slice_segment_address: u32, + /// Indices into `ReferenceFrames` (0xFF: none). + pub RefPicList: [[u8; 15]; 2], + pub LongSliceFlags: u32, + pub collocated_ref_idx: u8, + pub num_ref_idx_l0_active_minus1: u8, + pub num_ref_idx_l1_active_minus1: u8, + pub slice_qp_delta: i8, + pub slice_cb_qp_offset: i8, + pub slice_cr_qp_offset: i8, + pub slice_beta_offset_div2: i8, + pub slice_tc_offset_div2: i8, + pub luma_log2_weight_denom: u8, + pub delta_chroma_log2_weight_denom: i8, + pub delta_luma_weight_l0: [i8; 15], + pub luma_offset_l0: [i8; 15], + pub delta_chroma_weight_l0: [[i8; 2]; 15], + pub ChromaOffsetL0: [[i8; 2]; 15], + pub delta_luma_weight_l1: [i8; 15], + pub luma_offset_l1: [i8; 15], + pub delta_chroma_weight_l1: [[i8; 2]; 15], + pub ChromaOffsetL1: [[i8; 2]; 15], + pub five_minus_max_num_merge_cand: u8, + pub num_entry_point_offsets: u16, + pub entry_offset_to_subset_array: u16, + /// Emulation prevention bytes in the slice segment header. + pub slice_data_num_emu_prevn_bytes: u16, + pub va_reserved: [u32; VA_PADDING_LOW - 2], +} + +/// `VASliceParameterBufferHEVC::LongSliceFlags` bit positions (`slice_type` and `color_plane_id` 2 bits). +pub mod hevc_slice_bits { + pub const LAST_SLICE_OF_PIC: u32 = 0; + pub const DEPENDENT_SLICE_SEGMENT: u32 = 1; + pub const SLICE_TYPE: u32 = 2; + pub const COLOR_PLANE_ID: u32 = 4; + pub const SLICE_SAO_LUMA: u32 = 6; + pub const SLICE_SAO_CHROMA: u32 = 7; + pub const MVD_L1_ZERO: u32 = 8; + pub const CABAC_INIT: u32 = 9; + pub const SLICE_TEMPORAL_MVP_ENABLED: u32 = 10; + pub const SLICE_DEBLOCKING_FILTER_DISABLED: u32 = 11; + pub const COLLOCATED_FROM_L0: u32 = 12; + pub const SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED: u32 = 13; +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct VAIQMatrixBufferHEVC { + /// `ScalingList[sizeId][matrixId][j]` of the spec for sizeId 0..3, raster order. + pub ScalingList4x4: [[u8; 16]; 6], + pub ScalingList8x8: [[u8; 64]; 6], + pub ScalingList16x16: [[u8; 64]; 6], + pub ScalingList32x32: [[u8; 64]; 2], + pub ScalingListDC16x16: [u8; 6], + pub ScalingListDC32x32: [u8; 2], + pub va_reserved: [u32; VA_PADDING_LOW], +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct VAImageFormat { + pub fourcc: u32, + pub byte_order: u32, + pub bits_per_pixel: u32, + pub depth: u32, + pub red_mask: u32, + pub green_mask: u32, + pub blue_mask: u32, + pub alpha_mask: u32, + pub va_reserved: [u32; VA_PADDING_LOW], +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct VAImage { + pub image_id: VAImageID, + pub format: VAImageFormat, + pub buf: VABufferID, + pub width: u16, + pub height: u16, + pub data_size: u32, + pub num_planes: u32, + pub pitches: [u32; 3], + pub offsets: [u32; 3], + pub num_palette_entries: i32, + pub entry_bytes: i32, + pub component_order: [i8; 4], + pub va_reserved: [u32; VA_PADDING_LOW], +} + +/// The `libva.so.2` entry points used (all `VAStatus`-returning unless noted). +pub type vaInitialize = unsafe extern "C" fn(dpy: VADisplay, major: *mut c_int, minor: *mut c_int) -> VAStatus; +pub type vaTerminate = unsafe extern "C" fn(dpy: VADisplay) -> VAStatus; +pub type vaErrorStr = unsafe extern "C" fn(status: VAStatus) -> *const c_char; +pub type vaQueryVendorString = unsafe extern "C" fn(dpy: VADisplay) -> *const c_char; +pub type vaSetInfoCallback = unsafe extern "C" fn(dpy: VADisplay, callback: VAMessageCallback, user_context: *mut c_void) -> VAMessageCallback; +pub type vaSetErrorCallback = unsafe extern "C" fn(dpy: VADisplay, callback: VAMessageCallback, user_context: *mut c_void) -> VAMessageCallback; +pub type vaGetConfigAttributes = + unsafe extern "C" fn(dpy: VADisplay, profile: VAProfile, entrypoint: VAEntrypoint, attrib_list: *mut VAConfigAttrib, num_attribs: c_int) -> VAStatus; +pub type vaCreateConfig = unsafe extern "C" fn( + dpy: VADisplay, + profile: VAProfile, + entrypoint: VAEntrypoint, + attrib_list: *mut VAConfigAttrib, + num_attribs: c_int, + config_id: *mut VAConfigID, +) -> VAStatus; +pub type vaDestroyConfig = unsafe extern "C" fn(dpy: VADisplay, config_id: VAConfigID) -> VAStatus; +/// The last two arguments are `VASurfaceAttrib *attrib_list, unsigned int num_attribs`; this +/// backend passes none, so the attribute structure is not declared. +pub type vaCreateSurfaces = unsafe extern "C" fn( + dpy: VADisplay, + format: c_uint, + width: c_uint, + height: c_uint, + surfaces: *mut VASurfaceID, + num_surfaces: c_uint, + attrib_list: *mut c_void, + num_attribs: c_uint, +) -> VAStatus; +pub type vaDestroySurfaces = unsafe extern "C" fn(dpy: VADisplay, surfaces: *mut VASurfaceID, num_surfaces: c_int) -> VAStatus; +pub type vaCreateContext = unsafe extern "C" fn( + dpy: VADisplay, + config_id: VAConfigID, + picture_width: c_int, + picture_height: c_int, + flag: c_int, + render_targets: *mut VASurfaceID, + num_render_targets: c_int, + context: *mut VAContextID, +) -> VAStatus; +pub type vaDestroyContext = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID) -> VAStatus; +pub type vaCreateBuffer = unsafe extern "C" fn( + dpy: VADisplay, + context: VAContextID, + type_: VABufferType, + size: c_uint, + num_elements: c_uint, + data: *mut c_void, + buf_id: *mut VABufferID, +) -> VAStatus; +pub type vaDestroyBuffer = unsafe extern "C" fn(dpy: VADisplay, buffer_id: VABufferID) -> VAStatus; +pub type vaBeginPicture = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID, render_target: VASurfaceID) -> VAStatus; +pub type vaRenderPicture = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID, buffers: *mut VABufferID, num_buffers: c_int) -> VAStatus; +pub type vaEndPicture = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID) -> VAStatus; +pub type vaSyncSurface = unsafe extern "C" fn(dpy: VADisplay, render_target: VASurfaceID) -> VAStatus; +pub type vaCreateImage = unsafe extern "C" fn(dpy: VADisplay, format: *mut VAImageFormat, width: c_int, height: c_int, image: *mut VAImage) -> VAStatus; +pub type vaDestroyImage = unsafe extern "C" fn(dpy: VADisplay, image: VAImageID) -> VAStatus; +pub type vaGetImage = + unsafe extern "C" fn(dpy: VADisplay, surface: VASurfaceID, x: c_int, y: c_int, width: c_uint, height: c_uint, image: VAImageID) -> VAStatus; +pub type vaPutImage = unsafe extern "C" fn( + dpy: VADisplay, + surface: VASurfaceID, + image: VAImageID, + src_x: c_int, + src_y: c_int, + src_width: c_uint, + src_height: c_uint, + dest_x: c_int, + dest_y: c_int, + dest_width: c_uint, + dest_height: c_uint, +) -> VAStatus; +pub type vaMapBuffer = unsafe extern "C" fn(dpy: VADisplay, buf_id: VABufferID, pbuf: *mut *mut c_void) -> VAStatus; +pub type vaUnmapBuffer = unsafe extern "C" fn(dpy: VADisplay, buf_id: VABufferID) -> VAStatus; +/// `libva-drm.so.2`: the display of a DRM device (render node) file descriptor. +pub type vaGetDisplayDRM = unsafe extern "C" fn(fd: c_int) -> VADisplay; diff --git a/crates/platform/src/vaapi/h264.rs b/crates/platform/src/vaapi/h264.rs new file mode 100644 index 00000000..bc79ee9a --- /dev/null +++ b/crates/platform/src/vaapi/h264.rs @@ -0,0 +1,512 @@ +//! The H.264 side of the VA-API decoder: everything a stateless hardware decoder needs from the +//! host, done with the software decoder's own parsers (`filmcraft_h264::params`, `slice`) and its +//! decoded picture buffer (`filmcraft_h264::dpb`: reference marking, reference lists, output +//! order), so both decoders make the same decisions and output the same pictures in the same order. +//! +//! [`Front`] parses each access unit, fills the VA-API picture / matrix / slice buffers and hands +//! them to an [`Accel`] (the libva session on Linux, a recording stand-in in tests), then reads +//! back the pictures the DPB outputs. What the hardware path does not take is an error, so the +//! hybrid decoder replays the run in software: field / MBAFF coding, FMO, SP / SI and data +//! partitioning, stream changes the surfaces were not made for. +//! +//! Safe code; compiled on every target so its logic is tested everywhere. + +use std::sync::Arc; + +use filmcraft_bitstream::{length_prefixed_nals, unescape_rbsp}; +use filmcraft_codecs::{CodecError, DecodedFrame, Result}; +use filmcraft_h264::dpb::{Dpb, DpbEntry, Output, OutputMeta, RefMark}; +use filmcraft_h264::params::{Pps, Sps}; +use filmcraft_h264::slice::{NalHeader, Poc, PocState, SliceHeader, SliceType, is_new_picture, nal_type}; + +use super::Accel; +use super::ffi::{self, VAIQMatrixBufferH264, VAPictureH264, VAPictureParameterBufferH264, VASliceParameterBufferH264, pic_bits, seq_bits}; + +/// A picture in the DPB: its surface, field order counts (after an MMCO 5, relative to itself) and +/// place in decoding order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Surface { + pub index: usize, + pub top: i32, + pub bottom: i32, + pub seq: u64, +} + +/// A frame_num gap frame not given a surface yet. +const UNASSIGNED: usize = usize::MAX; + +/// The picture whose slices are being collected. +struct Pending { + target: usize, + sps: Arc, + pps: Arc, + first: SliceHeader, + poc: Poc, + pts: i64, + slices: Vec<(VASliceParameterBufferH264, Vec)>, +} + +/// H.264 front end of a stateless hardware decoder (see the module docs). +pub struct Front { + accel: A, + length_size: usize, + /// Coded size (macroblocks) the surfaces were made for. + mbs: (u32, u32), + spss: Vec>>, + ppss: Vec>>, + dpb: Dpb, + poc_state: PocState, + prev_ref_frame_num: u32, + active_sps: Option>, + pending: Option, + /// Pictures made so far (decoding order of [`Surface::seq`]). + seq: u64, +} + +fn decode_err(e: impl std::fmt::Display) -> CodecError { + CodecError::Decode(e.to_string()) +} + +fn unsupported(what: &str) -> CodecError { + CodecError::Decode(format!("the hardware decoder does not take {what}")) +} + +impl Front { + /// A front end for samples with `length_size`-byte NAL length prefixes whose sample entry + /// carried `parameter_sets` (SPS / PPS NAL units), decoding into `accel`'s surfaces of + /// `mbs` macroblocks. + pub fn new(accel: A, length_size: usize, mbs: (u32, u32), parameter_sets: &[Vec]) -> Result { + let mut f = Self { + accel, + length_size, + mbs, + spss: vec![None; 32], + ppss: vec![None; 256], + dpb: Dpb::new(), + poc_state: PocState::default(), + prev_ref_frame_num: 0, + active_sps: None, + pending: None, + seq: 0, + }; + f.load_parameter_sets(parameter_sets)?; + Ok(f) + } + + fn load_parameter_sets(&mut self, sets: &[Vec]) -> Result<()> { + let mut frames = Vec::new(); + for nal in sets { + self.handle_nal(nal, 0, &mut frames)?; + } + Ok(()) + } + + pub fn accel(&self) -> &A { + &self.accel + } + + /// Decode one access unit; pictures that left the DPB come back in output order. + pub fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + let nals = length_prefixed_nals(sample, self.length_size).map_err(decode_err)?; + let mut frames = Vec::new(); + for nal in nals { + self.handle_nal(nal, pts, &mut frames)?; + } + // the access unit is complete + self.submit_pending(&mut frames)?; + Ok(frames) + } + + /// Output every picture still held for reordering (references stay for what follows). + pub fn flush(&mut self) -> Result> { + let mut frames = Vec::new(); + self.submit_pending(&mut frames)?; + let mut outs = Vec::new(); + self.dpb.flush(&mut outs); + self.read_out(outs, &mut frames)?; + Ok(frames) + } + + /// Forget every picture (after a seek). + pub fn reset(&mut self) { + self.dpb = Dpb::new(); + self.poc_state = PocState::default(); + self.prev_ref_frame_num = 0; + self.active_sps = None; + self.pending = None; + } + + /// Read back the pictures the DPB output, right away: a surface leaving the DPB is reused by + /// the next picture. + fn read_out(&mut self, outs: Vec>, frames: &mut Vec) -> Result<()> { + for o in outs { + let frame = self.accel.read(o.frame.index).map_err(decode_err)?; + frames.push(DecodedFrame { pts: o.meta.pts, frame, draft: false }); + } + Ok(()) + } + + fn handle_nal(&mut self, nal: &[u8], pts: i64, frames: &mut Vec) -> Result<()> { + let Some(&first) = nal.first() else { return Ok(()) }; + let rest = nal.get(1..).unwrap_or_default(); + let hdr = NalHeader::parse(first).map_err(decode_err)?; + match hdr.nal_unit_type { + nal_type::SPS => { + let sps = Sps::parse(&unescape_rbsp(rest)).map_err(decode_err)?; + let slot = self.spss.get_mut(sps.id as usize).ok_or_else(|| decode_err("SPS id out of range"))?; + *slot = Some(Arc::new(sps)); + } + nal_type::PPS => { + let spss: Vec> = self.spss.iter().map(|s| s.as_ref().map(|s| (**s).clone())).collect(); + let pps = Pps::parse(&unescape_rbsp(rest), &spss).map_err(decode_err)?; + let slot = self.ppss.get_mut(pps.id as usize).ok_or_else(|| decode_err("PPS id out of range"))?; + *slot = Some(Arc::new(pps)); + } + nal_type::SLICE | nal_type::IDR => self.handle_slice(nal, hdr, pts, frames)?, + nal_type::SLICE_DPA | nal_type::SLICE_DPB | nal_type::SLICE_DPC => return Err(unsupported("data partitioning")), + nal_type::END_SEQ | nal_type::END_STREAM => self.submit_pending(frames)?, + // SEI, AUD, filler, extensions: nothing to decode + _ => {} + } + Ok(()) + } + + fn handle_slice(&mut self, nal: &[u8], hdr: NalHeader, pts: i64, frames: &mut Vec) -> Result<()> { + let rbsp = unescape_rbsp(nal.get(1..).unwrap_or_default()); + let (ppss, spss) = (&self.ppss, &self.spss); + let mut found: Option<(Arc, Arc)> = None; + let (sh, _, _) = SliceHeader::parse(&rbsp, hdr, |id| { + let missing = |what: String| filmcraft_h264::Error::MissingParameterSet(what); + let pps = ppss.get(id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("PPS {id}")))?; + let sps = spss.get(pps.sps_id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("SPS {}", pps.sps_id)))?; + found = Some((pps.clone(), sps.clone())); + Ok((&**pps, &**sps)) + }) + .map_err(decode_err)?; + let Some((pps, sps)) = found else { return Err(decode_err("slice without a PPS")) }; + sps.check_supported().map_err(decode_err)?; + if pps.num_slice_groups > 1 { + return Err(unsupported("slice groups (FMO)")); + } + if sh.field_pic { + return Err(unsupported("field pictures")); + } + if matches!(sh.slice_type, SliceType::Sp | SliceType::Si) { + return Err(unsupported("SP / SI slices")); + } + if sh.redundant_pic_cnt > 0 { + return Ok(()); // redundant slices are ignored, as in the software decoder + } + let new_pic = match &self.pending { + None => true, + Some(p) => is_new_picture(&p.first, &sh, &sps) || sh.first_mb_in_slice == 0 || !Arc::ptr_eq(&p.sps, &sps), + }; + if new_pic { + self.submit_pending(frames)?; + if sh.first_mb_in_slice != 0 && self.active_sps.is_none() { + return Err(decode_err("stream does not start with the first slice of a picture")); + } + self.start_picture(&sh, &sps, &pps, pts, frames)?; + } + let Some(pending) = self.pending.as_ref() else { return Err(decode_err("slice without a started picture")) }; + if !Arc::ptr_eq(&pending.pps, &pps) { + return Err(unsupported("a picture whose slices use different PPSs")); + } + let surfaces = self.accel.surfaces(); + let lists = self + .dpb + .build_ref_lists(&sh, pending.poc.frame(), sps.max_frame_num(), |e| (va_ref(e, surfaces), e.mark != RefMark::Unused)) + .map_err(decode_err)?; + if !sh.slice_type.is_intra() && (lists[0].is_empty() || (sh.slice_type.is_b() && lists[1].is_empty())) { + return Err(decode_err("no reference pictures available for an inter slice")); + } + // a missing reference (the software decoder conceals it with another picture) is not + // something to hand the hardware + if lists.iter().flatten().any(|(_, is_ref)| !is_ref) { + return Err(decode_err("a reference picture is missing")); + } + let refs = lists.map(|l| l.into_iter().map(|(p, _)| p).collect::>()); + let params = slice_params(&sh, nal.len(), &refs)?; + if let Some(p) = self.pending.as_mut() { + p.slices.push((params, nal.to_vec())); + } + Ok(()) + } + + fn start_picture(&mut self, sh: &SliceHeader, sps: &Arc, pps: &Arc, pts: i64, frames: &mut Vec) -> Result<()> { + if (sps.pic_width_in_mbs, sps.frame_height_in_mbs()) != self.mbs { + return Err(unsupported("a picture size change")); + } + if sps.max_dpb_frames() >= self.accel.surfaces().len() { + return Err(unsupported("a DPB larger than the decoder's surfaces")); + } + // a new SPS with another DPB size outputs and forgets what came before (as in software) + let changed = self.active_sps.as_ref().is_some_and(|a| a.max_dpb_frames() != sps.max_dpb_frames()); + let mut outs = Vec::new(); + if changed { + self.dpb.flush(&mut outs); + self.dpb.entries.clear(); + } + self.active_sps = Some(sps.clone()); + self.dpb.capacity = sps.max_dpb_frames(); + self.dpb.max_reorder = sps.max_num_reorder_frames().min(self.dpb.capacity); + if sh.idr { + self.dpb.idr(sh.no_output_of_prior_pics, &mut outs); + self.prev_ref_frame_num = 0; + } else { + let max = sps.max_frame_num(); + if sh.frame_num != self.prev_ref_frame_num && sh.frame_num != (self.prev_ref_frame_num + 1) % max { + self.fill_frame_num_gap(sh, sps, pts)?; + } + } + self.read_out(outs, frames)?; + let poc = self.poc_state.compute(sh, sps); + let target = + (0..self.accel.surfaces().len()).find(|&i| !self.dpb.entries.iter().any(|e| e.frame.index == i)).ok_or_else(|| decode_err("no free surface"))?; + self.pending = Some(Pending { target, sps: sps.clone(), pps: pps.clone(), first: sh.clone(), poc, pts, slices: Vec::new() }); + Ok(()) + } + + /// frame_num gap (8.2.5.2; also every start at a non-IDR picture, an open-GOP seek): "non-existing" + /// frames as the software decoder makes them, copies of the latest decoded frame (mid-gray when + /// there is none), so pictures predicted from them come out the same. + fn fill_frame_num_gap(&mut self, sh: &SliceHeader, sps: &Arc, pts: i64) -> Result<()> { + let max = sps.max_frame_num(); + let last = self.dpb.entries.iter().filter(|e| !e.non_existing).max_by_key(|e| e.frame.seq).map(|e| e.frame.index); + let meta = Arc::new(output_meta(sps, pts, false)); + let poc_state = &mut self.poc_state; + // surfaces are given afterwards, to the frames the sliding window keeps + let mut make = |_| (Surface { index: UNASSIGNED, top: 0, bottom: 0, seq: 0 }, 0, meta.clone()); + let mut update = |frame_num| poc_state.update_gap_frame(frame_num, sps); + self.dpb.fill_frame_num_gap(self.prev_ref_frame_num, sh.frame_num, max, sps.max_num_ref_frames as usize, &mut make, &mut update); + self.prev_ref_frame_num = (sh.frame_num + max - 1) % max; + let used: Vec = self.dpb.entries.iter().map(|e| e.frame.index).filter(|&i| i != UNASSIGNED).collect(); + let mut free = (0..self.accel.surfaces().len()).filter(|i| !used.contains(i)); + let mut fills = Vec::new(); + for e in self.dpb.entries.iter_mut().filter(|e| e.frame.index == UNASSIGNED) { + let index = free.next().ok_or_else(|| decode_err("no free surface for a frame_num gap"))?; + self.seq += 1; + e.frame = Surface { index, top: 0, bottom: 0, seq: self.seq }; + fills.push(index); + } + for index in fills { + self.accel.fill(index, last).map_err(decode_err)?; + } + Ok(()) + } + + /// Decode the collected picture, then do its reference marking and DPB insertion. + fn submit_pending(&mut self, frames: &mut Vec) -> Result<()> { + let Some(p) = self.pending.take() else { return Ok(()) }; + if p.slices.is_empty() { + return Err(decode_err("a picture without slices")); + } + let surfaces = self.accel.surfaces(); + let target_id = surfaces.get(p.target).copied().ok_or_else(|| decode_err("surface out of range"))?; + if self.dpb.entries.iter().filter(|e| e.mark != RefMark::Unused).count() > 16 { + return Err(decode_err("more than 16 reference frames")); + } + let mut pic = picture_params(&p.sps, &p.pps, &p.first); + pic.CurrPic = VAPictureH264 { + picture_id: target_id, + frame_idx: p.first.frame_num, + flags: 0, + TopFieldOrderCnt: p.poc.top, + BottomFieldOrderCnt: p.poc.bottom, + va_reserved: [0; ffi::VA_PADDING_LOW], + }; + for (slot, e) in pic.ReferenceFrames.iter_mut().zip(self.dpb.entries.iter().filter(|e| e.mark != RefMark::Unused)) { + *slot = va_ref(e, surfaces); + } + let iq = iq_matrix(&p.pps); + self.accel.decode_h264(p.target, &pic, &iq, &p.slices).map_err(decode_err)?; + + self.poc_state.update(&p.first, &p.poc); + let mmco5 = p.first.has_mmco5(); + if p.first.nal_ref_idc != 0 { + self.prev_ref_frame_num = if mmco5 { 0 } else { p.first.frame_num }; + } + // MMCO 5: the picture's order counts become relative to itself (tempPicOrderCnt, 8.2.1) + let temp = if mmco5 { p.poc.frame() } else { 0 }; + self.seq += 1; + let surface = Surface { index: p.target, top: p.poc.top.wrapping_sub(temp), bottom: p.poc.bottom.wrapping_sub(temp), seq: self.seq }; + let fpoc = if mmco5 { 0 } else { p.poc.frame() }; + let meta = Arc::new(output_meta(&p.sps, p.pts, p.first.idr)); + let mut outs = Vec::new(); + self.dpb.store_picture(&p.first, surface, fpoc, p.sps.max_frame_num(), p.sps.max_num_ref_frames as usize, meta, &mut outs); + self.read_out(outs, frames) + } +} + +/// A DPB entry as a VA-API reference picture. +fn va_ref(e: &DpbEntry, surfaces: &[ffi::VASurfaceID]) -> VAPictureH264 { + let long = e.mark == RefMark::Long; + VAPictureH264 { + picture_id: surfaces.get(e.frame.index).copied().unwrap_or(ffi::VA_INVALID_SURFACE), + frame_idx: if long { e.long_term_frame_idx } else { e.frame_num }, + flags: if long { ffi::VA_PICTURE_H264_LONG_TERM_REFERENCE } else { ffi::VA_PICTURE_H264_SHORT_TERM_REFERENCE }, + TopFieldOrderCnt: e.frame.top, + BottomFieldOrderCnt: e.frame.bottom, + va_reserved: [0; ffi::VA_PADDING_LOW], + } +} + +fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta { + let vui = sps.vui.clone().unwrap_or_default(); + OutputMeta { + pts, + key, + crop: sps.crop_rect(), + full_range: vui.full_range, + colour_primaries: vui.colour_primaries, + transfer_characteristics: vui.transfer_characteristics, + matrix_coefficients: vui.matrix_coefficients, + sar: vui.sar, + draft: false, + } +} + +fn flag(on: bool, bit: u32) -> u32 { + u32::from(on) << bit +} + +/// The picture parameters of a picture coded with `sps` / `pps` whose first slice is `sh` (the +/// current and reference pictures are filled in by the caller). +pub fn picture_params(sps: &Sps, pps: &Pps, sh: &SliceHeader) -> VAPictureParameterBufferH264 { + let seq_fields = (sps.chroma_format_idc & 3) << seq_bits::CHROMA_FORMAT_IDC + | flag(sps.separate_colour_plane, seq_bits::RESIDUAL_COLOUR_TRANSFORM) + | flag(sps.gaps_in_frame_num_allowed, seq_bits::GAPS_IN_FRAME_NUM_ALLOWED) + | flag(sps.frame_mbs_only, seq_bits::FRAME_MBS_ONLY) + | flag(sps.mb_adaptive_frame_field, seq_bits::MB_ADAPTIVE_FRAME_FIELD) + | flag(sps.direct_8x8_inference, seq_bits::DIRECT_8X8_INFERENCE) + // A.3.3.2: bi-prediction of blocks under 8x8 is not allowed from level 3.1 + | flag(sps.level_idc >= 31, seq_bits::MIN_LUMA_BI_PRED_SIZE_8X8) + | (sps.log2_max_frame_num.saturating_sub(4) & 15) << seq_bits::LOG2_MAX_FRAME_NUM_MINUS4 + | (sps.pic_order_cnt_type & 3) << seq_bits::PIC_ORDER_CNT_TYPE + | (sps.log2_max_poc_lsb.saturating_sub(4) & 15) << seq_bits::LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4 + | flag(sps.delta_pic_order_always_zero, seq_bits::DELTA_PIC_ORDER_ALWAYS_ZERO); + let pic_fields = flag(pps.entropy_coding_mode, pic_bits::ENTROPY_CODING_MODE) + | flag(pps.weighted_pred, pic_bits::WEIGHTED_PRED) + | (pps.weighted_bipred_idc & 3) << pic_bits::WEIGHTED_BIPRED_IDC + | flag(pps.transform_8x8_mode, pic_bits::TRANSFORM_8X8_MODE) + | flag(sh.field_pic, pic_bits::FIELD_PIC) + | flag(pps.constrained_intra_pred, pic_bits::CONSTRAINED_INTRA_PRED) + | flag(pps.bottom_field_pic_order_in_frame_present, pic_bits::PIC_ORDER_PRESENT) + | flag(pps.deblocking_filter_control_present, pic_bits::DEBLOCKING_FILTER_CONTROL_PRESENT) + | flag(pps.redundant_pic_cnt_present, pic_bits::REDUNDANT_PIC_CNT_PRESENT) + | flag(sh.nal_ref_idc != 0, pic_bits::REFERENCE_PIC); + // the parsers bound these (frame size by the level limits, qp offsets to ±12, qp to 0..51) + let narrow = |v: u32| u16::try_from(v.saturating_sub(1)).unwrap_or(u16::MAX); + let small = |v: i32| i8::try_from(v).unwrap_or(0); + VAPictureParameterBufferH264 { + CurrPic: VAPictureH264::INVALID, + ReferenceFrames: [VAPictureH264::INVALID; 16], + picture_width_in_mbs_minus1: narrow(sps.pic_width_in_mbs), + picture_height_in_mbs_minus1: narrow(sps.frame_height_in_mbs()), + bit_depth_luma_minus8: u8::try_from(sps.bit_depth_luma.saturating_sub(8)).unwrap_or(0), + bit_depth_chroma_minus8: u8::try_from(sps.bit_depth_chroma.saturating_sub(8)).unwrap_or(0), + num_ref_frames: u8::try_from(sps.max_num_ref_frames).unwrap_or(16), + seq_fields, + num_slice_groups_minus1: 0, + slice_group_map_type: 0, + slice_group_change_rate_minus1: 0, + pic_init_qp_minus26: small(pps.pic_init_qp - 26), + pic_init_qs_minus26: small(pps.pic_init_qs - 26), + chroma_qp_index_offset: small(pps.chroma_qp_index_offset), + second_chroma_qp_index_offset: small(pps.second_chroma_qp_index_offset), + pic_fields, + frame_num: u16::try_from(sh.frame_num).unwrap_or(0), + va_reserved: [0; ffi::VA_PADDING_MEDIUM], + } +} + +/// The scaling matrices of `pps` (after the SPS / PPS fall-back rules), raster order. +pub fn iq_matrix(pps: &Pps) -> VAIQMatrixBufferH264 { + VAIQMatrixBufferH264 { ScalingList4x4: pps.scaling.m4, ScalingList8x8: [pps.scaling.m8[0], pps.scaling.m8[1]], va_reserved: [0; ffi::VA_PADDING_LOW] } +} + +/// The slice parameters of the slice `sh` (a `nal_len`-byte NAL unit) with reference lists `refs`. +pub fn slice_params(sh: &SliceHeader, nal_len: usize, refs: &[Vec; 2]) -> Result { + let bit_offset = sh.header_bits.checked_add(8).and_then(|b| u16::try_from(b).ok()).ok_or_else(|| decode_err("slice header too long"))?; + let mut s = VASliceParameterBufferH264 { + slice_data_size: u32::try_from(nal_len).map_err(|_| decode_err("slice too large"))?, + slice_data_offset: 0, + slice_data_flag: ffi::VA_SLICE_DATA_FLAG_ALL, + slice_data_bit_offset: bit_offset, + first_mb_in_slice: u16::try_from(sh.first_mb_in_slice).map_err(|_| decode_err("first_mb_in_slice out of range"))?, + slice_type: (sh.slice_type_raw % 5) as u8, + direct_spatial_mv_pred_flag: u8::from(sh.direct_spatial_mv_pred), + num_ref_idx_l0_active_minus1: u8::try_from(sh.num_ref_idx_active[0].saturating_sub(1)).unwrap_or(31).min(31), + num_ref_idx_l1_active_minus1: u8::try_from(sh.num_ref_idx_active[1].saturating_sub(1)).unwrap_or(31).min(31), + cabac_init_idc: u8::try_from(sh.cabac_init_idc).unwrap_or(0), + slice_qp_delta: i8::try_from(sh.slice_qp_delta).map_err(|_| decode_err("slice_qp_delta out of range"))?, + disable_deblocking_filter_idc: u8::try_from(sh.disable_deblocking_filter_idc).unwrap_or(0), + slice_alpha_c0_offset_div2: i8::try_from(sh.slice_alpha_c0_offset_div2).unwrap_or(0), + slice_beta_offset_div2: i8::try_from(sh.slice_beta_offset_div2).unwrap_or(0), + RefPicList0: [VAPictureH264::INVALID; 32], + RefPicList1: [VAPictureH264::INVALID; 32], + luma_log2_weight_denom: 0, + chroma_log2_weight_denom: 0, + luma_weight_l0_flag: 0, + luma_weight_l0: [0; 32], + luma_offset_l0: [0; 32], + chroma_weight_l0_flag: 0, + chroma_weight_l0: [[0; 2]; 32], + chroma_offset_l0: [[0; 2]; 32], + luma_weight_l1_flag: 0, + luma_weight_l1: [0; 32], + luma_offset_l1: [0; 32], + chroma_weight_l1_flag: 0, + chroma_weight_l1: [[0; 2]; 32], + chroma_offset_l1: [[0; 2]; 32], + va_reserved: [0; ffi::VA_PADDING_LOW], + }; + for (dst, src) in s.RefPicList0.iter_mut().zip(&refs[0]) { + *dst = *src; + } + for (dst, src) in s.RefPicList1.iter_mut().zip(&refs[1]) { + *dst = *src; + } + // explicit weighted prediction: every entry, the default weight where none was coded + if let Some(w) = &sh.pred_weight_table { + s.luma_log2_weight_denom = u8::try_from(w.luma_log2_denom).unwrap_or(0); + s.chroma_log2_weight_denom = u8::try_from(w.chroma_log2_denom).unwrap_or(0); + let w16 = |v: i32| i16::try_from(v).unwrap_or(0); + let lists = [ + ( + &w.l0, + &mut s.luma_weight_l0, + &mut s.luma_offset_l0, + &mut s.chroma_weight_l0, + &mut s.chroma_offset_l0, + &mut s.luma_weight_l0_flag, + &mut s.chroma_weight_l0_flag, + ), + ( + &w.l1, + &mut s.luma_weight_l1, + &mut s.luma_offset_l1, + &mut s.chroma_weight_l1, + &mut s.chroma_offset_l1, + &mut s.luma_weight_l1_flag, + &mut s.chroma_weight_l1_flag, + ), + ]; + for (entries, lw, lo, cw, co, lflag, cflag) in lists { + if entries.is_empty() { + continue; + } + *lflag = 1; + *cflag = 1; + for (i, e) in entries.iter().take(32).enumerate() { + lw[i] = w16(e.luma_weight); + lo[i] = w16(e.luma_offset); + for c in 0..2 { + cw[i][c] = w16(e.chroma_weight[c]); + co[i][c] = w16(e.chroma_offset[c]); + } + } + } + } + Ok(s) +} diff --git a/crates/platform/src/vaapi/hevc.rs b/crates/platform/src/vaapi/hevc.rs new file mode 100644 index 00000000..730a139f --- /dev/null +++ b/crates/platform/src/vaapi/hevc.rs @@ -0,0 +1,612 @@ +//! The HEVC side of the VA-API decoder, like [`super::h264`]: the software decoder's own parsers +//! (`filmcraft_hevc::params`, `slice`) and decoded picture buffer (`filmcraft_hevc::dpb`: POC, +//! reference picture sets, reference lists, output order) drive a stateless hardware decoder, so +//! both decoders make the same decisions and output the same pictures in the same order, RASL +//! pictures of a CRA the run starts at left out as in software. +//! +//! What the hardware path does not take is an error, so the hybrid decoder replays the run in +//! software: missing reference pictures (the software decoder conceals them), more than 15 +//! reference frames, stream changes the surfaces were not made for, range extensions. +//! +//! Safe code; compiled on every target so its logic is tested everywhere. + +use std::sync::Arc; + +use filmcraft_bitstream::{length_prefixed_nals, unescape_rbsp}; +use filmcraft_codecs::{CodecError, DecodedFrame, Result}; +use filmcraft_hevc::dpb::{Dpb, Marking, Output, OutputMeta, PocState, Ref, RefPicSet, build_ref_lists}; +use filmcraft_hevc::params::{Layout, Pps, ScalingList, Sps}; +use filmcraft_hevc::slice::{NalHeader, SliceHeader, SliceType, nal_type}; + +use super::Accel; +use super::ffi::{ + self, VAIQMatrixBufferHEVC, VAPictureHEVC, VAPictureParameterBufferHEVC, VASliceParameterBufferHEVC, hevc_pic_bits, hevc_slice_bits, + hevc_slice_parsing_bits, +}; + +/// A picture in the DPB: its surface. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Surface { + pub index: usize, +} + +/// A reference picture that was never decoded (8.3.3): not handed to the hardware. +const MISSING: usize = usize::MAX; + +/// The picture whose slice segments are being collected. +struct Pending { + target: usize, + sps: Arc, + pps: Arc, + first: SliceHeader, + /// The latest independent slice segment's header (dependent ones continue it). + last: SliceHeader, + rps: RefPicSet, + /// The surfaces of `pic.ReferenceFrames`, in order. + ref_frames: Vec, + pic: VAPictureParameterBufferHEVC, + poc: i32, + output: bool, + meta: Arc, + slices: Vec<(VASliceParameterBufferHEVC, Vec)>, + all_intra: bool, + any_b: bool, +} + +/// HEVC front end of a stateless hardware decoder (see the module docs). +pub struct Front { + accel: A, + length_size: usize, + /// Coded size (luma samples) and bit depth the surfaces were made for. + size: (u32, u32), + bits: u32, + spss: Vec>>, + ppss: Vec>>, + dpb: Dpb, + poc_state: PocState, + active_sps: Option>, + pending: Option, + first_picture: bool, + skip_rasl: bool, + after_eos: bool, +} + +fn decode_err(e: impl std::fmt::Display) -> CodecError { + CodecError::Decode(e.to_string()) +} + +fn unsupported(what: &str) -> CodecError { + CodecError::Decode(format!("the hardware decoder does not take {what}")) +} + +impl Front { + /// A front end for samples with `length_size`-byte NAL length prefixes whose sample entry + /// carried `parameter_sets` (VPS / SPS / PPS NAL units), decoding into `accel`'s surfaces of + /// `size` luma samples and `bits` bits. + pub fn new(accel: A, length_size: usize, size: (u32, u32), bits: u32, parameter_sets: &[Vec]) -> Result { + let mut f = Self { + accel, + length_size, + size, + bits, + spss: vec![None; 16], + ppss: vec![None; 64], + dpb: Dpb::default(), + poc_state: PocState::default(), + active_sps: None, + pending: None, + first_picture: true, + skip_rasl: false, + after_eos: false, + }; + let mut frames = Vec::new(); + for nal in parameter_sets { + f.handle_nal(nal, 0, &mut frames)?; + } + Ok(f) + } + + pub fn accel(&self) -> &A { + &self.accel + } + + /// Decode one access unit; pictures that left the DPB come back in output order. + pub fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + let nals = length_prefixed_nals(sample, self.length_size).map_err(decode_err)?; + let mut frames = Vec::new(); + for nal in nals { + self.handle_nal(nal, pts, &mut frames)?; + } + self.submit_pending(&mut frames)?; + Ok(frames) + } + + /// Output every picture still held for reordering; what follows starts like a new stream + /// (a CRA's RASL pictures are left out), as in software. + pub fn flush(&mut self) -> Result> { + let mut frames = Vec::new(); + self.submit_pending(&mut frames)?; + let mut outs = Vec::new(); + self.dpb.flush(&mut outs); + self.read_out(outs, &mut frames)?; + self.first_picture = true; + Ok(frames) + } + + /// Forget every picture (after a seek). + pub fn reset(&mut self) { + self.dpb = Dpb::default(); + self.poc_state = PocState::default(); + self.active_sps = None; + self.pending = None; + self.first_picture = true; + self.skip_rasl = false; + self.after_eos = false; + } + + /// Read back the pictures the DPB output, right away: a surface leaving the DPB is reused. + fn read_out(&mut self, outs: Vec>, frames: &mut Vec) -> Result<()> { + for o in outs { + let frame = self.accel.read(o.frame.index).map_err(decode_err)?; + frames.push(DecodedFrame { pts: o.meta.pts, frame, draft: false }); + } + Ok(()) + } + + fn handle_nal(&mut self, nal: &[u8], pts: i64, frames: &mut Vec) -> Result<()> { + if nal.len() < 2 { + return Ok(()); + } + let hdr = NalHeader::parse(nal).map_err(decode_err)?; + if hdr.layer_id != 0 { + return Ok(()); + } + let rbsp = || unescape_rbsp(nal.get(2..).unwrap_or_default()); + match hdr.nal_type { + nal_type::SPS => { + let sps = Sps::parse(&rbsp()).map_err(decode_err)?; + let slot = self.spss.get_mut(sps.id as usize).ok_or_else(|| decode_err("SPS id out of range"))?; + *slot = Some(Arc::new(sps)); + } + nal_type::PPS => { + let pps = Pps::parse(&rbsp()).map_err(decode_err)?; + let slot = self.ppss.get_mut(pps.id as usize).ok_or_else(|| decode_err("PPS id out of range"))?; + *slot = Some(Arc::new(pps)); + } + nal_type::EOS | nal_type::EOB => { + self.submit_pending(frames)?; + self.after_eos = true; + } + 0..=9 | 16..=21 => self.handle_slice(nal, hdr, pts, frames)?, + // VPS, AUD, SEI, filler, reserved: nothing to decode + _ => {} + } + Ok(()) + } + + fn handle_slice(&mut self, nal: &[u8], hdr: NalHeader, pts: i64, frames: &mut Vec) -> Result<()> { + let rbsp = unescape_rbsp(nal.get(2..).unwrap_or_default()); + let first = rbsp.first().is_some_and(|b| b & 0x80 != 0); + if first { + self.submit_pending(frames)?; + } + if hdr.is_rasl() && (self.skip_rasl || self.first_picture) { + return Ok(()); + } + let (ppss, spss) = (&self.ppss, &self.spss); + let prev = self.pending.as_ref().map(|p| &p.last); + let (sh, pps, sps) = SliceHeader::parse( + &rbsp, + hdr, + |id| { + let missing = |what: String| filmcraft_hevc::Error::MissingParameterSet(what); + let pps = ppss.get(id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("PPS {id}")))?; + let sps = spss.get(pps.sps_id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("SPS {}", pps.sps_id)))?; + Ok((pps.clone(), sps.clone())) + }, + prev, + ) + .map_err(decode_err)?; + sps.check_supported().map_err(decode_err)?; + pps.check_supported().map_err(decode_err)?; + if first { + self.start_picture(&sh, &sps, &pps, pts, frames)?; + } + let Some(p) = self.pending.as_mut() else { return Err(decode_err("slice segment without the first slice of its picture")) }; + if !Arc::ptr_eq(&p.pps, &pps) { + return Err(decode_err("PPS changed within a picture")); + } + let refs = build_ref_lists(&sh, &p.rps).map_err(decode_err)?; + let params = slice_params(&sh, nal, &refs, &p.ref_frames)?; + if !sh.dependent { + p.last = sh.clone(); + } + p.all_intra &= sh.is_intra(); + p.any_b |= sh.is_b(); + p.slices.push((params, nal.to_vec())); + Ok(()) + } + + fn start_picture(&mut self, sh: &SliceHeader, sps: &Arc, pps: &Arc, pts: i64, frames: &mut Vec) -> Result<()> { + if !sh.first_slice_segment_in_pic || sh.dependent { + return Err(decode_err("invalid first slice segment")); + } + if (sps.width, sps.height) != self.size || sps.bit_depth_luma != self.bits || sps.bit_depth_chroma != self.bits || sps.chroma_format_idc != 1 { + return Err(unsupported("a picture size or format change")); + } + if sps.max_dec_pic_buffering as usize >= self.accel.surfaces().len() { + return Err(unsupported("a DPB larger than the decoder's surfaces")); + } + let hdr = sh.nal; + let changed = self.active_sps.as_ref().is_some_and(|a| a.log2_ctb != sps.log2_ctb); + let mut outs = Vec::new(); + if changed { + self.dpb.flush(&mut outs); + } + self.active_sps = Some(sps.clone()); + self.dpb.max_dec_pic_buffering = sps.max_dec_pic_buffering as usize; + self.dpb.max_num_reorder = sps.max_num_reorder as usize; + self.dpb.max_latency = if sps.max_latency_increase_plus1 != 0 { sps.max_num_reorder + sps.max_latency_increase_plus1 - 1 } else { 0 }; + let irap_no_rasl = hdr.is_irap() && (hdr.is_idr() || hdr.is_bla() || self.first_picture || self.after_eos); + if hdr.is_irap() { + self.skip_rasl = irap_no_rasl; + } + let max_lsb = sps.max_poc_lsb(); + let poc = self.poc_state.compute(sh, max_lsb, irap_no_rasl); + let mut make_missing = |_| Surface { index: MISSING }; + let rps = self.dpb.apply_rps(sh, poc, max_lsb, irap_no_rasl, &mut make_missing).map_err(decode_err)?; + if [&rps.st_curr_before, &rps.st_curr_after, &rps.lt_curr].iter().any(|l| l.iter().any(|r| r.frame.index == MISSING)) { + // the software decoder conceals them with mid-gray pictures + return Err(unsupported("missing reference pictures")); + } + if irap_no_rasl && !self.first_picture { + if sh.no_output_of_prior_pics && hdr.is_idr() { + self.dpb.clear(); + } else { + self.dpb.flush(&mut outs); + } + } else { + self.dpb.bump_before_decode(&mut outs); + } + self.read_out(outs, frames)?; + let surfaces = self.accel.surfaces(); + let target = (0..surfaces.len()).find(|&i| !self.dpb.entries.iter().any(|e| e.frame.index == i)).ok_or_else(|| decode_err("no free surface"))?; + let target_id = surfaces.get(target).copied().ok_or_else(|| decode_err("surface out of range"))?; + let mut pic = picture_params(sps, pps, sh)?; + pic.CurrPic = VAPictureHEVC { picture_id: target_id, pic_order_cnt: poc, flags: 0, va_reserved: [0; ffi::VA_PADDING_LOW] }; + // the reference frames: every picture the RPS kept, flagged with its part of the RPS + let rps_flag = |index: usize| { + let in_list = |l: &[Ref]| l.iter().any(|r| r.frame.index == index); + if in_list(&rps.st_curr_before) { + ffi::VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE + } else if in_list(&rps.st_curr_after) { + ffi::VA_PICTURE_HEVC_RPS_ST_CURR_AFTER + } else if in_list(&rps.lt_curr) { + ffi::VA_PICTURE_HEVC_RPS_LT_CURR + } else { + 0 + } + }; + let refs: Vec<_> = self.dpb.entries.iter().filter(|e| e.marking != Marking::Unused).collect(); + if refs.len() > pic.ReferenceFrames.len() { + return Err(unsupported("more than 15 reference frames")); + } + let mut ref_frames = Vec::with_capacity(refs.len()); + for (slot, e) in pic.ReferenceFrames.iter_mut().zip(&refs) { + let long = if e.marking == Marking::Long { ffi::VA_PICTURE_HEVC_LONG_TERM_REFERENCE } else { 0 }; + let picture_id = surfaces.get(e.frame.index).copied().ok_or_else(|| decode_err("surface out of range"))?; + *slot = VAPictureHEVC { picture_id, pic_order_cnt: e.poc, flags: long | rps_flag(e.frame.index), va_reserved: [0; ffi::VA_PADDING_LOW] }; + ref_frames.push(e.frame.index); + } + let output = sh.pic_output && !(hdr.is_rasl() && self.skip_rasl); + let meta = Arc::new(output_meta(sps, pts, hdr.is_irap())); + self.pending = Some(Pending { + target, + sps: sps.clone(), + pps: pps.clone(), + first: sh.clone(), + last: sh.clone(), + rps, + ref_frames, + pic, + poc, + output, + meta, + slices: Vec::new(), + all_intra: true, + any_b: false, + }); + self.first_picture = false; + self.after_eos = false; + Ok(()) + } + + /// Decode the collected picture, then insert it into the DPB. + fn submit_pending(&mut self, frames: &mut Vec) -> Result<()> { + let Some(mut p) = self.pending.take() else { return Ok(()) }; + let Some(last) = p.slices.last_mut() else { return Err(decode_err("a picture without slices")) }; + last.0.LongSliceFlags |= 1 << hevc_slice_bits::LAST_SLICE_OF_PIC; + p.pic.slice_parsing_fields |= u32::from(p.all_intra) << hevc_slice_parsing_bits::INTRA_PIC; + p.pic.pic_fields |= u32::from(!p.any_b) << hevc_pic_bits::NO_BI_PRED; + let iq = if p.sps.scaling_list_enabled { Some(iq_matrix(p.pps.scaling_list.as_ref().or(p.sps.scaling_list.as_ref()))) } else { None }; + self.accel.decode_hevc(p.target, &p.pic, iq.as_ref(), &p.slices).map_err(decode_err)?; + self.poc_state.update(&p.first, p.poc); + let mut outs = Vec::new(); + self.dpb.insert(Surface { index: p.target }, p.poc, p.output, p.meta, &mut outs); + self.read_out(outs, frames) + } +} + +fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta { + let vui = sps.vui.clone().unwrap_or_default(); + OutputMeta { + pts, + key, + crop: sps.crop_rect(), + full_range: vui.full_range, + colour_primaries: vui.colour_primaries, + transfer_characteristics: vui.transfer_characteristics, + matrix_coefficients: vui.matrix_coefficients, + sar: vui.sar, + bit_depth: sps.bit_depth_luma, + draft: false, + } +} + +fn flag(on: bool, bit: u32) -> u32 { + u32::from(on) << bit +} + +fn u8_of(v: u32) -> u8 { + u8::try_from(v).unwrap_or(u8::MAX) +} + +fn i8_of(v: i32) -> i8 { + i8::try_from(v).unwrap_or(0) +} + +/// The picture parameters of a picture coded with `sps` / `pps` whose first slice segment is `sh` +/// (the current and reference pictures, `IntraPicFlag` and `NoBiPredFlag` are filled in by the caller). +pub fn picture_params(sps: &Sps, pps: &Pps, sh: &SliceHeader) -> Result { + use hevc_pic_bits as p; + use hevc_slice_parsing_bits as s; + let pic_fields = (sps.chroma_format_idc & 3) << p::CHROMA_FORMAT_IDC + | flag(sps.separate_colour_plane, p::SEPARATE_COLOUR_PLANE) + | flag(sps.pcm, p::PCM_ENABLED) + | flag(sps.scaling_list_enabled, p::SCALING_LIST_ENABLED) + | flag(pps.transform_skip, p::TRANSFORM_SKIP_ENABLED) + | flag(sps.amp, p::AMP_ENABLED) + | flag(sps.strong_intra_smoothing, p::STRONG_INTRA_SMOOTHING_ENABLED) + | flag(pps.sign_data_hiding, p::SIGN_DATA_HIDING_ENABLED) + | flag(pps.constrained_intra_pred, p::CONSTRAINED_INTRA_PRED) + | flag(pps.cu_qp_delta_enabled, p::CU_QP_DELTA_ENABLED) + | flag(pps.weighted_pred, p::WEIGHTED_PRED) + | flag(pps.weighted_bipred, p::WEIGHTED_BIPRED) + | flag(pps.transquant_bypass, p::TRANSQUANT_BYPASS_ENABLED) + | flag(pps.tiles_enabled, p::TILES_ENABLED) + | flag(pps.entropy_coding_sync, p::ENTROPY_CODING_SYNC_ENABLED) + | flag(pps.loop_filter_across_slices, p::PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED) + | flag(pps.loop_filter_across_tiles, p::LOOP_FILTER_ACROSS_TILES_ENABLED) + | flag(sps.pcm_loop_filter_disabled, p::PCM_LOOP_FILTER_DISABLED) + | flag(sps.max_num_reorder == 0, p::NO_PIC_REORDERING); + let slice_parsing_fields = flag(pps.lists_modification_present, s::LISTS_MODIFICATION_PRESENT) + | flag(sps.long_term_refs_present, s::LONG_TERM_REF_PICS_PRESENT) + | flag(sps.temporal_mvp, s::SPS_TEMPORAL_MVP_ENABLED) + | flag(pps.cabac_init_present, s::CABAC_INIT_PRESENT) + | flag(pps.output_flag_present, s::OUTPUT_FLAG_PRESENT) + | flag(pps.dependent_slice_segments_enabled, s::DEPENDENT_SLICE_SEGMENTS_ENABLED) + | flag(pps.slice_chroma_qp_offsets_present, s::PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT) + | flag(sps.sao, s::SAMPLE_ADAPTIVE_OFFSET_ENABLED) + | flag(pps.deblocking_override_enabled, s::DEBLOCKING_FILTER_OVERRIDE_ENABLED) + | flag(pps.deblocking_disabled, s::PPS_DISABLE_DEBLOCKING_FILTER) + | flag(pps.slice_header_extension_present, s::SLICE_SEGMENT_HEADER_EXTENSION_PRESENT) + | flag(sh.nal.is_irap(), s::RAP_PIC) + | flag(sh.nal.is_idr(), s::IDR_PIC); + // tile sizes in CTBs, uniform spacing resolved + let layout = Layout::new(sps, pps).map_err(decode_err)?; + let mut column_width_minus1 = [0u16; 19]; + let mut row_height_minus1 = [0u16; 21]; + if pps.tiles_enabled { + let sizes = |bd: &[u32], out: &mut [u16]| -> Result<()> { + let n = bd.len().saturating_sub(1); + if n > out.len() { + return Err(unsupported("that many tiles")); + } + for (o, w) in out.iter_mut().zip(bd.windows(2)) { + *o = u16::try_from(w[1].saturating_sub(w[0]).saturating_sub(1)).unwrap_or(0); + } + Ok(()) + }; + sizes(&layout.col_bd, &mut column_width_minus1)?; + sizes(&layout.row_bd, &mut row_height_minus1)?; + } + let narrow = |v: u32| u16::try_from(v).map_err(|_| decode_err("picture too large")); + Ok(VAPictureParameterBufferHEVC { + CurrPic: VAPictureHEVC::INVALID, + ReferenceFrames: [VAPictureHEVC::INVALID; 15], + pic_width_in_luma_samples: narrow(sps.width)?, + pic_height_in_luma_samples: narrow(sps.height)?, + pic_fields, + sps_max_dec_pic_buffering_minus1: u8_of(sps.max_dec_pic_buffering.saturating_sub(1)), + bit_depth_luma_minus8: u8_of(sps.bit_depth_luma.saturating_sub(8)), + bit_depth_chroma_minus8: u8_of(sps.bit_depth_chroma.saturating_sub(8)), + pcm_sample_bit_depth_luma_minus1: u8_of(sps.pcm_bit_depth_luma.saturating_sub(1)), + pcm_sample_bit_depth_chroma_minus1: u8_of(sps.pcm_bit_depth_chroma.saturating_sub(1)), + log2_min_luma_coding_block_size_minus3: u8_of(sps.log2_min_cb.saturating_sub(3)), + log2_diff_max_min_luma_coding_block_size: u8_of(sps.log2_ctb.saturating_sub(sps.log2_min_cb)), + log2_min_transform_block_size_minus2: u8_of(sps.log2_min_tb.saturating_sub(2)), + log2_diff_max_min_transform_block_size: u8_of(sps.log2_max_tb.saturating_sub(sps.log2_min_tb)), + log2_min_pcm_luma_coding_block_size_minus3: u8_of(sps.log2_min_pcm.saturating_sub(3)), + log2_diff_max_min_pcm_luma_coding_block_size: u8_of(sps.log2_max_pcm.saturating_sub(sps.log2_min_pcm)), + max_transform_hierarchy_depth_intra: u8_of(sps.max_th_depth_intra), + max_transform_hierarchy_depth_inter: u8_of(sps.max_th_depth_inter), + init_qp_minus26: i8_of(pps.init_qp - 26), + diff_cu_qp_delta_depth: u8_of(pps.diff_cu_qp_delta_depth), + pps_cb_qp_offset: i8_of(pps.cb_qp_offset), + pps_cr_qp_offset: i8_of(pps.cr_qp_offset), + log2_parallel_merge_level_minus2: u8_of(pps.log2_parallel_merge_level.saturating_sub(2)), + num_tile_columns_minus1: u8_of(pps.num_tile_columns.saturating_sub(1)), + num_tile_rows_minus1: u8_of(pps.num_tile_rows.saturating_sub(1)), + column_width_minus1, + row_height_minus1, + slice_parsing_fields, + log2_max_pic_order_cnt_lsb_minus4: u8_of(sps.log2_max_poc_lsb.saturating_sub(4)), + num_short_term_ref_pic_sets: u8_of(u32::try_from(sps.st_rps.len()).unwrap_or(u32::MAX)), + num_long_term_ref_pic_sps: u8_of(u32::try_from(sps.lt_ref_pics.len()).unwrap_or(u32::MAX)), + num_ref_idx_l0_default_active_minus1: u8_of(pps.num_ref_idx_l0_default.saturating_sub(1)), + num_ref_idx_l1_default_active_minus1: u8_of(pps.num_ref_idx_l1_default.saturating_sub(1)), + pps_beta_offset_div2: i8_of(pps.beta_offset_div2), + pps_tc_offset_div2: i8_of(pps.tc_offset_div2), + num_extra_slice_header_bits: u8_of(pps.num_extra_slice_header_bits), + st_rps_bits: u32::try_from(sh.st_rps_bits).unwrap_or(0), + va_reserved: [0; ffi::VA_PADDING_MEDIUM], + }) +} + +/// The 8x8 base list `coded` (up-right diagonal order, as coded and stored) in raster order. +fn raster8(list: &ScalingList, size_id: usize, matrix_id: usize) -> [u8; 64] { + let mut l = list.clone(); + let row = |s: usize| l.lists.get(s).and_then(|m| m.get(matrix_id)).copied(); + if let Some(src) = row(size_id) + && let Some(dst) = l.lists.get_mut(1).and_then(|m| m.get_mut(matrix_id)) + { + *dst = src; + } + let mut out = [16u8; 64]; + for (o, v) in out.iter_mut().zip(l.factor(1, matrix_id)) { + *o = v; + } + out +} + +/// The scaling lists in force (`list`, or the default lists), in raster order. +pub fn iq_matrix(list: Option<&ScalingList>) -> VAIQMatrixBufferHEVC { + let defaults = ScalingList::default_lists(); + let l = list.unwrap_or(&defaults); + let mut iq = VAIQMatrixBufferHEVC { + ScalingList4x4: [[16; 16]; 6], + ScalingList8x8: [[16; 64]; 6], + ScalingList16x16: [[16; 64]; 6], + ScalingList32x32: [[16; 64]; 2], + ScalingListDC16x16: [16; 6], + ScalingListDC32x32: [16; 2], + va_reserved: [0; ffi::VA_PADDING_LOW], + }; + for m in 0..6 { + for (o, v) in iq.ScalingList4x4[m].iter_mut().zip(l.factor(0, m)) { + *o = v; + } + iq.ScalingList8x8[m] = raster8(l, 1, m); + iq.ScalingList16x16[m] = raster8(l, 2, m); + iq.ScalingListDC16x16[m] = l.dc[0][m]; + } + // 32x32: matrixId 0 (intra) and 3 (inter) + for (i, m) in [0, 3].into_iter().enumerate() { + iq.ScalingList32x32[i] = raster8(l, 3, m); + iq.ScalingListDC32x32[i] = l.dc[1][m]; + } + iq +} + +/// Emulation prevention bytes among the first `rbsp_len` RBSP bytes of a NAL unit payload. +fn emulation_bytes_before(payload: &[u8], rbsp_len: usize) -> usize { + let (mut zeros, mut out, mut ep) = (0usize, 0usize, 0usize); + for &b in payload { + if out >= rbsp_len { + break; + } + if zeros >= 2 && b == 3 { + ep += 1; + zeros = 0; + continue; + } + zeros = if b == 0 { zeros + 1 } else { 0 }; + out += 1; + } + ep +} + +/// The slice parameters of the slice segment `sh` (NAL unit `nal`) with reference lists `refs`, +/// whose pictures are `ref_frames` (the surfaces of `ReferenceFrames`, in order). +pub fn slice_params(sh: &SliceHeader, nal: &[u8], refs: &[Vec>; 2], ref_frames: &[usize]) -> Result { + use hevc_slice_bits as b; + let slice_type = match sh.slice_type { + SliceType::B => 0, + SliceType::P => 1, + SliceType::I => 2, + }; + let flags = flag(sh.dependent, b::DEPENDENT_SLICE_SEGMENT) + | slice_type << b::SLICE_TYPE + | flag(sh.sao_luma, b::SLICE_SAO_LUMA) + | flag(sh.sao_chroma, b::SLICE_SAO_CHROMA) + | flag(sh.mvd_l1_zero, b::MVD_L1_ZERO) + | flag(sh.cabac_init_flag, b::CABAC_INIT) + | flag(sh.temporal_mvp, b::SLICE_TEMPORAL_MVP_ENABLED) + | flag(sh.deblocking_disabled, b::SLICE_DEBLOCKING_FILTER_DISABLED) + | flag(sh.collocated_from_l0, b::COLLOCATED_FROM_L0) + | flag(sh.loop_filter_across_slices, b::SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED); + let header_bytes = sh.data_offset.checked_add(2).ok_or_else(|| decode_err("slice header too long"))?; + let ep = emulation_bytes_before(nal.get(2..).unwrap_or_default(), sh.data_offset); + let mut s = VASliceParameterBufferHEVC { + slice_data_size: u32::try_from(nal.len()).map_err(|_| decode_err("slice too large"))?, + slice_data_offset: 0, + slice_data_flag: ffi::VA_SLICE_DATA_FLAG_ALL, + slice_data_byte_offset: u32::try_from(header_bytes).map_err(|_| decode_err("slice header too long"))?, + slice_segment_address: sh.segment_address, + RefPicList: [[0xFF; 15]; 2], + LongSliceFlags: flags, + collocated_ref_idx: u8_of(sh.collocated_ref_idx), + num_ref_idx_l0_active_minus1: u8_of(sh.num_ref_idx[0].saturating_sub(1)), + num_ref_idx_l1_active_minus1: u8_of(sh.num_ref_idx[1].saturating_sub(1)), + slice_qp_delta: i8_of(sh.qp_delta), + slice_cb_qp_offset: i8_of(sh.cb_qp_offset), + slice_cr_qp_offset: i8_of(sh.cr_qp_offset), + slice_beta_offset_div2: i8_of(sh.beta_offset_div2), + slice_tc_offset_div2: i8_of(sh.tc_offset_div2), + luma_log2_weight_denom: 0, + delta_chroma_log2_weight_denom: 0, + delta_luma_weight_l0: [0; 15], + luma_offset_l0: [0; 15], + delta_chroma_weight_l0: [[0; 2]; 15], + ChromaOffsetL0: [[0; 2]; 15], + delta_luma_weight_l1: [0; 15], + luma_offset_l1: [0; 15], + delta_chroma_weight_l1: [[0; 2]; 15], + ChromaOffsetL1: [[0; 2]; 15], + five_minus_max_num_merge_cand: u8_of(5u32.saturating_sub(sh.max_num_merge_cand)), + num_entry_point_offsets: u16::try_from(sh.entry_points.len()).map_err(|_| decode_err("too many entry points"))?, + entry_offset_to_subset_array: 0, + slice_data_num_emu_prevn_bytes: u16::try_from(ep).unwrap_or(u16::MAX), + va_reserved: [0; ffi::VA_PADDING_LOW - 2], + }; + for (l, list) in refs.iter().enumerate() { + if list.len() > 15 { + return Err(decode_err("reference list longer than 15")); + } + for (i, r) in list.iter().enumerate() { + let idx = ref_frames.iter().position(|&f| f == r.frame.index).ok_or_else(|| decode_err("a reference is not among the reference frames"))?; + s.RefPicList[l][i] = u8::try_from(idx).unwrap_or(0xFF); + } + } + if let Some(w) = &sh.pwt { + let luma_denom = i32::try_from(w.luma_log2_denom).unwrap_or(0); + let chroma_denom = i32::try_from(w.chroma_log2_denom).unwrap_or(0); + s.luma_log2_weight_denom = u8_of(w.luma_log2_denom); + s.delta_chroma_log2_weight_denom = i8_of(chroma_denom - luma_denom); + let lists = [ + (&w.l[0], &mut s.delta_luma_weight_l0, &mut s.luma_offset_l0, &mut s.delta_chroma_weight_l0, &mut s.ChromaOffsetL0), + (&w.l[1], &mut s.delta_luma_weight_l1, &mut s.luma_offset_l1, &mut s.delta_chroma_weight_l1, &mut s.ChromaOffsetL1), + ]; + for (entries, lw, lo, cw, co) in lists { + for (i, e) in entries.iter().take(15).enumerate() { + lw[i] = i8_of(e.luma.0 - (1 << luma_denom)); + lo[i] = i8_of(e.luma.1); + for c in 0..2 { + cw[i][c] = i8_of(e.chroma[c].0 - (1 << chroma_denom)); + co[i][c] = i8_of(e.chroma[c].1); + } + } + } + } + Ok(s) +} diff --git a/crates/platform/src/vaapi/mod.rs b/crates/platform/src/vaapi/mod.rs new file mode 100644 index 00000000..a78d1830 --- /dev/null +++ b/crates/platform/src/vaapi/mod.rs @@ -0,0 +1,217 @@ +//! Hardware decoding through VA-API (Linux: Intel, AMD and other Mesa / vendor drivers). +//! +//! VA-API decoding is stateless: the application parses the stream and keeps the decoded picture +//! buffer, and the GPU decodes one picture's slice data at a time into a surface. [`h264`] does the +//! host side with the software decoder's own parsers and DPB (so the two decide alike), [`va`] +//! drives libva (loaded at run time; no libva means no hardware decoder, never a failure to start), +//! and [`VaDecoder`] puts them together as a [`VideoDecoder`]. H.264 8-bit 4:2:0 progressive +//! (Constrained Baseline / Main / High) and HEVC 4:2:0 (Main, Main 10, Main Still Picture), the +//! latter through [`hevc`] the same way; everything else stays in software. +//! +//! The FFI is in [`ffi`] (declarations) and `va` (calls); the rest of this module is safe code. + +pub mod ffi; +pub mod h264; +pub mod hevc; +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +pub mod va; + +#[cfg(test)] +mod abi_tests; +#[cfg(test)] +mod tests; + +#[cfg(target_os = "linux")] +pub use linux::VaDecoder; + +use filmcraft_frame::VideoFrame; + +/// The hardware a front end ([`h264::Front`], [`hevc::Front`]) drives: a fixed set of surfaces, +/// picture decoding into one of them, and reading one back or filling it. +pub trait Accel: Send { + /// The VA surface ids of the decoder's surfaces; front ends refer to them by index. + fn surfaces(&self) -> &[ffi::VASurfaceID]; + /// Decode one H.264 picture (its slices: parameters and the NAL unit, header byte and + /// emulation prevention included) into surface `target` (an index into [`Accel::surfaces`]). + fn decode_h264( + &mut self, + target: usize, + pic: &ffi::VAPictureParameterBufferH264, + iq: &ffi::VAIQMatrixBufferH264, + slices: &[(ffi::VASliceParameterBufferH264, Vec)], + ) -> std::result::Result<(), String>; + /// Decode one HEVC picture (its slice segments: parameters and the NAL unit as stored) into + /// surface `target`; `iq` when scaling lists are enabled. + fn decode_hevc( + &mut self, + target: usize, + pic: &ffi::VAPictureParameterBufferHEVC, + iq: Option<&ffi::VAIQMatrixBufferHEVC>, + slices: &[(ffi::VASliceParameterBufferHEVC, Vec)], + ) -> std::result::Result<(), String>; + /// The picture in surface `index`, once decoded. + fn read(&mut self, index: usize) -> std::result::Result; + /// Make surface `index` a copy of surface `from`, or mid-gray (every sample half the range) + /// when `None`: the stand-ins for reference pictures that were never decoded, as the software + /// decoders make them. + fn fill(&mut self, index: usize, from: Option) -> std::result::Result<(), String>; +} + +#[cfg(target_os = "linux")] +mod linux { + use filmcraft_bitstream::unescape_rbsp; + use filmcraft_codecs::hw::{NalCodec, NalStreamInfo}; + use filmcraft_codecs::{CodecError, DecodedFrame, Result, VideoDecoder}; + + use super::va::Session; + use super::{ffi, h264, hevc}; + use crate::biplanar::Geometry; + + /// The VA-API profiles that decode an H.264 stream of `profile_idc`, best fit first (a Main or + /// High decoder also decodes what the smaller profiles allow). + pub(crate) fn h264_profiles(profile_idc: u8) -> Option<&'static [ffi::VAProfile]> { + Some(match profile_idc { + 66 => &[ffi::VAProfileH264ConstrainedBaseline, ffi::VAProfileH264Main, ffi::VAProfileH264High], + 77 => &[ffi::VAProfileH264Main, ffi::VAProfileH264High], + 100 => &[ffi::VAProfileH264High], + _ => return None, + }) + } + + /// The VA-API profiles that decode an HEVC stream of `general_profile_idc` (Main, Main 10, + /// Main Still Picture) at `bits`, best fit first. + pub(crate) fn hevc_profiles(profile_idc: u8, bits: u32) -> Option<&'static [ffi::VAProfile]> { + Some(match (profile_idc, bits) { + (1 | 3, 8) => &[ffi::VAProfileHEVCMain, ffi::VAProfileHEVCMain10], + (2, 8) => &[ffi::VAProfileHEVCMain10, ffi::VAProfileHEVCMain], + (2, 10) => &[ffi::VAProfileHEVCMain10], + _ => return None, + }) + } + + enum Front { + H264(Box>), + Hevc(Box>), + } + + /// A VA-API decoder for one H.264 or HEVC stream. + pub struct VaDecoder { + front: Front, + info: NalStreamInfo, + name: String, + fed: u64, + fail_after: Option, + } + + /// The first parameter set of NAL unit type `ty` among `sets`, without its header (`header` + /// bytes) and emulation prevention. + fn parameter_set(sets: &[Vec], header: usize, ty: impl Fn(u8) -> bool) -> Option> { + sets.iter().find(|n| n.first().is_some_and(|&h| ty(h))).and_then(|n| Some(unescape_rbsp(n.get(header..)?))) + } + + impl VaDecoder { + /// A decoder for the stream `info` describes, or why VA-API does not take it on this system. + pub fn new(info: NalStreamInfo) -> std::result::Result { + let bits = info.bit_depth_luma; + if info.chroma_format_idc != 1 || info.bit_depth_chroma != bits || info.interlaced { + return Err(format!( + "{} chroma format {} at {bits}/{}-bit is not taken", + if info.interlaced { "interlaced" } else { "progressive" }, + info.chroma_format_idc, + info.bit_depth_chroma + )); + } + let geometry = Geometry { crop: info.crop, bits, color: info.color, par: info.par }; + let (front, codec, vendor) = match info.codec { + NalCodec::H264 => { + if bits != 8 { + return Err(format!("{bits}-bit H.264 is not taken")); + } + let profiles = h264_profiles(info.profile_idc).ok_or_else(|| format!("H.264 profile_idc {} is not taken", info.profile_idc))?; + let rbsp = parameter_set(&info.parameter_sets, 1, |h| h & 0x1f == 7).ok_or("no SPS")?; + let sps = filmcraft_h264::params::Sps::parse(&rbsp).map_err(|e| e.to_string())?; + sps.check_supported().map_err(|e| e.to_string())?; + let mbs = (sps.pic_width_in_mbs, sps.frame_height_in_mbs()); + // the DPB, the picture being decoded, and one spare + let count = sps.max_dpb_frames().saturating_add(2).min(18); + let session = Session::new(profiles, (sps.width(), sps.height()), count, geometry)?; + let vendor = session.vendor().to_string(); + (Front::H264(Box::new(h264::Front::new(session, info.length_size, mbs, &info.parameter_sets).map_err(|e| e.to_string())?)), "H.264", vendor) + } + NalCodec::Hevc => { + let profiles = + hevc_profiles(info.profile_idc, bits).ok_or_else(|| format!("HEVC profile {} at {bits}-bit is not taken", info.profile_idc))?; + let rbsp = parameter_set(&info.parameter_sets, 2, |h| (h >> 1) & 0x3f == 33).ok_or("no SPS")?; + let sps = filmcraft_hevc::params::Sps::parse(&rbsp).map_err(|e| e.to_string())?; + sps.check_supported().map_err(|e| e.to_string())?; + if sps.range_extension { + return Err("HEVC range extensions are not taken".into()); + } + let count = (sps.max_dec_pic_buffering as usize).saturating_add(2).min(18); + let size = (sps.width, sps.height); + let session = Session::new(profiles, size, count, geometry)?; + let vendor = session.vendor().to_string(); + ( + Front::Hevc(Box::new(hevc::Front::new(session, info.length_size, size, bits, &info.parameter_sets).map_err(|e| e.to_string())?)), + "HEVC", + vendor, + ) + } + }; + let name = format!("VA-API {codec} ({vendor})"); + Ok(Self { front, info, name, fed: 0, fail_after: None }) + } + + /// Test hook: fail every decode call after the first `n` (a GPU lost mid-stream). + pub fn fail_after(&mut self, n: u64) { + self.fail_after = Some(n); + } + } + + impl VideoDecoder for VaDecoder { + fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { + self.fed += 1; + if self.fail_after.is_some_and(|n| self.fed > n) { + return Err(CodecError::Decode("VA-API decoder failed (test hook)".into())); + } + match &mut self.front { + Front::H264(f) => f.decode(sample, pts), + Front::Hevc(f) => f.decode(sample, pts), + } + } + + fn flush(&mut self) -> Vec { + let flushed = match &mut self.front { + Front::H264(f) => f.flush(), + Front::Hevc(f) => f.flush(), + }; + match flushed { + Ok(out) => out, + Err(e) => { + log::warn!("{}: {e} while flushing", self.name); + Vec::new() + } + } + } + + fn reset(&mut self) { + match &mut self.front { + Front::H264(f) => f.reset(), + Front::Hevc(f) => f.reset(), + } + } + + fn name(&self) -> &str { + &self.name + } + + fn is_random_access(&self, sample: &[u8]) -> Option { + self.info.is_random_access(sample) + } + + fn is_disposable(&self, sample: &[u8]) -> bool { + self.info.is_disposable(sample) + } + } +} diff --git a/crates/platform/src/vaapi/tests.rs b/crates/platform/src/vaapi/tests.rs new file mode 100644 index 00000000..df2fe7a4 --- /dev/null +++ b/crates/platform/src/vaapi/tests.rs @@ -0,0 +1,381 @@ +//! The H.264 and HEVC front ends against a recording stand-in for the hardware, on every system: +//! for x264 streams with B-pyramids, several references, weighted prediction, several slices and +//! open GOPs, and x265 streams (Main and Main 10, open GOP with RASL pictures, B-pyramids), +//! the pictures come out in the software decoder's order with its pts (whole stream, after seeks, +//! after a mid-stream flush), and every buffer handed to the hardware is consistent: the current +//! picture's surface is free, every reference is a decoded surface listed in the DPB, reference +//! lists have the active number of entries, and slice data offsets lie inside the slice. + +use std::collections::BTreeSet; +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +use filmcraft_codecs::DecodedFrame; +use filmcraft_codecs::hw::NalStreamInfo; +use filmcraft_frame::VideoFrame; + +use super::Accel; +use super::ffi::{self, VAIQMatrixBufferH264, VAPictureH264, VAPictureParameterBufferH264, VASliceParameterBufferH264}; +use super::h264::Front; +use crate::biplanar::{self, Biplanar, Geometry}; + +/// Records what it is asked to do and checks it; "decodes" nothing. +struct Recorder { + ids: Vec, + decoded: Vec, + pictures: usize, + frame: VideoFrame, +} + +impl Recorder { + fn new(count: usize) -> Self { + let g = Geometry { crop: (0, 0, 2, 2), bits: 8, color: Default::default(), par: (1, 1) }; + let frame = biplanar::to_frame(&Biplanar { luma: &[128; 4], chroma: &[128; 2], stride: 2, rows: 2 }, &g).unwrap(); + Self { ids: (0..count as u32).map(|i| 0x100 + i).collect(), decoded: vec![false; count], pictures: 0, frame } + } +} + +fn valid(p: &VAPictureH264) -> bool { + p.flags & ffi::VA_PICTURE_H264_INVALID == 0 +} + +impl Accel for Recorder { + fn surfaces(&self) -> &[ffi::VASurfaceID] { + &self.ids + } + + fn decode_hevc( + &mut self, + target: usize, + pic: &ffi::VAPictureParameterBufferHEVC, + _iq: Option<&ffi::VAIQMatrixBufferHEVC>, + slices: &[(ffi::VASliceParameterBufferHEVC, Vec)], + ) -> Result<(), String> { + assert_eq!(pic.CurrPic.picture_id, self.ids[target], "current picture surface"); + let valid = |p: &ffi::VAPictureHEVC| p.flags & ffi::VA_PICTURE_HEVC_INVALID == 0; + let n = pic.ReferenceFrames.iter().take_while(|p| valid(p)).count(); + assert!(pic.ReferenceFrames[n..].iter().all(|p| !valid(p)), "reference frames packed at the front"); + let ids: BTreeSet<_> = pic.ReferenceFrames[..n].iter().map(|r| r.picture_id).collect(); + assert_eq!(ids.len(), n, "each reference frame once"); + assert!(!ids.contains(&pic.CurrPic.picture_id), "the target is not a reference"); + for r in &pic.ReferenceFrames[..n] { + let i = self.ids.iter().position(|&id| id == r.picture_id).expect("reference is a decoder surface"); + assert!(self.decoded[i], "reference surface {i} holds a picture"); + } + let rps = ffi::VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE | ffi::VA_PICTURE_HEVC_RPS_ST_CURR_AFTER | ffi::VA_PICTURE_HEVC_RPS_LT_CURR; + assert!(!slices.is_empty()); + let last = ffi::hevc_slice_bits::LAST_SLICE_OF_PIC; + for (k, (s, data)) in slices.iter().enumerate() { + assert_eq!(s.slice_data_size as usize, data.len()); + assert!((s.slice_data_byte_offset as usize) <= data.len(), "slice data starts inside the slice"); + assert_eq!(s.LongSliceFlags >> last & 1 == 1, k + 1 == slices.len(), "LastSliceOfPic on the last slice only"); + let ty = (s.LongSliceFlags >> ffi::hevc_slice_bits::SLICE_TYPE) & 3; + let lists: &[(usize, u8)] = match ty { + 2 => &[], + 1 => &[(0, s.num_ref_idx_l0_active_minus1)], + _ => &[(0, s.num_ref_idx_l0_active_minus1), (1, s.num_ref_idx_l1_active_minus1)], + }; + for &(l, minus1) in lists { + let len = minus1 as usize + 1; + for &i in &s.RefPicList[l][..len] { + assert!((i as usize) < n, "list entry {i} is a reference frame"); + assert!(pic.ReferenceFrames[i as usize].flags & rps != 0, "list entries are in the current RPS"); + } + assert!(s.RefPicList[l][len..].iter().all(|&i| i == 0xFF), "nothing past the active entries"); + } + } + self.decoded[target] = true; + self.pictures += 1; + Ok(()) + } + + fn decode_h264( + &mut self, + target: usize, + pic: &VAPictureParameterBufferH264, + _iq: &VAIQMatrixBufferH264, + slices: &[(VASliceParameterBufferH264, Vec)], + ) -> Result<(), String> { + assert_eq!(pic.CurrPic.picture_id, self.ids[target], "current picture surface"); + let refs: Vec<_> = pic.ReferenceFrames.iter().filter(|r| valid(r)).collect(); + let ref_ids: BTreeSet<_> = refs.iter().map(|r| r.picture_id).collect(); + assert_eq!(ref_ids.len(), refs.len(), "each reference frame once"); + assert!(!ref_ids.contains(&pic.CurrPic.picture_id), "the target is not a reference"); + for r in &refs { + let i = self.ids.iter().position(|&id| id == r.picture_id).expect("reference is a decoder surface"); + assert!(self.decoded[i], "reference surface {i} holds a picture"); + assert!(r.flags & (ffi::VA_PICTURE_H264_SHORT_TERM_REFERENCE | ffi::VA_PICTURE_H264_LONG_TERM_REFERENCE) != 0, "reference flags"); + } + assert!(!slices.is_empty()); + for (s, data) in slices { + assert_eq!(s.slice_data_size as usize, data.len()); + assert!((s.slice_data_bit_offset as usize) < data.len() * 8, "slice data starts inside the slice"); + let ty = s.slice_type; + let lists: &[(&[VAPictureH264; 32], u8)] = match ty { + 2 => &[], + 0 => &[(&s.RefPicList0, s.num_ref_idx_l0_active_minus1)], + _ => &[(&s.RefPicList0, s.num_ref_idx_l0_active_minus1), (&s.RefPicList1, s.num_ref_idx_l1_active_minus1)], + }; + for (list, minus1) in lists { + let n = *minus1 as usize + 1; + assert!(list[..n].iter().all(valid), "{n} active entries"); + assert!(list[n..].iter().all(|p| !valid(p)), "nothing past the active entries"); + assert!(list[..n].iter().all(|p| ref_ids.contains(&p.picture_id)), "list entries are reference frames"); + } + } + self.decoded[target] = true; + self.pictures += 1; + Ok(()) + } + + fn read(&mut self, index: usize) -> Result { + assert!(self.decoded[index], "read back a decoded surface"); + Ok(self.frame.clone()) + } + + fn fill(&mut self, index: usize, from: Option) -> Result<(), String> { + assert!(from.is_none_or(|f| self.decoded[f])); + self.decoded[index] = true; + Ok(()) + } +} + +/// (file, x264 profile, x264 parameters). +const STREAMS: &[(&str, &str, &str)] = &[ + ("va_front_high.mp4", "high", "bframes=3:b-pyramid=normal:keyint=12:min-keyint=12:scenecut=0:ref=4:weightp=2"), + ("va_front_slices.mp4", "baseline", "keyint=12:min-keyint=12:scenecut=0:slices=4:ref=2"), + ("va_front_open_gop.mp4", "high", "bframes=3:b-pyramid=strict:keyint=12:min-keyint=12:scenecut=0:open-gop=1:direct=temporal"), +]; + +fn fixture(ff: &std::path::Path, name: &str, profile: &str, params: &str) -> Option { + let out = filmcraft_testkit::fixtures_dir("platform").join(name); + let src = "testsrc2=s=320x184:r=24:d=2"; + filmcraft_testkit::fixtures::generate(&out, |tmp| { + Command::new(ff) + .args(["-y", "-v", "error", "-f", "lavfi", "-i", src, "-c:v", "libx264", "-profile:v", profile, "-x264-params", params, "-pix_fmt", "yuv420p"]) + .arg(tmp) + .stdin(Stdio::null()) + .status() + .is_ok_and(|s| s.success()) + }) +} + +struct Stream { + entry: filmcraft_isobmff::SampleEntry, + samples: Vec<(Vec, i64)>, + sync: Vec, +} + +fn read_stream(path: &std::path::Path) -> Stream { + let bytes = std::fs::read(path).unwrap(); + let file = filmcraft_isobmff::open(bytes.clone()).unwrap(); + let t = file.track_of_kind(filmcraft_isobmff::TrackKind::Video).unwrap(); + let track = &file.tracks[t]; + let samples = (0..track.samples.len()).map(|i| (file.read_sample(&bytes, t, i).unwrap(), track.samples[i].pts)).collect(); + Stream { entry: track.entries[0].clone(), samples, sync: track.samples.iter().map(|s| s.is_sync).collect() } +} + +fn front(s: &Stream) -> Front { + let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap(); + let mbs = (info.coded.0 / 16, info.coded.1 / 16); + Front::new(Recorder::new(18), info.length_size, mbs, &info.parameter_sets).unwrap() +} + +fn pts(frames: &[DecodedFrame]) -> Vec { + frames.iter().map(|f| f.pts).collect() +} + +fn run_front(f: &mut Front, samples: &[(Vec, i64)]) -> Vec { + let mut out = Vec::new(); + for (s, p) in samples { + out.extend(pts(&f.decode(s, *p).unwrap())); + } + out.extend(pts(&f.flush().unwrap())); + out +} + +fn run_software(d: &mut dyn filmcraft_codecs::VideoDecoder, samples: &[(Vec, i64)]) -> Vec { + let mut out = Vec::new(); + for (s, p) in samples { + out.extend(pts(&d.decode(s, *p).unwrap())); + } + out.extend(pts(&d.flush())); + out +} + +#[test] +fn output_order_and_buffers_match_the_software_decoder() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, profile, params) in STREAMS { + let Some(path) = fixture(&ff, name, profile, params) else { continue }; + let s = read_stream(&path); + let mut f = front(&s); + let mut sw = filmcraft_codecs::software_video_decoder(&s.entry).unwrap(); + let a = run_front(&mut f, &s.samples); + assert_eq!(a, run_software(sw.as_mut(), &s.samples), "{name}"); + assert_eq!(f.accel().pictures, s.samples.len(), "{name}: every picture decoded"); + // seeks to every sync sample (open-GOP I pictures included: frame_num gap stand-ins) + for k in (1..s.samples.len()).filter(|&i| s.sync[i]) { + f.reset(); + sw.reset(); + let end = (k + 13).min(s.samples.len()); + assert_eq!(run_front(&mut f, &s.samples[k..end]), run_software(sw.as_mut(), &s.samples[k..end]), "{name} from sample {k}"); + } + // a flush mid-GOP, then on + f.reset(); + sw.reset(); + let mid = s.samples.len() / 2 + 1; + let mut a = run_front(&mut f, &s.samples[..mid]); + a.extend(run_front(&mut f, &s.samples[mid..])); + let mut b = run_software(sw.as_mut(), &s.samples[..mid]); + b.extend(run_software(sw.as_mut(), &s.samples[mid..])); + assert_eq!(a, b, "{name} with a flush at {mid}"); + } +} + +/// Damaged samples give errors (the hybrid decoder then continues in software), never a panic. +#[test] +fn damaged_samples_are_errors() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let (name, profile, params) = STREAMS[0]; + let Some(path) = fixture(&ff, name, profile, params) else { return }; + let s = read_stream(&path); + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for round in 0..40 { + let mut f = front(&s); + for (i, (smp, p)) in s.samples.iter().enumerate() { + let mut m = smp.clone(); + if i % 3 == round % 3 { + match next() % 3 { + 0 => m.truncate(next() as usize % m.len().max(1)), + 1 => { + let at = next() as usize % m.len().max(1); + if let Some(b) = m.get_mut(at) { + *b ^= 1 << (next() % 8); + } + } + _ => m.iter_mut().skip(4).take(6).for_each(|b| *b = next() as u8), + } + } + if f.decode(&m, *p).is_err() { + f.reset(); + } + } + let _ = f.flush(); + } +} + +/// (file, pixel format, x265 parameters). +const HEVC_STREAMS: &[(&str, &str, &str)] = &[ + ("va_front_hevc_open_gop.mp4", "yuv420p", "log-level=error:bframes=4:b-pyramid=1:keyint=12:min-keyint=12:scenecut=0:open-gop=1:ref=3"), + ("va_front_hevc_main10.mp4", "yuv420p10le", "log-level=error:bframes=3:keyint=12:min-keyint=12:scenecut=0:slices=2:weightp=1"), +]; + +fn hevc_fixture(ff: &std::path::Path, name: &str, pix_fmt: &str, params: &str) -> Option { + let out = filmcraft_testkit::fixtures_dir("platform").join(name); + let src = "testsrc2=s=320x184:r=24:d=2"; + let profile = if pix_fmt.contains("10") { "main10" } else { "main" }; + filmcraft_testkit::fixtures::generate(&out, |tmp| { + Command::new(ff) + .args([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + src, + "-c:v", + "libx265", + "-profile:v", + profile, + "-tag:v", + "hvc1", + "-x265-params", + params, + "-pix_fmt", + pix_fmt, + ]) + .arg(tmp) + .stdin(Stdio::null()) + .status() + .is_ok_and(|s| s.success()) + }) +} + +fn hevc_front(s: &Stream) -> super::hevc::Front { + let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap(); + super::hevc::Front::new(Recorder::new(18), info.length_size, info.coded, info.bit_depth_luma, &info.parameter_sets).unwrap() +} + +fn run_hevc(f: &mut super::hevc::Front, samples: &[(Vec, i64)]) -> Vec { + let mut out = Vec::new(); + for (s, p) in samples { + out.extend(pts(&f.decode(s, *p).unwrap())); + } + out.extend(pts(&f.flush().unwrap())); + out +} + +#[test] +fn hevc_output_order_and_buffers_match_the_software_decoder() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, pix_fmt, params) in HEVC_STREAMS { + let Some(path) = hevc_fixture(&ff, name, pix_fmt, params) else { continue }; + let s = read_stream(&path); + let mut f = hevc_front(&s); + let mut sw = filmcraft_codecs::software_video_decoder(&s.entry).unwrap(); + assert_eq!(run_hevc(&mut f, &s.samples), run_software(sw.as_mut(), &s.samples), "{name}"); + // seeks to every sync sample (open-GOP CRA pictures included: RASL pictures left out) + for k in (1..s.samples.len()).filter(|&i| s.sync[i]) { + f.reset(); + sw.reset(); + let end = (k + 13).min(s.samples.len()); + assert_eq!(run_hevc(&mut f, &s.samples[k..end]), run_software(sw.as_mut(), &s.samples[k..end]), "{name} from sample {k}"); + } + } +} + +/// Damaged HEVC samples give errors, never a panic. +#[test] +fn hevc_damaged_samples_are_errors() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let (name, pix_fmt, params) = HEVC_STREAMS[0]; + let Some(path) = hevc_fixture(&ff, name, pix_fmt, params) else { return }; + let s = read_stream(&path); + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for round in 0..40 { + let mut f = hevc_front(&s); + for (i, (smp, p)) in s.samples.iter().enumerate() { + let mut m = smp.clone(); + if i % 3 == round % 3 { + match next() % 3 { + 0 => m.truncate(next() as usize % m.len().max(1)), + 1 => { + let at = next() as usize % m.len().max(1); + if let Some(b) = m.get_mut(at) { + *b ^= 1 << (next() % 8); + } + } + _ => m.iter_mut().skip(4).take(6).for_each(|b| *b = next() as u8), + } + } + if f.decode(&m, *p).is_err() { + f.reset(); + } + } + let _ = f.flush(); + } +} diff --git a/crates/platform/src/vaapi/va.rs b/crates/platform/src/vaapi/va.rs new file mode 100644 index 00000000..5ee5e06e --- /dev/null +++ b/crates/platform/src/vaapi/va.rs @@ -0,0 +1,525 @@ +//! libva itself (Linux): `libva.so.2` and `libva-drm.so.2` are loaded at run time (a system without +//! them simply has no hardware decoder), a display is opened on a DRM render node, and a [`Session`] +//! (decode configuration, context and surfaces for one stream) decodes pictures and reads them back +//! as NV12 images. +//! +//! FFI module (docs/adr/0001-platform-ffi.md): every `unsafe` block has a `// SAFETY:` comment, the +//! public API is safe, and the one callback into Rust (libva's error messages) runs under +//! `catch_unwind`. + +use std::ffi::{CStr, c_char, c_int, c_uint, c_void}; +use std::fs::{File, OpenOptions}; +use std::os::fd::AsRawFd; +use std::sync::OnceLock; + +use filmcraft_frame::VideoFrame; +use libloading::Library; + +use super::Accel; +use super::ffi::{self, VAConfigAttrib, VAImage, VAImageFormat, VASurfaceID}; +use crate::biplanar::{self, Biplanar, Geometry}; + +/// The libva entry points (the libraries stay loaded for the life of the process). +struct Api { + _va: Library, + _drm: Library, + initialize: ffi::vaInitialize, + terminate: ffi::vaTerminate, + error_str: ffi::vaErrorStr, + vendor_string: ffi::vaQueryVendorString, + set_info_callback: ffi::vaSetInfoCallback, + set_error_callback: ffi::vaSetErrorCallback, + get_config_attributes: ffi::vaGetConfigAttributes, + create_config: ffi::vaCreateConfig, + destroy_config: ffi::vaDestroyConfig, + create_surfaces: ffi::vaCreateSurfaces, + destroy_surfaces: ffi::vaDestroySurfaces, + create_context: ffi::vaCreateContext, + destroy_context: ffi::vaDestroyContext, + create_buffer: ffi::vaCreateBuffer, + destroy_buffer: ffi::vaDestroyBuffer, + begin_picture: ffi::vaBeginPicture, + render_picture: ffi::vaRenderPicture, + end_picture: ffi::vaEndPicture, + sync_surface: ffi::vaSyncSurface, + create_image: ffi::vaCreateImage, + destroy_image: ffi::vaDestroyImage, + get_image: ffi::vaGetImage, + put_image: ffi::vaPutImage, + map_buffer: ffi::vaMapBuffer, + unmap_buffer: ffi::vaUnmapBuffer, + get_display_drm: ffi::vaGetDisplayDRM, +} + +static API: OnceLock> = OnceLock::new(); + +fn api() -> Result<&'static Api, String> { + API.get_or_init(load).as_ref().map_err(Clone::clone) +} + +fn load() -> Result { + // SAFETY: loading libva runs its library initialisers, which only set up libva's own state; + // it is the system's VA-API runtime, loaded the way every VA-API client loads it. + let va = unsafe { Library::new("libva.so.2") }.map_err(|e| format!("libva.so.2: {e}"))?; + // SAFETY: as above, for libva's DRM display library. + let drm = unsafe { Library::new("libva-drm.so.2") }.map_err(|e| format!("libva-drm.so.2: {e}"))?; + macro_rules! sym { + ($lib:expr, $name:ident) => {{ + // SAFETY: `ffi::$name` is the C signature of the libva function of that name, + // transcribed from va.h and checked against it (abi_tests.rs). The pointer is copied + // out; the library it points into is kept loaded in `Api` for the life of the process. + let s = unsafe { $lib.get::(concat!(stringify!($name), "\0").as_bytes()) }; + *s.map_err(|e| format!("{}: {e}", stringify!($name)))? + }}; + } + Ok(Api { + initialize: sym!(va, vaInitialize), + terminate: sym!(va, vaTerminate), + error_str: sym!(va, vaErrorStr), + vendor_string: sym!(va, vaQueryVendorString), + set_info_callback: sym!(va, vaSetInfoCallback), + set_error_callback: sym!(va, vaSetErrorCallback), + get_config_attributes: sym!(va, vaGetConfigAttributes), + create_config: sym!(va, vaCreateConfig), + destroy_config: sym!(va, vaDestroyConfig), + create_surfaces: sym!(va, vaCreateSurfaces), + destroy_surfaces: sym!(va, vaDestroySurfaces), + create_context: sym!(va, vaCreateContext), + destroy_context: sym!(va, vaDestroyContext), + create_buffer: sym!(va, vaCreateBuffer), + destroy_buffer: sym!(va, vaDestroyBuffer), + begin_picture: sym!(va, vaBeginPicture), + render_picture: sym!(va, vaRenderPicture), + end_picture: sym!(va, vaEndPicture), + sync_surface: sym!(va, vaSyncSurface), + create_image: sym!(va, vaCreateImage), + destroy_image: sym!(va, vaDestroyImage), + get_image: sym!(va, vaGetImage), + put_image: sym!(va, vaPutImage), + map_buffer: sym!(va, vaMapBuffer), + unmap_buffer: sym!(va, vaUnmapBuffer), + get_display_drm: sym!(drm, vaGetDisplayDRM), + _va: va, + _drm: drm, + }) +} + +/// libva's error messages, into our log (instead of stderr). +extern "C" fn on_error(_user: *mut c_void, message: *const c_char) { + let _ = std::panic::catch_unwind(|| { + if message.is_null() { + return; + } + // SAFETY: libva passes a NUL-terminated message that is valid for the duration of the call. + let text = unsafe { CStr::from_ptr(message) }.to_string_lossy(); + log::warn!("libva: {}", text.trim_end()); + }); +} + +/// An initialised VA display on a DRM render node. +pub struct Display { + api: &'static Api, + dpy: ffi::VADisplay, + vendor: String, + /// The render node; open for as long as the display uses it. + _device: File, +} + +// SAFETY: a VA display is not tied to the thread that opened it, and a `Display` is only used +// through its single owner (`&mut` or `&` of one `Session` at a time), never concurrently. +unsafe impl Send for Display {} + +impl Display { + /// The first DRM render node with a working VA-API driver. + pub fn open() -> Result { + let api = api()?; + let mut why = "no DRM render node (/dev/dri/renderD*)".to_string(); + for n in 128..144 { + let path = format!("/dev/dri/renderD{n}"); + let Ok(file) = OpenOptions::new().read(true).write(true).open(&path) else { continue }; + match Self::init(api, file) { + Ok(d) => return Ok(d), + Err(e) => why = format!("{path}: {e}"), + } + } + Err(why) + } + + fn init(api: &'static Api, device: File) -> Result { + // SAFETY: the descriptor is an open render node, kept open by `device` (moved into the + // returned display) for as long as the display exists. + let dpy = unsafe { (api.get_display_drm)(device.as_raw_fd()) }; + if dpy.is_null() { + return Err("vaGetDisplayDRM failed".into()); + } + // From here on `Drop` terminates the display. + let mut d = Self { api, dpy, vendor: String::new(), _device: device }; + // SAFETY: `dpy` is a display from vaGetDisplayDRM; a null info callback turns libva's info + // messages off, and `on_error` matches `VAMessageCallback` and never unwinds. + unsafe { + (api.set_info_callback)(dpy, None, std::ptr::null_mut()); + (api.set_error_callback)(dpy, Some(on_error), std::ptr::null_mut()); + } + let (mut major, mut minor): (c_int, c_int) = (0, 0); + // SAFETY: `dpy` is valid; the version outputs point to live locals. + let st = unsafe { (api.initialize)(dpy, &mut major, &mut minor) }; + d.check(st, "vaInitialize")?; + // SAFETY: `dpy` is an initialised display; the vendor string is NUL-terminated and owned by + // the driver for the life of the display (copied out here). + let vendor = unsafe { (api.vendor_string)(dpy) }; + if !vendor.is_null() { + // SAFETY: non-null, NUL-terminated (see above). + d.vendor = unsafe { CStr::from_ptr(vendor) }.to_string_lossy().into_owned(); + } + Ok(d) + } + + /// The driver's description ("Intel iHD driver …"). + pub fn vendor(&self) -> &str { + &self.vendor + } + + fn check(&self, st: ffi::VAStatus, what: &str) -> Result<(), String> { + if st == ffi::VA_STATUS_SUCCESS { + return Ok(()); + } + // SAFETY: vaErrorStr takes any status and returns a static NUL-terminated string. + let text = unsafe { (self.api.error_str)(st) }; + let text = if text.is_null() { + String::new() + } else { + // SAFETY: non-null static NUL-terminated string (see above). + unsafe { CStr::from_ptr(text) }.to_string_lossy().into_owned() + }; + Err(format!("{what}: {text} ({st:#x})")) + } +} + +impl Drop for Display { + fn drop(&mut self) { + // SAFETY: `dpy` came from vaGetDisplayDRM and is terminated exactly once, here, after + // every session object made on it was destroyed (`Session` drops its objects first). + unsafe { (self.api.terminate)(self.dpy) }; + } +} + +/// One stream's decoder: a VLD configuration of a profile, its surfaces and context, and the image +/// pictures are read back through. +pub struct Session { + display: Display, + config: ffi::VAConfigID, + context: ffi::VAContextID, + surfaces: Vec, + image: Option, + /// Coded size of the surfaces (luma samples). + size: (u32, u32), + geometry: Geometry, +} + +/// The surface and image formats of 8-bit (NV12) or 10-bit (P010) 4:2:0 pictures: +/// (render-target format, image fourcc, bits per pixel of the image). +fn formats(bits: u32) -> Result<(c_uint, u32, u32), String> { + match bits { + 8 => Ok((ffi::VA_RT_FORMAT_YUV420, ffi::VA_FOURCC_NV12, 12)), + 10 => Ok((ffi::VA_RT_FORMAT_YUV420_10, ffi::VA_FOURCC_P010, 24)), + _ => Err(format!("{bits}-bit pictures are not decoded through VA-API")), + } +} + +impl Session { + /// A decoder for the first of `profiles` the driver decodes (VLD, 4:2:0 of `geometry.bits`: + /// 8 or 10), with `count` surfaces of `size` (coded luma samples); pictures read back are cut + /// to `geometry`. + pub fn new(profiles: &[ffi::VAProfile], size: (u32, u32), count: usize, geometry: Geometry) -> Result { + let (rt_format, _, _) = formats(geometry.bits)?; + let display = Display::open()?; + let api = display.api; + let dpy = display.dpy; + let mut config = None; + let mut why = "no VA-API profile for the stream".to_string(); + for &profile in profiles { + let mut attr = VAConfigAttrib { type_: ffi::VAConfigAttribRTFormat, value: 0 }; + // SAFETY: `dpy` is initialised; `attr` is one live attribute (count 1). + let st = unsafe { (api.get_config_attributes)(dpy, profile, ffi::VAEntrypointVLD, &mut attr, 1) }; + if let Err(e) = display.check(st, "vaGetConfigAttributes") { + why = e; + continue; + } + if attr.value & rt_format == 0 { + why = format!("profile {profile} does not decode to {}-bit 4:2:0", geometry.bits); + continue; + } + attr.value = rt_format; + let mut id = ffi::VA_INVALID_ID; + // SAFETY: as above; `id` is a live output. + let st = unsafe { (api.create_config)(dpy, profile, ffi::VAEntrypointVLD, &mut attr, 1, &mut id) }; + match display.check(st, "vaCreateConfig") { + Ok(()) => { + config = Some(id); + break; + } + Err(e) => why = e, + } + } + let config = config.ok_or(why)?; + // From here on `Drop` releases what was made. + let mut s = Self { display, config, context: ffi::VA_INVALID_ID, surfaces: Vec::new(), image: None, size, geometry }; + let count_c = c_uint::try_from(count).map_err(|_| "too many surfaces")?; + let mut surfaces = vec![ffi::VA_INVALID_SURFACE; count]; + // SAFETY: `surfaces` has room for `count` ids; no attributes are passed (null, 0). + let st = unsafe { (api.create_surfaces)(dpy, rt_format, size.0, size.1, surfaces.as_mut_ptr(), count_c, std::ptr::null_mut(), 0) }; + s.display.check(st, "vaCreateSurfaces")?; + s.surfaces = surfaces; + let (w, h) = (c_int::try_from(size.0).map_err(|_| "picture too wide")?, c_int::try_from(size.1).map_err(|_| "picture too tall")?); + let n = c_int::try_from(count).map_err(|_| "too many surfaces")?; + let mut context = ffi::VA_INVALID_ID; + // SAFETY: `config` and the `count` surfaces were made on this display; `context` is a live output. + let st = unsafe { (api.create_context)(dpy, s.config, w, h, ffi::VA_PROGRESSIVE, s.surfaces.as_mut_ptr(), n, &mut context) }; + s.display.check(st, "vaCreateContext")?; + s.context = context; + Ok(s) + } + + pub fn vendor(&self) -> &str { + self.display.vendor() + } + + /// A buffer of `ty` holding a copy of `len` bytes at `data`. + fn buffer(&self, ty: ffi::VABufferType, data: *const c_void, len: usize) -> Result { + let size = c_uint::try_from(len).map_err(|_| "buffer too large")?; + let mut id = ffi::VA_INVALID_ID; + // SAFETY: `data` points to `len` readable bytes (the callers pass a live value or slice); + // libva copies them into the new buffer and never writes through the pointer. + let st = unsafe { (self.display.api.create_buffer)(self.display.dpy, self.context, ty, size, 1, data.cast_mut(), &mut id) }; + self.display.check(st, "vaCreateBuffer")?; + Ok(id) + } + + fn value_buffer(&self, ty: ffi::VABufferType, value: &T) -> Result { + self.buffer(ty, (value as *const T).cast(), std::mem::size_of::()) + } + + fn render(&self, buffers: &mut [ffi::VABufferID]) -> Result<(), String> { + let n = c_int::try_from(buffers.len()).map_err(|_| "too many buffers")?; + // SAFETY: `buffers` are live buffer ids made on this context, `n` of them. + let st = unsafe { (self.display.api.render_picture)(self.display.dpy, self.context, buffers.as_mut_ptr(), n) }; + self.display.check(st, "vaRenderPicture") + } + + /// Decode one picture into `surface`: the picture parameters `pic`, the scaling matrices `iq` + /// when given, and a parameter + data buffer per slice; the buffers made go into `buffers`. + fn decode_into( + &self, + surface: VASurfaceID, + buffers: &mut Vec, + pic: &P, + iq: Option<&Q>, + slices: &[(S, Vec)], + ) -> Result<(), String> { + buffers.push(self.value_buffer(ffi::VAPictureParameterBufferType, pic)?); + if let Some(iq) = iq { + buffers.push(self.value_buffer(ffi::VAIQMatrixBufferType, iq)?); + } + let head = buffers.len(); + for (params, data) in slices { + buffers.push(self.value_buffer(ffi::VASliceParameterBufferType, params)?); + buffers.push(self.buffer(ffi::VASliceDataBufferType, data.as_ptr().cast(), data.len())?); + } + let api = self.display.api; + // SAFETY: `surface` is one of this context's render targets. + let st = unsafe { (api.begin_picture)(self.display.dpy, self.context, surface) }; + self.display.check(st, "vaBeginPicture")?; + let (picture, slice_buffers) = buffers.split_at_mut(head); + let mut rendered = self.render(picture); + for pair in slice_buffers.chunks_mut(2) { + if rendered.is_err() { + break; + } + rendered = self.render(pair); + } + // a begun picture is always ended, so the context stays usable + // SAFETY: the picture was begun on this context above. + let st = unsafe { (api.end_picture)(self.display.dpy, self.context) }; + rendered?; + self.display.check(st, "vaEndPicture") + } + + fn image(&mut self) -> Result { + if let Some(image) = self.image { + return Ok(image); + } + let (_, fourcc, bits_per_pixel) = formats(self.geometry.bits)?; + let mut format = VAImageFormat { fourcc, byte_order: 1, bits_per_pixel, ..Default::default() }; + let mut image = VAImage { image_id: ffi::VA_INVALID_ID, buf: ffi::VA_INVALID_ID, ..Default::default() }; + let (w, h) = (c_int::try_from(self.size.0).map_err(|_| "picture too wide")?, c_int::try_from(self.size.1).map_err(|_| "picture too tall")?); + // SAFETY: `format` and `image` are live; the driver fills `image` in. + let st = unsafe { (self.display.api.create_image)(self.display.dpy, &mut format, w, h, &mut image) }; + self.display.check(st, "vaCreateImage")?; + self.image = Some(image); + Ok(image) + } + + /// The picture in surface `index`, copied into the session's image once decoded. + fn get_image(&mut self, index: usize) -> Result { + let surface = *self.surfaces.get(index).ok_or("surface out of range")?; + let image = self.image()?; + let api = self.display.api; + // SAFETY: `surface` is one of this session's surfaces. + let st = unsafe { (api.sync_surface)(self.display.dpy, surface) }; + self.display.check(st, "vaSyncSurface")?; + // SAFETY: the image and the surface were made on this display, with the same size. + let st = unsafe { (api.get_image)(self.display.dpy, surface, 0, 0, self.size.0, self.size.1, image.image_id) }; + self.display.check(st, "vaGetImage")?; + Ok(image) + } + + /// Every sample of `image` set to mid-gray: 128 in NV12, 512 (0x8000 in the high bits of + /// little-endian 16-bit words) in P010. + fn write_gray(&self, image: &VAImage) -> Result<(), String> { + let api = self.display.api; + let mut ptr: *mut c_void = std::ptr::null_mut(); + // SAFETY: `image.buf` is the data buffer of an image made on this display; `ptr` is a live output. + let st = unsafe { (api.map_buffer)(self.display.dpy, image.buf, &mut ptr) }; + self.display.check(st, "vaMapBuffer")?; + if !ptr.is_null() { + // SAFETY: a mapped image buffer holds `data_size` writable bytes at `ptr` until it is + // unmapped below; the slice does not outlive this block. + let data = unsafe { std::slice::from_raw_parts_mut(ptr.cast::(), image.data_size as usize) }; + if self.geometry.bits > 8 { + for w in data.as_chunks_mut::<2>().0 { + w.copy_from_slice(&0x8000u16.to_le_bytes()); + } + } else { + data.fill(128); + } + } + // SAFETY: the buffer was mapped above. + let st = unsafe { (api.unmap_buffer)(self.display.dpy, image.buf) }; + self.display.check(st, "vaUnmapBuffer")?; + if ptr.is_null() { Err("vaMapBuffer returned no data".into()) } else { Ok(()) } + } + + fn decode_picture(&mut self, target: usize, pic: &P, iq: Option<&Q>, slices: &[(S, Vec)]) -> Result<(), String> { + let surface = *self.surfaces.get(target).ok_or("surface out of range")?; + let mut buffers = Vec::with_capacity(2 + 2 * slices.len()); + let decoded = self.decode_into(surface, &mut buffers, pic, iq, slices); + for id in buffers { + // SAFETY: each id is a buffer made above on this display, destroyed once; vaEndPicture + // has returned, so the driver no longer reads it. + unsafe { (self.display.api.destroy_buffer)(self.display.dpy, id) }; + } + decoded + } + + fn read_image(&self, image: &VAImage) -> Result { + let api = self.display.api; + let mut ptr: *mut c_void = std::ptr::null_mut(); + // SAFETY: `image.buf` is the data buffer of an image made on this display; `ptr` is a live output. + let st = unsafe { (api.map_buffer)(self.display.dpy, image.buf, &mut ptr) }; + self.display.check(st, "vaMapBuffer")?; + let result = if ptr.is_null() { + Err("vaMapBuffer returned no data".to_string()) + } else { + // SAFETY: a mapped image buffer holds `data_size` bytes at `ptr` until it is unmapped + // below; the slice does not outlive this block. + let data = unsafe { std::slice::from_raw_parts(ptr.cast::().cast_const(), image.data_size as usize) }; + biplanar_view(image, data).and_then(|b| biplanar::to_frame(&b, &self.geometry)) + }; + // SAFETY: the buffer was mapped above; `data` is no longer used. + let st = unsafe { (api.unmap_buffer)(self.display.dpy, image.buf) }; + let frame = result?; + self.display.check(st, "vaUnmapBuffer")?; + Ok(frame) + } +} + +/// The two planes of a mapped NV12 / P010 image. +fn biplanar_view<'a>(image: &VAImage, data: &'a [u8]) -> Result, String> { + if !matches!(image.format.fourcc, ffi::VA_FOURCC_NV12 | ffi::VA_FOURCC_P010) || image.num_planes < 2 { + return Err(format!("the read-back image is not NV12 / P010 (fourcc {:#x}, {} planes)", image.format.fourcc, image.num_planes)); + } + let [p0, p1, _] = image.pitches; + if p0 != p1 { + return Err(format!("NV12 planes with different pitches ({p0}, {p1})")); + } + let luma = data.get(image.offsets[0] as usize..).ok_or("NV12 luma offset out of range")?; + let chroma = data.get(image.offsets[1] as usize..).ok_or("NV12 chroma offset out of range")?; + Ok(Biplanar { luma, chroma, stride: p0 as usize, rows: usize::from(image.height) }) +} + +impl Accel for Session { + fn surfaces(&self) -> &[VASurfaceID] { + &self.surfaces + } + + fn decode_h264( + &mut self, + target: usize, + pic: &ffi::VAPictureParameterBufferH264, + iq: &ffi::VAIQMatrixBufferH264, + slices: &[(ffi::VASliceParameterBufferH264, Vec)], + ) -> Result<(), String> { + self.decode_picture(target, pic, Some(iq), slices) + } + + fn decode_hevc( + &mut self, + target: usize, + pic: &ffi::VAPictureParameterBufferHEVC, + iq: Option<&ffi::VAIQMatrixBufferHEVC>, + slices: &[(ffi::VASliceParameterBufferHEVC, Vec)], + ) -> Result<(), String> { + self.decode_picture(target, pic, iq, slices) + } + + fn read(&mut self, index: usize) -> Result { + let image = self.get_image(index)?; + self.read_image(&image) + } + + fn fill(&mut self, index: usize, from: Option) -> Result<(), String> { + let surface = *self.surfaces.get(index).ok_or("surface out of range")?; + let image = match from { + Some(from) => self.get_image(from)?, + None => { + let image = self.image()?; + self.write_gray(&image)?; + image + } + }; + let (w, h) = self.size; + // SAFETY: the image and the surface were made on this display, with the same size. + let st = unsafe { (self.display.api.put_image)(self.display.dpy, surface, image.image_id, 0, 0, w, h, 0, 0, w, h) }; + self.display.check(st, "vaPutImage") + } +} + +impl Drop for Session { + fn drop(&mut self) { + let (api, dpy) = (self.display.api, self.display.dpy); + // SAFETY: each object below was made on this display and is destroyed once; the display + // itself is terminated afterwards (when the `display` field is dropped). + unsafe { + if let Some(image) = self.image.take() { + (api.destroy_image)(dpy, image.image_id); + } + if self.context != ffi::VA_INVALID_ID { + (api.destroy_context)(dpy, self.context); + } + if !self.surfaces.is_empty() { + let n = c_int::try_from(self.surfaces.len()).unwrap_or(0); + (api.destroy_surfaces)(dpy, self.surfaces.as_mut_ptr(), n); + } + if self.config != ffi::VA_INVALID_ID { + (api.destroy_config)(dpy, self.config); + } + } + } +} + +/// Whether this system has a working VA-API driver on a DRM render node (checked once), with the +/// driver's description, or why not. +pub fn probe() -> &'static Result { + static PROBE: OnceLock> = OnceLock::new(); + PROBE.get_or_init(|| Display::open().map(|d| d.vendor().to_string())) +} diff --git a/crates/platform/tests/hevc_probe.rs b/crates/platform/tests/hevc_probe.rs new file mode 100644 index 00000000..ba394a75 --- /dev/null +++ b/crates/platform/tests/hevc_probe.rs @@ -0,0 +1,29 @@ +//! `register()` answers the HEVC availability probe in the background, so the first draw of the +//! format list does not create a hardware session on the UI thread. One test in this file on +//! purpose: anything else that asked `hevc_available()` first would make the wait below meaningless. +#![cfg(target_os = "macos")] + +use std::time::{Duration, Instant}; + +use filmcraft_platform::hardware_encode::{hevc_available, hevc_probed, warm_hevc_probe}; + +#[test] +fn register_answers_the_hevc_probe_in_the_background() { + assert!(!hevc_probed(), "nothing has asked yet"); + + filmcraft_platform::register(); + + // nobody calls `hevc_available()` here: only the thread `register` started can answer + let t0 = Instant::now(); + while !hevc_probed() { + assert!(t0.elapsed() < Duration::from_secs(60), "register() did not start the probe"); + std::thread::sleep(Duration::from_millis(5)); + } + + // asking again, or registering again, changes nothing + let answer = hevc_available(); + warm_hevc_probe(); + filmcraft_platform::register(); + assert_eq!(hevc_available(), answer); + assert!(hevc_probed()); +} diff --git a/crates/platform/tests/vaapi.rs b/crates/platform/tests/vaapi.rs new file mode 100644 index 00000000..507e7768 --- /dev/null +++ b/crates/platform/tests/vaapi.rs @@ -0,0 +1,491 @@ +//! VA-API decoding against our software decoders (Linux): every picture of H.264 and HEVC fixtures +//! covering the stream features the hardware path takes (H.264: CAVLC and CABAC, B-pyramids, +//! several references, weighted prediction, temporal direct, custom scaling matrices, several +//! slices per picture; HEVC Main and Main 10: open GOP with RASL pictures, scaling lists, several +//! slices with wavefront parallel processing, weighted prediction, transform skip and AMP; 1080p / +//! 2160p) must be identical (bit-exact planes, colour, pixel aspect, pts and presentation +//! order), also after `reset` + reseek and a mid-stream `flush`; a forced mid-stream failure must +//! continue in software with the software decoder's exact output; what the hardware path does not +//! take (interlaced and 10-bit H.264, 4:2:2 HEVC) is declined; damaged samples and parameter sets give +//! errors or fall back, never crash or hang. Skips without ffmpeg (fixture generator) or without a +//! VA-API driver. +#![cfg(target_os = "linux")] + +mod common; + +use std::time::Duration; + +use common::*; +use filmcraft_codecs::VideoDecoder; +use filmcraft_codecs::hw::NalStreamInfo; +use filmcraft_platform::HybridDecoder; +use filmcraft_platform::vaapi::VaDecoder; + +/// The hardware decoder for a stream, or `None` (skip) when this machine has none for it. +fn hardware(s: &Stream) -> Option> { + match filmcraft_platform::vaapi_factory(&s.entry) { + Some(Ok(d)) => { + assert!(d.name().starts_with("VA-API"), "{}", d.name()); + Some(d) + } + Some(Err(e)) => panic!("factory error: {e}"), + None => { + eprintln!("SKIPPED: no VA-API hardware decoder for {}", s.entry.codec.name()); + None + } + } +} + +fn software(s: &Stream) -> Box { + filmcraft_codecs::software_video_decoder(&s.entry).unwrap() +} + +/// H.264 fixtures for the stream features the hardware path takes: (file, x264 profile, x264 +/// parameters). All 640x360 (coded 368: cropping), three seconds at 24 fps, keyframes every second. +const H264: &[(&str, &str, &str)] = &[ + ("va_h264_cbp_slices.mp4", "baseline", "keyint=24:min-keyint=24:scenecut=0:slices=3:ref=3"), + ("va_h264_main_weighted.mp4", "main", "bframes=3:b-pyramid=normal:keyint=24:min-keyint=24:scenecut=0:ref=4:weightp=2:weightb=1"), + ("va_h264_temporal_direct.mp4", "high", "bframes=2:b-pyramid=strict:direct=temporal:keyint=24:min-keyint=24:scenecut=0:ref=3:weightb=0"), + ("va_h264_cqm_jvt.mp4", "high", "bframes=3:keyint=24:min-keyint=24:scenecut=0:cqm=jvt:8x8dct=1:slices=2"), + ("va_h264_open_gop.mp4", "high", "bframes=3:b-pyramid=normal:keyint=24:min-keyint=24:scenecut=0:open-gop=1"), +]; + +fn h264_fixture(ff: &std::path::Path, name: &str, profile: &str, params: &str) -> Option { + let src = "testsrc2=s=640x360:r=24:d=3,noise=alls=12:allf=t"; + fixture(ff, name, &["-f", "lavfi", "-i", src, "-c:v", "libx264", "-profile:v", profile, "-preset", "fast", "-x264-params", params, "-pix_fmt", "yuv420p"]) +} + +/// HEVC fixtures for the stream features the hardware path takes: (file, pixel format, x265 +/// parameters). 640x360, three seconds at 24 fps, keyframes every second. +const HEVC: &[(&str, &str, &str)] = &[ + ("va_hevc_scaling_lists.mp4", "yuv420p", "log-level=error:bframes=3:keyint=24:min-keyint=24:scenecut=0:scaling-list=default"), + ("va_hevc_slices_wpp.mp4", "yuv420p10le", "log-level=error:bframes=3:keyint=24:min-keyint=24:scenecut=0:slices=3:wpp=1"), + ("va_hevc_weighted.mp4", "yuv420p", "log-level=error:bframes=3:keyint=24:min-keyint=24:scenecut=0:weightp=1:weightb=1:ref=4"), + ("va_hevc_tskip_amp.mp4", "yuv420p10le", "log-level=error:bframes=4:b-pyramid=1:keyint=24:min-keyint=24:scenecut=0:tskip=1:amp=1:rect=1:open-gop=1"), +]; + +fn hevc_fixture(ff: &std::path::Path, name: &str, pix_fmt: &str, params: &str) -> Option { + let src = "testsrc2=s=640x360:r=24:d=3,noise=alls=12:allf=t"; + let profile = if pix_fmt.contains("10") { "main10" } else { "main" }; + fixture( + ff, + name, + &["-f", "lavfi", "-i", src, "-c:v", "libx265", "-preset", "fast", "-profile:v", profile, "-tag:v", "hvc1", "-x265-params", params, "-pix_fmt", pix_fmt], + ) +} + +/// The small fixtures: the shared H.264 High, HEVC Main (open GOP) and Main 10 ones and the +/// feature fixtures above. +fn small_streams(ff: &std::path::Path) -> Vec<(String, Stream)> { + let mut out = Vec::new(); + for name in ["h264_high.mp4", "hevc_main.mp4", "hevc_main10.mp4"] { + if let Some(path) = named(ff, name) { + out.push((name.to_string(), read_stream(&path))); + } + } + for (name, profile, params) in H264 { + if let Some(path) = h264_fixture(ff, name, profile, params) { + out.push((name.to_string(), read_stream(&path))); + } + } + for (name, pix_fmt, params) in HEVC { + if let Some(path) = hevc_fixture(ff, name, pix_fmt, params) { + out.push((name.to_string(), read_stream(&path))); + } + } + out +} + +/// Every picture of `s` from the hardware decoder equals the software decoder's: whole stream, +/// after `reset` + reseek to each later sync sample, after a mid-stream `flush`, and a full pass +/// after all the resets. +fn parity(name: &str, s: &Stream) { + let Some(mut hw) = hardware(s) else { return }; + let mut sw = software(s); + let a = decode_all(hw.as_mut(), &s.samples); + let b = decode_all(sw.as_mut(), &s.samples); + assert!(a.len() >= 40, "{name}: {} pictures", a.len()); + assert_same(name, &a, &b); + assert!(hw.name().starts_with("VA-API"), "{name}: stayed in hardware ({})", hw.name()); + + // reset + reseek to each later sync sample, decode a stretch. After a seek to a non-IDR I picture + // (open GOP), the pictures shown before it are predicted from frames that were never decoded: + // both decoders conceal them (the software decoder from mid-gray stand-ins, the hardware also + // from motion data those stand-ins do not have), so only the pictures from the seek point on + // must match, as they must. + let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap(); + let syncs: Vec = (1..s.samples.len()).filter(|&i| s.sync[i]).collect(); + assert!(!syncs.is_empty(), "{name}: more than one GOP"); + for &k in syncs.iter().rev() { + let end = (k + 17).min(s.samples.len()); + for d in [&mut hw, &mut sw] { + d.reset(); + } + let mut a = decode_all(hw.as_mut(), &s.samples[k..end]); + let mut b = decode_all(sw.as_mut(), &s.samples[k..end]); + assert_eq!(a.len(), b.len(), "{name} from sample {k}: pictures"); + if info.codec == filmcraft_codecs::hw::NalCodec::H264 && !info.nal_types(&s.samples[k].0).contains(&5) { + let start = s.samples[k].1; + a.retain(|f| f.pts >= start); + b.retain(|f| f.pts >= start); + } + assert!(!a.is_empty(), "{name}: pictures after seeking to {k}"); + assert_same(&format!("{name} from sample {k}"), &a, &b); + } + // a flush in the middle of a GOP, then carrying on: H.264 only, where references survive a + // flush (an HEVC flush ends the stream; the GOP cache always seeks after one) + if info.codec == filmcraft_codecs::hw::NalCodec::H264 { + for d in [&mut hw, &mut sw] { + d.reset(); + } + let mid = s.samples.len() / 2; + let mut a = Vec::new(); + let mut b = Vec::new(); + for (d, out) in [(&mut hw, &mut a), (&mut sw, &mut b)] { + for (smp, pts) in &s.samples[..mid] { + out.extend(d.decode(smp, *pts).unwrap()); + } + out.extend(d.flush()); + out.extend(decode_all(d.as_mut(), &s.samples[mid..])); + } + assert_same(&format!("{name} with a flush at {mid}"), &a, &b); + } + // back to the start after seeking around: the whole stream again + hw.reset(); + sw.reset(); + assert_same(&format!("{name} after resets"), &decode_all(hw.as_mut(), &s.samples), &decode_all(sw.as_mut(), &s.samples)); + assert!(hw.name().starts_with("VA-API"), "{name}: stayed in hardware ({})", hw.name()); + // random-access / disposable answers match the software decoder's + for (smp, _) in &s.samples { + assert_eq!(hw.is_random_access(smp), sw.is_random_access(smp), "{name}"); + assert_eq!(hw.is_disposable(smp), sw.is_disposable(smp), "{name}"); + } +} + +/// [`parity`] for large streams, with little memory: the two decoders run side by side and each +/// picture is compared and dropped as soon as both have output it (a 4K 10-bit stream held whole +/// takes gigabytes). The whole stream, then from the last sync sample after a reset. +fn parity_streaming(name: &str, s: &Stream) { + let Some(mut hw) = hardware(s) else { return }; + let mut sw = software(s); + let last_sync = (1..s.samples.len()).rev().find(|&i| s.sync[i]).unwrap_or(0); + for start in [0, last_sync] { + hw.reset(); + sw.reset(); + let (mut a, mut b) = (std::collections::VecDeque::new(), std::collections::VecDeque::new()); + let mut compared = 0usize; + let mut compare = |a: &mut std::collections::VecDeque, + b: &mut std::collections::VecDeque| { + while !a.is_empty() && !b.is_empty() { + let (Some(x), Some(y)) = (a.pop_front(), b.pop_front()) else { break }; + assert_same(&format!("{name} from sample {start}"), &[x], &[y]); + compared += 1; + } + }; + for (smp, pts) in &s.samples[start..] { + a.extend(hw.decode(smp, *pts).unwrap()); + b.extend(sw.decode(smp, *pts).unwrap()); + compare(&mut a, &mut b); + } + a.extend(hw.flush()); + b.extend(sw.flush()); + compare(&mut a, &mut b); + assert!(a.is_empty() && b.is_empty(), "{name} from sample {start}: picture counts differ"); + assert!(compared >= s.samples.len().saturating_sub(start) / 2, "{name} from sample {start}: {compared} pictures"); + assert!(hw.name().starts_with("VA-API"), "{name}: stayed in hardware ({})", hw.name()); + } +} + +#[test] +fn bit_exact_with_the_software_decoder() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let streams = small_streams(&ff); + assert!(streams.len() > H264.len(), "fixtures"); + for (name, s) in streams { + parity(&name, &s); + } +} + +fn large(size: &str) { + let ff = filmcraft_testkit::require_ffmpeg!(); + let (_, h) = size.split_once('x').unwrap(); + let src = format!("testsrc2=s={size}:r=24:d=2,noise=alls=12:allf=t"); + let args = [ + "-f", + "lavfi", + "-i", + &src, + "-c:v", + "libx264", + "-profile:v", + "high", + "-preset", + "veryfast", + "-x264-params", + "bframes=3:b-pyramid=normal:keyint=24:min-keyint=24:scenecut=0", + "-pix_fmt", + "yuv420p", + ]; + let Some(path) = fixture(&ff, &format!("h264_{h}.mp4"), &args) else { return }; + parity_streaming(&format!("h264 {size}"), &read_stream(&path)); +} + +#[test] +fn h264_1080p_and_2160p_are_bit_exact() { + large("1920x1080"); + large("3840x2160"); +} + +fn large_hevc(size: &str, pix_fmt: &str) { + let ff = filmcraft_testkit::require_ffmpeg!(); + let (_, h) = size.split_once('x').unwrap(); + let src = format!("testsrc2=s={size}:r=24:d=2,noise=alls=12:allf=t"); + let ten = pix_fmt.contains("10"); + let args = [ + "-f", + "lavfi", + "-i", + &src, + "-c:v", + "libx265", + "-preset", + "ultrafast", + "-profile:v", + if ten { "main10" } else { "main" }, + "-tag:v", + "hvc1", + "-x265-params", + "log-level=error:bframes=4:keyint=24:min-keyint=24:scenecut=0:open-gop=1", + "-pix_fmt", + pix_fmt, + ]; + let name = format!("va_hevc_{}_{h}.mp4", if ten { "main10" } else { "main" }); + let Some(path) = fixture(&ff, &name, &args) else { return }; + parity_streaming(&name, &read_stream(&path)); +} + +#[test] +fn hevc_main_and_main10_1080p_and_2160p_are_bit_exact() { + for size in ["1920x1080", "3840x2160"] { + large_hevc(size, "yuv420p"); + large_hevc(size, "yuv420p10le"); + } +} + +#[test] +fn mid_stream_failure_continues_in_software() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, s) in small_streams(&ff) { + let name = name.as_str(); + let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap(); + let Ok(_) = VaDecoder::new(info.clone()) else { + eprintln!("SKIPPED: no VA-API hardware decoder for {name}"); + continue; + }; + let reference = decode_all(software(&s).as_mut(), &s.samples); + assert!(!reference.is_empty(), "{name}"); + // fail at the first sample, inside the first GOP, at a sync sample and just after one + let k = (1..s.samples.len()).find(|&i| s.sync[i]).unwrap(); + for fail_at in [0, 5, k, k + 1, k + 9] { + let mut va = VaDecoder::new(info.clone()).unwrap(); + va.fail_after(fail_at as u64); + let mut d = HybridDecoder::new(Box::new(va), s.entry.clone(), info.clone()); + let before = filmcraft_codecs::hw::hw_stats().fallbacks; + let out = decode_all(&mut d, &s.samples); + assert!(!d.is_hardware(), "{name}: switched to software"); + assert!(filmcraft_codecs::hw::hw_stats().fallbacks > before, "{name}: fallback counted"); + assert_same(&format!("{name} failing at sample {fail_at}"), &out, &reference); + d.reset(); + assert_same(&format!("{name} after the fallback and a reset"), &decode_all(&mut d, &s.samples), &reference); + } + } +} + +/// What the hardware path does not take is declined (the software decoders keep it): interlaced +/// and 10-bit H.264, and HEVC (not decoded through VA-API yet). +#[test] +fn declines_what_the_hardware_path_does_not_take() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let src = "testsrc2=s=640x360:r=24:d=1"; + let cases: [(&str, Vec<&str>); 2] = [ + ("va_h264_interlaced.mp4", vec!["-f", "lavfi", "-i", src, "-c:v", "libx264", "-x264-params", "interlaced=1:keyint=12", "-pix_fmt", "yuv420p"]), + ("va_h264_10bit.mp4", vec!["-f", "lavfi", "-i", src, "-c:v", "libx264", "-profile:v", "high10", "-pix_fmt", "yuv420p10le"]), + ]; + for (name, args) in cases { + // an ffmpeg whose x264 lacks 10-bit leaves the fixture out + let Some(path) = fixture(&ff, name, &args) else { continue }; + let s = read_stream(&path); + assert!(filmcraft_platform::vaapi_factory(&s.entry).is_none(), "{name}: declined"); + } + let args = ["-f", "lavfi", "-i", src, "-c:v", "libx265", "-tag:v", "hvc1", "-x265-params", "log-level=error:keyint=12", "-pix_fmt", "yuv422p10le"]; + if let Some(path) = fixture(&ff, "va_hevc_422.mp4", &args) { + let s = read_stream(&path); + assert!(filmcraft_platform::vaapi_factory(&s.entry).is_none(), "HEVC 4:2:2: declined"); + } + // Settings ▸ Playback ▸ Hardware decoding Off + let Some(path) = named(&ff, "h264_high.mp4") else { return }; + let s = read_stream(&path); + filmcraft_codecs::hw::set_hardware_decoding(false); + let off = filmcraft_platform::vaapi_factory(&s.entry).is_none(); + filmcraft_codecs::hw::set_hardware_decoding(true); + assert!(off, "declined while hardware decoding is Off"); +} + +/// Run `f` on a thread; fail if it panics or takes longer than `limit`. +fn bounded(what: &str, limit: Duration, f: impl FnOnce() + Send + 'static) { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + let _ = tx.send(r.is_ok()); + }); + match rx.recv_timeout(limit) { + Ok(true) => {} + Ok(false) => panic!("{what}: panicked"), + Err(_) => panic!("{what}: hung"), + } +} + +#[test] +fn damaged_samples_and_parameter_sets_never_crash_or_hang() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, s) in small_streams(&ff) { + let name = name.as_str(); + if hardware(&s).is_none() { + continue; + } + // damaged samples (bit flips, truncation, corrupt length prefixes): through the hybrid, and + // straight into the hardware decoder (no software to fall back to) + for seed in 1..=8u64 { + let samples = s.samples.clone(); + let entry = s.entry.clone(); + bounded(&format!("{name} seed {seed}"), Duration::from_secs(60), move || { + let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut d = filmcraft_platform::vaapi_factory(&entry).unwrap().unwrap(); + let info = NalStreamInfo::from_entry(&entry).unwrap().unwrap(); + let mut raw = VaDecoder::new(info).unwrap(); + for (i, (smp, pts)) in samples.iter().enumerate().take(40) { + let mut m = smp.clone(); + match xorshift(&mut seed) % 4 { + 0 => { + for _ in 0..1 + xorshift(&mut seed) % 8 { + let at = (xorshift(&mut seed) as usize) % m.len().max(1); + if let Some(b) = m.get_mut(at) { + *b ^= 1 << (xorshift(&mut seed) % 8); + } + } + } + 1 => m.truncate((xorshift(&mut seed) as usize) % m.len().max(1)), + 2 if m.len() > 4 => m[..4].copy_from_slice(&(xorshift(&mut seed) as u32).to_be_bytes()), + _ => {} + } + let _ = d.decode(&m, *pts); + let _ = raw.decode(&m, *pts); + if i % 13 == 12 { + d.reset(); + raw.reset(); + } + } + let _ = d.flush(); + let _ = raw.flush(); + }); + } + // damaged parameter sets: the decoder declines, errors or decodes; never crashes + let filmcraft_isobmff::CodecConfig::Avc(a) = &s.entry.codec else { continue }; + let rec = a.to_bytes(); + let mut seed = 0x1234_5678u64; + for round in 0..24 { + let mut r = rec.clone(); + if round % 3 == 2 { + r.truncate(r.len() * (round + 1) / 30); + } else { + for _ in 0..1 + round % 4 { + let at = 6 + (xorshift(&mut seed) as usize) % r.len().saturating_sub(6).max(1); + if let Some(b) = r.get_mut(at) { + *b ^= 1 << (xorshift(&mut seed) % 8); + } + } + } + let samples: Vec<_> = s.samples.iter().take(12).cloned().collect(); + bounded(&format!("{name} record round {round}"), Duration::from_secs(60), move || { + let Ok(info) = NalStreamInfo::from_avcc(&r) else { return }; + let Ok(mut va) = VaDecoder::new(info) else { return }; + for (smp, pts) in &samples { + if va.decode(smp, *pts).is_err() { + break; + } + } + let _ = va.flush(); + }); + } + } +} + +/// Creating and dropping decoders again and again (every clip on a timeline makes one) neither +/// fails nor leaks VA-API objects into the next one. +#[test] +fn repeated_initialisation() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let Some(path) = named(&ff, "h264_high.mp4") else { return }; + let s = read_stream(&path); + let reference = decode_all(software(&s).as_mut(), &s.samples[..13]); + for round in 0..40 { + let Some(mut hw) = hardware(&s) else { return }; + if round % 8 == 0 { + assert_same(&format!("round {round}"), &decode_all(hw.as_mut(), &s.samples[..13]), &reference); + } + } +} + +/// Several decoders at once on different threads (the frame workers) give the software decoder's +/// pictures. +#[test] +fn concurrent_decoders() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let mut paths: Vec<_> = named(&ff, "h264_high.mp4").into_iter().collect(); + for (name, profile, params) in H264.iter().take(3) { + paths.extend(h264_fixture(&ff, name, profile, params)); + } + let jobs: Vec<_> = paths + .into_iter() + .chain(named(&ff, "h264_high.mp4")) + .map(|path| { + std::thread::spawn(move || { + let s = read_stream(&path); + let Some(mut hw) = hardware(&s) else { return }; + let n = s.samples.len().min(40); + let a = decode_all(hw.as_mut(), &s.samples[..n]); + hw.reset(); + let b = decode_all(hw.as_mut(), &s.samples[..n]); + let c = decode_all(software(&s).as_mut(), &s.samples[..n]); + assert_same("concurrent", &a, &c); + assert_same("concurrent, after a reset", &b, &c); + }) + }) + .collect(); + for j in jobs { + j.join().unwrap(); + } +} + +/// A decoder used and moved between threads (decoders migrate between frame workers). +#[test] +fn decoder_moves_between_threads() { + let ff = filmcraft_testkit::require_ffmpeg!(); + let Some(path) = named(&ff, "h264_high.mp4") else { return }; + let s = read_stream(&path); + let Some(mut hw) = hardware(&s) else { return }; + let reference = decode_all(software(&s).as_mut(), &s.samples[..30]); + let samples = s.samples.clone(); + let (hw, out) = std::thread::spawn(move || { + let out = decode_all(hw.as_mut(), &samples[..30]); + (hw, out) + }) + .join() + .unwrap(); + assert_same("first thread", &out, &reference); + let mut hw = hw; + hw.reset(); + let samples = s.samples.clone(); + let out = std::thread::spawn(move || decode_all(hw.as_mut(), &samples[..30])).join().unwrap(); + assert_same("second thread", &out, &reference); +} diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 45438fee..20cf7355 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -338,6 +338,18 @@ pub fn clip_signal(item: &TrackItem, start: i64, frames: usize, sr: u32, sources Some(out) } +/// Whether a clip plays one source channel on both sides: a mono source, or one picked channel +/// (Modify ▸ Audio Channels, Breakout to Mono). Loudness Auto-Match measures such a clip as the one +/// channel it is, as BS.1770 (and ffmpeg's `ebur128`) measure a mono programme, not as the +/// dual-mono pair [`clip_signal`] plays, which reads 3 LU louder. `None` when the clip has no +/// audio source. +pub fn clip_is_mono(item: &TrackItem, sources: &dyn SourceProvider) -> Option { + let src = sources.source(item.item)?; + let available = src.info().audio.as_ref()?.channels as usize; + let (left, right) = source_pair(item, available); + Some(left == right) +} + /// The clip's audio (after speed/reverse) for timeline samples `[x0, x0 + len)` as `w` channels /// (2: stereo, mono sources on both sides; 6: 5.1), with silence outside the clip. fn raw_channels(item: &TrackItem, src: SourceAudio, x0: i64, len: usize, sr: u32, w: usize) -> Vec> { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index bcf4a0dd..78004218 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -489,11 +489,24 @@ fn plain_layer(frame: &VideoFrame, n: usize, opacity: f32, blend: Blend) -> Laye /// convert it with. fn media_frame(item: &TrackItem, t: Tick, sources: &dyn SourceProvider, want: f32, src_size: (u32, u32)) -> Option<(SharedSource, Arc, usize)> { let src = sources.source(item.item)?; - let frame = src.video_frame(FrameRequest { time: item.source_time_at(t), scale: want }).ok()?; + let frame = src.video_frame(FrameRequest { time: video_source_time(item, t, src.info().frame_rate()), scale: want }).ok()?; let n = decimation(frame.width as f32, src_size.0 as f32 * want); Some((src, frame, n)) } +/// The media time to ask a source for the video frame a clip shows at timeline `t`. A reversed +/// clip's source time is the instant just before its mirrored position; readers that round the +/// request to the nearest unit of a coarse timescale (MP4) land on the next frame from there, +/// showing one frame past the clip's source range and never its In (#318). Snapped down to the +/// start of the `media_rate` frame that contains it, every reader shows that frame. +pub(crate) fn video_source_time(item: &TrackItem, t: Tick, media_rate: filmcraft_time::FrameRate) -> Tick { + let ft = item.source_time_at(t); + if !item.reverse || item.frame_hold.is_some() || media_rate.frame_duration().0 <= 0 { + return ft; + } + media_rate.tick_of(media_rate.frame_at(ft)).clamp(item.source_in.min(ft), ft) +} + /// The linear image of a media clip's `frame`, colour managed, and blended with the next frame when /// the clip's speed asks for it. #[allow(clippy::too_many_arguments)] diff --git a/crates/render/src/plan.rs b/crates/render/src/plan.rs index bf4875e0..45bd1162 100644 --- a/crates/render/src/plan.rs +++ b/crates/render/src/plan.rs @@ -275,8 +275,7 @@ fn push_item( } // Graphic clips without standard effects: the layers are rasterised (cached) into one tight // image the GPU places as a layer. - if !(opts.effects && item.has_standard_effects()) - && !item.has_opacity_masks() + if !(opts.effects && item.has_standard_effects() || item.has_opacity_masks()) && project.item(item.item).is_some_and(|p| matches!(p.kind, ItemKind::Graphic { .. })) { let Some(size) = crate::source_size(project, item.item) else { return }; @@ -293,7 +292,8 @@ fn push_item( let motion = motion_matrix(seq, item, size, mt); let lin = ((motion.a * motion.a + motion.b * motion.b).sqrt()).max((motion.c * motion.c + motion.d * motion.d).sqrt()); let want = (lin * opts.scale as f64).clamp(1.0 / 64.0, 1.0) as f32; - let Ok(frame) = src.video_frame(FrameRequest { time: ft, scale: want }) else { return }; + let time = crate::video_source_time(item, t, src.info().frame_rate()); + let Ok(frame) = src.video_frame(FrameRequest { time, scale: want }) else { return }; let cs = crate::colorman::source_space(project, item.item, &frame); // log / HDR / wide-gamut media is converted on the CPU (below), and so are blended // in-between frames (Frame Blending / Optical Flow on speed-changed clips) diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index 20ec4fef..4707051c 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -369,3 +369,87 @@ fn a_multicam_clip_showing_itself_plans_and_renders_to_an_end() { let img = render_sequence(&p, a, Tick::ZERO, opts, &sources); assert!(img.w > 0 && img.h > 0); } + +/// A 24 fps source that picks frames like the MP4 reader: the requested time is converted to the +/// track timescale rounding to the nearest unit, then the frame at or before it is shown. It +/// records the frames it was asked for. +struct NearestUnitSource { + inner: GeneratorSource, + asked: std::sync::Mutex>, +} + +impl MediaSource for NearestUnitSource { + fn info(&self) -> &filmcraft_media::MediaInfo { + self.inner.info() + } + fn video_frame(&self, req: FrameRequest) -> filmcraft_media::Result> { + // 12288 units a second: 512 a frame at 24 fps + let units = req.time.to_rational_round(1, 12_288); + self.asked.lock().unwrap().push(units.div_euclid(512)); + self.inner.video_frame(req) + } + fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { + self.inner.audio(start, frames, sample_rate) + } +} + +/// #318: a trimmed clip (source frames 10–29) played in reverse shows 29 down to 10. Its source +/// time is the instant just before the mirrored position; a reader that rounds to the nearest +/// timescale unit used to land on the next frame, showing frame 30 (outside the clip) and never 10. +#[test] +fn a_reversed_clip_shows_exactly_its_source_range() { + let r = FrameRate::FPS_24; + let mut p = Project::new("r"); + let g = GeneratorSource::new(Generator::ColorMatte { color: [0.2, 0.4, 0.6, 1.0] }, 64, 64, r, Tick(4 * TICKS_PER_SECOND)); + let info = g.info().clone(); + let item = p.add_item( + "ids", + Label::Iris, + ItemKind::Media(MediaClip { + media: MediaRef::Generator(g.generator.clone()), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }), + None, + ); + let src = Arc::new(NearestUnitSource { inner: g, asked: Default::default() }); + let mut map = SourceMap::default(); + map.0.insert(item, src.clone() as SharedSource); + let seq = p.new_sequence("s", SequenceSettings { width: 64, height: 64, frame_rate: r, ..Default::default() }, 1, 0, None); + let ti = p.make_track_item(item, TrackKind::Video, Tick::ZERO, TimeRange::new(r.tick_of(10), r.tick_of(20)), r).unwrap(); + let id = ti.id; + p.sequence_mut(seq).unwrap().video_tracks[0].items.push(ti); + let opts = RenderOptions { scale: 1.0, ..Default::default() }; + let shown = |p: &Project, f: i64, planned: bool| -> Vec { + src.asked.lock().unwrap().clear(); + if planned { + plan::execute_cpu(&plan::plan_frame(p, seq, r.tick_of(f), opts, &map)); + } else { + render_sequence(p, seq, r.tick_of(f), opts, &map); + } + let mut asked = src.asked.lock().unwrap().clone(); + asked.dedup(); + asked + }; + // forward control: 10, 17, 29 + for (f, want) in [(0, 10), (7, 17), (19, 29)] { + for planned in [false, true] { + assert_eq!(shown(&p, f, planned), [want], "forward, timeline frame {f}, planned {planned}"); + } + } + p.sequence_mut(seq).unwrap().find_item_mut(id).unwrap().1.reverse = true; + for (f, want) in [(0, 29), (7, 22), (19, 10)] { + for planned in [false, true] { + assert_eq!(shown(&p, f, planned), [want], "reversed, timeline frame {f}, planned {planned}"); + } + } + // every timeline frame stays inside the clip's source range, each source frame shown once + let all: Vec = (0..20).flat_map(|f| shown(&p, f, false)).collect(); + assert_eq!(all, (10..=29).rev().collect::>()); +} diff --git a/crates/text/Cargo.toml b/crates/text/Cargo.toml index d9f7f8e6..00989623 100644 --- a/crates/text/Cargo.toml +++ b/crates/text/Cargo.toml @@ -19,3 +19,5 @@ harfrust = { version = "0.12", default-features = false, features = ["std"] } unicode-bidi = "0.3.18" # UAX #14 line breaking (Apache-2.0). unicode-linebreak = "0.1.5" +# UAX #29 extended grapheme clusters (MIT OR Apache-2.0). +unicode-segmentation = "1.13" diff --git a/crates/text/src/fonts.rs b/crates/text/src/fonts.rs index 963f28e9..6cceff10 100644 --- a/crates/text/src/fonts.rs +++ b/crates/text/src/fonts.rs @@ -215,6 +215,23 @@ impl Face { pub fn has_char(&self, c: char) -> bool { self.glyph(c).is_some() } + /// Height of the glyph's vertical origin above its baseline, in pixels. + /// OpenType VORG takes precedence; TrueType uses the ink top plus vmtx's top bearing. + pub(crate) fn vertical_origin(&self, gid: u32, px: f32) -> Option { + use skrifa::raw::TableProvider; + let font = self.font()?; + let glyph = skrifa::GlyphId::new(gid); + let units = match font.vorg() { + Ok(table) => f32::from(table.vertical_origin_y(glyph)), + Err(_) => { + let bounds = font.glyph_metrics(Size::unscaled(), LocationRef::default()).bounds(glyph)?; + bounds.y_max + f32::from(font.vmtx().ok()?.side_bearing(glyph)?) + } + }; + let origin = units * px / self.units_per_em(); + (origin.is_finite() && origin.abs() < px * 4.0).then_some(origin) + } + pub fn metrics(&self, px: f32) -> VMetrics { let Some(f) = self.font() else { return VMetrics { diff --git a/crates/text/src/layout.rs b/crates/text/src/layout.rs index 31281afb..c28b8479 100644 --- a/crates/text/src/layout.rs +++ b/crates/text/src/layout.rs @@ -20,6 +20,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use harfrust::{Direction, Feature, Tag, UnicodeBuffer}; use unicode_bidi::{BidiInfo, Level}; +use unicode_segmentation::UnicodeSegmentation; use crate::fonts::{self, FaceId, Resolved}; @@ -218,7 +219,7 @@ pub struct Layout { /// Lines were left out because they do not fit the box height. pub overflow: bool, pub text_len: usize, - /// Laid out vertically (one line per character; columns right to left). + /// Laid out vertically (one line per grapheme; columns right to left). pub vertical: bool, } @@ -319,7 +320,11 @@ fn upper_single(c: char) -> char { const SMALL_CAPS_SCALE: f32 = 0.78; -fn shape_item(text: &str, chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, style: &TextStyle) -> Vec { +fn uses_vertical_form(c: char) -> bool { + matches!(c as u32, 0x2E80..=0x9FFF | 0xF900..=0xFAFF | 0xFE10..=0xFE1F | 0xFE30..=0xFE4F | 0xFF00..=0xFFEF | 0x20000..=0x3FFFF) +} + +fn shape_item(text: &str, chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, style: &TextStyle, vertical: bool) -> Vec { let f = fonts::face(face); let (Some(font), Some(data)) = (f.font(), f.shaper_data()) else { return Vec::new() }; let shaper = data.shaper(&font).build(); @@ -330,6 +335,10 @@ fn shape_item(text: &str, chars: &[(usize, char, char)], rtl: bool, face: FaceId buf.set_direction(if rtl { Direction::RightToLeft } else { Direction::LeftToRight }); buf.guess_segment_properties(); let mut feats = Vec::new(); + if vertical && chars.first().is_some_and(|c| uses_vertical_form(c.1)) { + feats.push(Feature::new(Tag::new(b"vert"), 1, ..)); + feats.push(Feature::new(Tag::new(b"vrt2"), 1, ..)); + } if !style.kerning { feats.push(Feature::new(Tag::new(b"kern"), 0, ..)); } @@ -416,7 +425,7 @@ fn paragraph(text: &str, base: usize, sty: &Styles, para: &ParagraphStyle) -> Ve let st = sty.list[si]; let sub: Vec<(usize, char, char)> = chars[i..j].iter().map(|c| (c.0, c.1, c.2)).collect(); let size = if small { st.size * SMALL_CAPS_SCALE } else { st.size }; - let mut glyphs = shape_item(text, &sub, rtl, face, size, st); + let mut glyphs = shape_item(text, &sub, rtl, face, size, st, para.vertical); // tracking after each cluster if st.tracking != 0.0 { let t = st.tracking * st.size / 1000.0; @@ -707,13 +716,13 @@ pub fn layout_rich_uncached(text: &str, style: &TextStyle, runs: &[StyleRun], pa lay } -/// Vertical layout: each character is its own line (so carets, hit testing and selection work +/// Vertical layout: each grapheme is its own line (so carets, hit testing and selection work /// unchanged), stacked top to bottom and centred on its column; columns run right to left. fn layout_vertical(text: &str, sty: &Styles, para: &ParagraphStyle, col_w: f32, mut lay: Layout) -> Layout { let style = sty.list[0]; let pm = fonts::face(sty.prim[0].face).metrics(style.size); let row = (pm.ascent + pm.descent).max(style.size * 0.5) + style.tracking * style.size / 1000.0; - let flat = ParagraphStyle::default(); + let flat = ParagraphStyle { vertical: true, ..Default::default() }; lay.vertical = true; let mut base = 0usize; let mut ncols = 0usize; @@ -724,8 +733,8 @@ fn layout_vertical(text: &str, sty: &Styles, para: &ParagraphStyle, col_w: f32, if p_clean.is_empty() { cells.extend(paragraph("", base, sty, &flat)); } - for (off, ch) in p_clean.char_indices() { - cells.extend(paragraph(&p_clean[off..off + ch.len_utf8()], base + off, sty, &flat)); + for (off, cluster) in p_clean.grapheme_indices(true) { + cells.extend(paragraph(cluster, base + off, sty, &flat)); } let h = row * cells.len() as f32; let y0 = match para.align { @@ -740,7 +749,16 @@ fn layout_vertical(text: &str, sty: &Styles, para: &ParagraphStyle, col_w: f32, let g0 = lay.glyphs.len(); lay.glyphs.extend(pl.glyphs.into_iter().map(|mut g| { g.x += shift; - g.y += baseline; + // The alternate glyph hangs from its vertical origin. A horizontal ascent + // is not the top bearing of a vertical punctuation glyph (OpenType VORG/vmtx). + let font = fonts::face(g.face); + if text.get(g.cluster..).and_then(|s| s.chars().next()).is_some_and(uses_vertical_form) + && let Some(origin) = font.vertical_origin(g.id, g.size) + { + g.y += baseline - pl.ascent + origin; + } else { + g.y += baseline; + } g })); lay.lines.push(Line { @@ -910,6 +928,53 @@ mod tests { assert!(l.glyphs[3].x < l.glyphs[0].x); } + #[test] + fn japanese_vertical_punctuation_uses_alternates_at_the_top_right() { + use skrifa::{ + MetadataProvider, + instance::{LocationRef, Size}, + }; + fonts::scan_system(); + let face = fonts::all_faces().into_iter().find(|f| { + ((f.info.family.contains("Hiragino") && !f.info.family.contains(" GB") && !f.info.family.contains(" TC")) + || f.info.family.contains("Shippori") + || f.info.family.contains("BIZ UD")) + && "国、。「」ー".chars().all(|c| f.has_char(c)) + && (f.has_feature(b"vert") || f.has_feature(b"vrt2")) + }); + let Some(face) = face else { + eprintln!("SKIPPED: no Japanese font with vertical alternates available"); + return; + }; + let style = TextStyle { family: face.info.family.clone(), style: face.info.style.clone(), ..st(48.0) }; + let text = "国、。「」ーABC"; + let horizontal = layout_uncached(text, &style, &ParagraphStyle::default()); + let vertical = layout_uncached(text, &style, &ParagraphStyle { vertical: true, ..Default::default() }); + assert_eq!(vertical.glyphs.len(), 9); + assert_eq!(vertical.glyphs[0].id, horizontal.glyphs[0].id); + for i in 1..6 { + assert_ne!(vertical.glyphs[i].id, horizontal.glyphs[i].id, "{}: vertical form for glyph {i}", face.info.family); + } + for i in [1, 2] { + let g = &vertical.glyphs[i]; + let line = &vertical.lines[i]; + let font = fonts::face(g.face); + let bounds = font.font().unwrap().glyph_metrics(Size::unscaled(), LocationRef::default()).bounds(skrifa::GlyphId::new(g.id)).unwrap(); + let scale = g.size / font.units_per_em(); + let ink_x = g.x + (bounds.x_min + bounds.x_max) * scale / 2.0; + let ink_y = g.y - (bounds.y_min + bounds.y_max) * scale / 2.0; + let centre_x = line.x + line.width / 2.0; + let cell_top = line.baseline - line.ascent; + assert!(ink_x > centre_x, "{}: punctuation ink on the right ({ink_x}, {centre_x})", face.info.family); + assert!(ink_y < cell_top + g.size / 2.0, "{}: punctuation ink at the top ({ink_y}, {cell_top})", face.info.family); + } + for i in 6..9 { + assert_eq!(vertical.glyphs[i].id, horizontal.glyphs[i].id, "Latin shaping stays unchanged"); + } + assert_eq!(layout_uncached(text, &style, &ParagraphStyle::default()), horizontal); + eprintln!("verified vertical punctuation with {} / {}", face.info.family, face.info.style); + } + #[test] fn vertical_text_stacks_characters_in_columns() { let p = ParagraphStyle { vertical: true, ..Default::default() }; @@ -933,6 +998,51 @@ mod tests { assert_eq!(h.lines.len(), 2); } + #[test] + fn vertical_cells_keep_combining_marks_with_their_base() { + let style = st(32.0); + let para = ParagraphStyle { vertical: true, ..Default::default() }; + let decomposed = layout_uncached("e\u{301}x", &style, ¶); + let composed = layout_uncached("éx", &style, ¶); + assert_eq!(decomposed.lines.len(), 2, "the combining acute must not get another vertical cell"); + assert_eq!(decomposed.lines[0].range, 0..3); + assert_eq!(decomposed.lines[1].range, 3..4); + assert_eq!(decomposed.text_len, 4, "retain the original UTF-8 source offsets"); + assert_eq!( + decomposed.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::>(), + composed.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::>() + ); + assert_eq!(decomposed.bounds, composed.bounds); + assert_eq!(decomposed.caret(0).1, decomposed.caret(1).1, "base and combining mark occupy the same cell"); + for text in ["葛\u{e0101}x", "👩\u{200d}👧x"] { + let lay = layout_uncached(text, &style, ¶); + assert_eq!(lay.lines.len(), 2, "variation selectors and ZWJ sequences stay with their base"); + assert_eq!(lay.lines[0].range, 0..text.len() - 1); + assert_eq!(lay.lines[1].range, text.len() - 1..text.len()); + } + } + + #[test] + fn vertical_kana_keep_decomposed_dakuten_in_the_same_cell() { + fonts::scan_system(); + let Some(face) = fonts::all_faces().into_iter().find(|f| { + (f.info.family.contains("Hiragino") || f.info.family.contains("Shippori") || f.info.family.contains("BIZ UD")) + && "かがはぱ\u{3099}\u{309a}".chars().all(|c| f.has_char(c)) + }) else { + eprintln!("SKIPPED: no Japanese font covering combining dakuten/handakuten"); + return; + }; + let style = TextStyle { family: face.info.family.clone(), style: face.info.style.clone(), ..st(48.0) }; + let para = ParagraphStyle { vertical: true, ..Default::default() }; + let nfd = layout_uncached("か\u{3099}は\u{309a}", &style, ¶); + let nfc = layout_uncached("がぱ", &style, ¶); + assert_eq!(nfd.lines.len(), 2); + assert_eq!(nfd.lines[0].range, 0..6); + assert_eq!(nfd.lines[1].range, 6..12); + assert_eq!(nfd.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::>(), nfc.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::>()); + assert_eq!(nfd.bounds, nfc.bounds); + } + #[test] fn carets_and_hit_testing() { let l = layout("abc\nde", &st(40.0), &ParagraphStyle::default()); diff --git a/crates/time/src/lib.rs b/crates/time/src/lib.rs index 231120a3..4da7e27a 100644 --- a/crates/time/src/lib.rs +++ b/crates/time/src/lib.rs @@ -224,11 +224,20 @@ impl FrameRate { (t.0 as i128 * r.num as i128).div_euclid(TICKS_PER_SECOND as i128 * r.den as i128) as i64 } - /// Start tick of frame `f`. + /// Start tick of frame `f` (ceiling division). + /// + /// Uses ceiling (not floor) so that `frame_at(tick_of(f)) == f` holds for every + /// frame index, even when the frame duration is not an integer number of ticks + /// (exotic rates like 37.516 fps where `TICKS_PER_SECOND * den / num` has a + /// fractional remainder). Floor division truncates the remainder, causing each + /// frame boundary to drift backward by one tick until `frame_at` returns `f - 1` + /// instead of `f` — frame stepping then stalls or moves backward. + /// Ceiling is correct because the frame duration is always > 1 tick, so + /// `floor(ceil(f·D) / D) == f` for every integer `f`. pub fn tick_of(self, f: i64) -> Tick { let r = self.sane(); let n = f as i128 * TICKS_PER_SECOND as i128 * r.den as i128; - Tick(n.div_euclid(r.num as i128) as i64) + Tick((-(-n).div_euclid(r.num as i128)) as i64) } /// Snap `t` down to a frame boundary. @@ -571,6 +580,85 @@ mod tests { assert_eq!(FrameRate::FPS_23_976.label(), "23.976"); } + #[test] + fn exotic_frame_rate_roundtrip() { + // Issue #301: 37.516 fps footage. The frame duration is + // 254_016_000_000 * 250 / 9379 = 6_770_871_094.999… ticks — not an + // integer. Floor division in `tick_of` truncated to 6_770_871_094, + // losing ~1 tick/frame so `frame_at(tick_of(f))` returned `f - 1`. + // Ceiling division makes the round-trip exact for all integer frames. + let rates: &[(FrameRate, &str)] = &[ + (FrameRate::from_f64(37.516), "37.516"), + (FrameRate::from_f64(37.5161), "37.5161"), + (FrameRate::from_f64(12.345), "12.345"), + (FrameRate::from_f64(59.9401), "59.9401"), + (FrameRate::from_f64(23.9761), "23.9761"), + ]; + for &(r, name) in rates { + // Round-trip over a wide range including negatives. + let bad: Vec = (-20_000..20_000).filter(|&f| r.frame_at(r.tick_of(f)) != f).collect(); + assert!(bad.is_empty(), "{name}: {} round-trip failures, first {bad:?}", r); + + // snap(tick_of(f)) == tick_of(f): snapping a frame boundary is a no-op. + for f in -100i64..100 { + assert_eq!(r.snap(r.tick_of(f)), r.tick_of(f), "{name}: snap(tick_of({f}))"); + } + + // Step-back: the last tick of frame f is one tick before tick_of(f+1), + // and frame_at of that is still f. Also check the frame_duration path: + // frame_at(tick_of(f) - frame_duration()) == f - 1. + for f in 1..2000i64 { + let last_tick = r.tick_of(f + 1) - Tick(1); + assert_eq!(r.frame_at(last_tick), f, "{name}: last tick of frame {f}"); + let prev = r.tick_of(f) - r.frame_duration(); + assert_eq!(r.frame_at(prev), f - 1, "{name}: tick_of({f}) - frame_duration"); + } + } + + // Stepping simulation: frame_at(playhead) + 1, tick_of, snap. + // Before the fix this stalled at frame 0 (step forward did nothing). + let r = FrameRate::from_f64(37.516); + let mut tick = Tick::ZERO; + for expected in 0..50 { + let frame = r.frame_at(tick); + assert_eq!(frame, expected, "stepping forward at frame {expected}"); + tick = r.tick_of(frame + 1); + tick = r.snap(tick); // set_playhead calls snap + } + + // Stepping backward from frame 50 should land on 49, 48, … 0 + tick = r.tick_of(50); + let mut last = 50; + while last > 0 { + let frame = r.frame_at(tick); + let target = frame - 1; + if target < 0 { + break; + } + tick = r.snap(r.tick_of(target)); + let landed = r.frame_at(tick); + assert_eq!(landed, target, "stepping backward to frame {target}"); + last = landed; + } + } + + #[test] + fn exotic_frame_rate_durations() { + // frame_duration uses floor (not round) so that `end - frame_duration()` + // lands on the previous frame, not two frames back, for exotic rates. + let r = FrameRate::from_f64(37.516); + // 254_016_000_000 * 250 / 9379 = 6_770_871_094.999… → floor = 6_770_871_094 + assert_eq!(r.frame_duration().0, 6_770_871_094); + + // Standard rates must remain exact (unchanged). + assert_eq!(FrameRate::FPS_23_976.frame_duration().0, 10_594_584_000); + assert_eq!(FrameRate::FPS_24.frame_duration().0, 10_584_000_000); + assert_eq!(FrameRate::FPS_25.frame_duration().0, 10_160_640_000); + assert_eq!(FrameRate::FPS_29_97.frame_duration().0, 8_475_667_200); + assert_eq!(FrameRate::FPS_30.frame_duration().0, 8_467_200_000); + assert_eq!(FrameRate::FPS_60.frame_duration().0, 4_233_600_000); + } + proptest! { #[test] fn frame_tick_roundtrip(f in -1_000_000i64..10_000_000, ri in 0usize..11) { @@ -579,6 +667,16 @@ mod tests { prop_assert_eq!(r.frame_at(r.tick_of(f) + r.frame_duration() - Tick(1)), f); } + #[test] + fn exotic_frame_tick_roundtrip(f in -100_000i64..100_000, num in 1u64..200_000, den in 1u64..2_000) { + // Skip rates that divide evenly (those are covered by COMMON above). + let r = FrameRate::new(num as i64, den as i64); + if (TICKS_PER_SECOND as i128 * r.den as i128) % r.num as i128 == 0 { + return Ok(()); + } + prop_assert_eq!(r.frame_at(r.tick_of(f)), f, "rate={}/{}, f={}", r.num, r.den, f); + } + #[test] fn drop_frame_bijection(f in 0i64..(24 * 107_892)) { let r = FrameRate::FPS_29_97; diff --git a/crates/ui-egui/examples/thumb_burst.rs b/crates/ui-egui/examples/thumb_burst.rs new file mode 100644 index 00000000..87b18f42 --- /dev/null +++ b/crates/ui-egui/examples/thumb_burst.rs @@ -0,0 +1,88 @@ +//! Memory of the thumbnail burst that follows an import (issue: 6+ GB while importing 42 4K H.264 +//! clips with hardware decoding off). +//! +//! Imports every `*.mp4` of a directory one by one, as the Project panel does, and asks for each +//! clip's poster thumbnail (priority 50, 160 px) through the real [`FrameServer`] right after its +//! import. Prints the wall time until every thumbnail is ready and the peak number of live +//! decoders. Measure the process' private bytes from outside (`stress.py`'s sampler, or any +//! process monitor). +//! +//! ```sh +//! cargo run --release -p filmcraft-ui-egui --example thumb_burst -- [--hw off|auto] [--limit N] +//! ``` + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use filmcraft_engine::Session; +use filmcraft_project::ItemId; +use filmcraft_ui_egui::frames::{FrameKey, FrameServer, Target}; +use serde_json::json; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let value = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned(); + let Some(dir) = args.first().filter(|a| !a.starts_with("--")).map(PathBuf::from) else { + eprintln!("usage: thumb_burst [--hw off|auto] [--limit N]"); + return; + }; + filmcraft_codecs::hw::set_hardware_decoding(value("--hw").as_deref() != Some("off")); + let limit = value("--limit").and_then(|v| v.parse().ok()).unwrap_or(usize::MAX); + let mut clips: Vec = std::fs::read_dir(&dir).map(|d| d.filter_map(|e| e.ok().map(|e| e.path())).collect()).unwrap_or_default(); + clips.retain(|p| p.extension().is_some_and(|e| e == "mp4")); + clips.sort(); + clips.truncate(limit); + + let mut s = Session::default(); + let server = FrameServer::new(s.media.clone(), s.services.clone(), s.previews.clone(), FrameServer::default_workers()); + + let peak = std::sync::Arc::new(AtomicUsize::new(0)); + let stop = std::sync::Arc::new(AtomicBool::new(false)); + let sampler = { + let (peak, stop) = (peak.clone(), stop.clone()); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + peak.fetch_max(filmcraft_codecs::live_decoders(), Ordering::Relaxed); + std::thread::sleep(Duration::from_millis(5)); + } + }) + }; + + let t0 = Instant::now(); + let mut keys = Vec::new(); + for path in &clips { + let r = s.execute("file.import", json!({"paths": [path.to_string_lossy()]})).expect("import"); + let id = ItemId(r["items"][0].as_u64().expect("item")); + let (rate, width) = match s.project.item(id).map(|i| (i.frame_rate(), &i.kind)) { + Some((rate, filmcraft_project::ItemKind::Media(m))) => (rate, m.info.video.as_ref().map_or(1920, |v| v.width)), + _ => continue, + }; + let rev = { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + if let Some(filmcraft_project::ItemKind::Media(m)) = s.project.item(id).map(|i| &i.kind) { + filmcraft_engine::media_pool::media_key(m).hash(&mut h); + false.hash(&mut h); + } + h.finish() + }; + let key = FrameKey { target: Target::Item(id), frame: 0, size: 160, revision: rev, draft: false }; + server.request(key, rate.tick_of(0), 160.0 / width.max(1) as f32, &s.project.clone(), 50); + keys.push(key); + } + let imported = t0.elapsed(); + while keys.iter().any(|k| !server.is_ready(k)) && t0.elapsed() < Duration::from_secs(300) { + std::thread::sleep(Duration::from_millis(10)); + } + stop.store(true, Ordering::Relaxed); + let _ = sampler.join(); + println!( + "clips {} imported in {:.2}s, all thumbnails ready after {:.2}s, peak live decoders {}, gop frames cached {:.0} MB", + keys.len(), + imported.as_secs_f64(), + t0.elapsed().as_secs_f64(), + peak.load(Ordering::Relaxed), + filmcraft_codecs::cached_bytes() as f64 / 1048576.0 + ); +} diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index e491e36a..1f14c14b 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -13,7 +13,8 @@ //! - `ui.click {id | x,y, button?, count?, modifiers?}` / `ui.move {x,y}` / `ui.scroll {x,y,dx,dy}` //! - `ui.drag {from:{id|x,y}, to:{id|x,y}, steps?, modifiers?}`: synthetic press-move-release //! - `ui.key {key, command?, shift?, alt?, ctrl?}` / `ui.type {text}` -//! - `ui.timeline.hit {x, y}`: what the timeline shows at a point (track, clip, edge, time) +//! - `ui.timeline.hit {x, y, modifiers?}`: what the timeline shows at a point (track, clip, edge, +//! time) and `kind`, the trim a press of the current tool with `modifiers` starts there //! - `ui.timeline.locate {clip, edge?}`: screen point of a clip body/edge (for drags) //! - `ui.playback {action: play|stop|toggle, speed?}` //! - `ui.screenshot {path?, panel?}`: PNG of the window (or one panel) @@ -62,10 +63,23 @@ fn err(e: impl std::fmt::Display) -> Outcome { Outcome::Done(json!({"ok": false, "error": e.to_string()})) } +/// Modifier flags from `{modifiers: {ctrl, command, shift, alt}}` (or the same keys at the top +/// level), as the OS would report them pressed. fn modifiers(p: &Value) -> egui::Modifiers { let m = p.get("modifiers").unwrap_or(p); let b = |k: &str| m.get(k).and_then(Value::as_bool).unwrap_or(false); - egui::Modifiers { alt: b("alt"), ctrl: b("ctrl"), shift: b("shift"), mac_cmd: cfg!(target_os = "macos") && b("command"), command: b("command") } + as_pressed(egui::Modifiers { alt: b("alt"), ctrl: b("ctrl"), shift: b("shift"), mac_cmd: cfg!(target_os = "macos") && b("command"), command: b("command") }) +} + +/// `m` as egui-winit reports a physical press: off macOS Control is the primary modifier, so +/// `ctrl` and `command` are one key and a press of either carries both. A synthetic `Ctrl+Z` then +/// matches a `Cmd+Z` binding exactly as the real key does (#245). +fn as_pressed(mut m: egui::Modifiers) -> egui::Modifiers { + if !cfg!(target_os = "macos") && (m.ctrl || m.command) { + m.ctrl = true; + m.command = true; + } + m } /// Resolve a point from `{id}` (element centre) or `{x, y}`. @@ -332,9 +346,8 @@ pub fn handle(app: &mut FilmcraftApp, ctx: &egui::Context, req: &ControlRequest) } "ui.key" => { let Some(name) = s("key") else { return err("missing `key`") }; - let Some((mut m, key)) = crate::menus::parse_shortcut(name) else { return err(format!("unknown key `{name}`")) }; - let extra = modifiers(p); - m |= extra; + let Some((m, key)) = crate::menus::parse_shortcut(name) else { return err(format!("unknown key `{name}`")) }; + let m = as_pressed(m | modifiers(p)); app.synthetic.push(egui::Event::Key { key, physical_key: None, pressed: true, repeat: false, modifiers: m }); app.synthetic.push(egui::Event::Key { key, physical_key: None, pressed: false, repeat: false, modifiers: m }); Outcome::AfterInput @@ -350,7 +363,7 @@ pub fn handle(app: &mut FilmcraftApp, ctx: &egui::Context, req: &ControlRequest) Ok(p) => p, Err(e) => return err(e), }; - ok(crate::panels::timeline::hit_json(app, pos)) + ok(crate::panels::timeline::hit_json(app, pos, modifiers(p))) } "ui.timeline.locate" => { let Some(c) = p.get("clip").and_then(Value::as_u64) else { return err("need `clip`") }; diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 6e0ef1ef..522f53f2 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -35,6 +35,22 @@ pub enum Target { MulticamAngle(ItemId, u32), } +/// Item jobs at this priority or a larger number are background work: thumbnails of the Project +/// panel and the timeline (50) and the Media Browser (40). Monitors and playback use 0..=3. Their +/// decoders are small and are not kept (`filmcraft_media::cancel::with_background`). +pub const BACKGROUND_PRIO: u32 = 40; + +/// Whether a job is a thumbnail: a single item at a background priority. Sequence frames are never +/// background, whatever their priority (the scopes ask for the program frame at 40 while playing, +/// and must not turn the program's decoders into small ones). +fn is_background(job: &Job) -> bool { + background_work(job.prio, job.key.target) +} + +fn background_work(prio: u32, target: Target) -> bool { + prio >= BACKGROUND_PRIO && matches!(target, Target::Item(_)) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct FrameKey { pub target: Target, @@ -535,10 +551,15 @@ impl FrameServer { // A new frame on screen for this view replaces the one asked for before: scrubbing // asks for one per refresh, and without this the oldest position would decode // first (or keep a decoder busy seeking to it) while the newest waits. + // + // A job already running for the same frame at an older revision (a value being + // dragged asks for a new revision per refresh) is left to finish: it needs the same + // source frames, so it holds no decoder back, and when frames take longer than a + // refresh, cancelling each one for the next would show nothing until the mouse rests. let same_view = |k: &FrameKey| k.target == key.target && k.size == key.size; q.retain(|j| !(j.prio == 0 && !j.prefetch && same_view(&j.key))); for (k, c, prefetch) in self.shared.in_flight.lock().unwrap_or_else(|e| e.into_inner()).iter() { - if !prefetch && *k != key && same_view(k) { + if !prefetch && same_view(k) && k.frame != key.frame { c.store(true, Ordering::Relaxed); } } @@ -804,23 +825,25 @@ fn run_job(sh: &Shared, job: &Job, pool: &Arc, services: &Arc, services: &Arc, pr #[cfg(test)] mod tests { - use super::{PlaybackMeter, playback_plan}; + use super::{PlaybackMeter, Target, background_work, playback_plan}; + use filmcraft_project::ItemId; + + #[test] + fn only_item_thumbnails_are_background_work() { + let item = Target::Item(ItemId(1)); + // thumbnails (50) and the Media Browser (40) + assert!(background_work(50, item)); + assert!(background_work(40, item)); + // the Source monitor shows an item at a foreground priority + assert!(!background_work(1, item)); + // the scopes ask for the program frame at 40 while playing: never background + assert!(!background_work(40, Target::Sequence(ItemId(1)))); + assert!(!background_work(50, Target::SequencePlan(ItemId(1)))); + } #[test] fn draft_decoding_only_for_reduced_resolution_playback() { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 461aa36e..79c75b57 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -130,6 +130,8 @@ pub enum Dialog { DeleteTracks, /// Sequence ▸ Add Tracks…. AddTracks, + /// Sequence ▸ Sequence Settings…. + SequenceSettings, } #[derive(Default)] @@ -172,6 +174,17 @@ const SCREENSHOT_TIMEOUT_S: f64 = 10.0; /// timeline zoom animation finished) before capturing what is there. const SCREENSHOT_SETTLE_MAX_S: f64 = 5.0; +/// File ▸ Export entries that run from the menus through the save panel: command, filter label, +/// extension. +pub(crate) const EXPORT_SAVE_DIALOGS: [(&str, &str, &str); 6] = [ + ("file.exportEdl", "EDL", "edl"), + ("file.exportFcp7Xml", "Final Cut Pro XML", "xml"), + ("file.exportFcpxml", "FCPXML", "fcpxml"), + ("file.exportOtio", "OpenTimelineIO", "otio"), + ("file.exportAle", "Avid Log Exchange", "ale"), + ("file.exportSelectionProject", "FilmCraft Project", "fcproj"), +]; + pub struct FilmcraftApp { pub session: Session, pub ui: UiState, @@ -200,6 +213,11 @@ pub struct FilmcraftApp { /// A paused monitor drew a stand-in (nearest cached) picture last frame: its exact frame is /// still decoding. pub(crate) monitor_inexact: bool, + /// Per monitor: the frames it asked for on its latest paused passes, oldest first (stand-ins + /// while the newest renders, see `panels::monitor`). + pub(crate) monitor_asked: HashMap>, + /// The frame whose picture the Program monitor drew last: the frame due, or its stand-in. + pub(crate) program_shown: Option, /// Consecutive frames the timeline zoom / scroll has been at rest. `ui.elements` answers from /// the frame before the last one, so its timeline rects are final from 2 on. pub(crate) timeline_still: u32, @@ -396,6 +414,8 @@ impl FilmcraftApp { pending_screenshots: Vec::new(), queued_screenshots: Vec::new(), monitor_inexact: false, + monitor_asked: HashMap::new(), + program_shown: None, timeline_still: 0, input_waiters: Vec::new(), next_token: 1, @@ -892,6 +912,17 @@ impl FilmcraftApp { self.textures.get(name).map(|(_, t)| (t.id(), t.size_vec2())) } + /// The frame a named texture shows. + pub fn texture_key(&self, name: &str) -> Option { + self.textures.get(name).map(|(k, _)| *k) + } + + /// The frame whose picture the Program monitor drew last: the frame due at the playhead, or + /// the stand-in shown while that one renders (`perf.stats`, tests). + pub fn program_picture(&self) -> Option { + self.program_shown + } + /// Get a thumbnail texture for an item at a media time (requested at low priority). /// Cache revision of a project item's own frames: media changes only when its file does /// (relink, Make Offline, proxies on/off), so its frames survive unrelated edits; other items @@ -938,6 +969,11 @@ impl FilmcraftApp { // ---------------------------------------------------------------- files + /// Destination of imports started from the currently shown Project panel view. + pub fn import_bin(&self) -> filmcraft_project::BinId { + panels::project::view_of(self, panels::project::shown_inst(self)).bin + } + pub fn file_dialog(&mut self, id: &str, params: &Value) -> Result { match id { "file.import" => { @@ -952,7 +988,8 @@ impl FilmcraftApp { if paths.is_empty() { return Ok(Value::Null); } - let r = self.session.execute("file.import", json!({"paths": paths})).map_err(|e| e.to_string()); + let bin = params.get("bin").cloned().unwrap_or_else(|| json!(self.import_bin().0)); + let r = self.session.execute("file.import", json!({"paths": paths, "bin": bin})).map_err(|e| e.to_string()); if let Ok(v) = &r && let Some(errs) = v.get("errors").and_then(Value::as_array) && !errs.is_empty() @@ -1007,10 +1044,40 @@ impl FilmcraftApp { p["path"] = json!(path); self.session.execute("captions.export", p).map_err(|e| e.to_string()) } - _ => Err(format!("no dialog for {id}")), + _ => match EXPORT_SAVE_DIALOGS.iter().find(|(c, ..)| *c == id) { + Some(&(_, filter, ext)) => self.export_save_dialog(id, filter, ext, params), + None => Err(format!("no dialog for {id}")), + }, } } + /// File ▸ Export ▸ EDL…, Final Cut Pro XML… etc. from the menus (#382): ask where to save, then + /// run the command with that `path` and the rest of `params`. + fn export_save_dialog(&mut self, id: &str, filter: &str, ext: &str, params: &Value) -> Result { + filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&self.session))?; + let Some(pick) = self.hooks.pick_save_as.as_mut() else { + return Err("no save dialog available: run the command with a `path`".into()); + }; + // Timelines are named after the sequence; a selection must not suggest the open project's file. + let sequence = self.session.state.active_sequence.and_then(|s| self.session.project.item(s)).map(|i| i.name.clone()); + let stem = match id { + "file.exportAle" => self.session.project.name.clone(), + "file.exportSelectionProject" => format!("{} selection", self.session.project.name), + _ => sequence.unwrap_or_else(|| self.session.project.name.clone()), + }; + let stem: String = stem.chars().map(|c| if matches!(c, '/' | '\\') || c.is_control() { '-' } else { c }).collect(); + let stem = if stem.trim().is_empty() { "Untitled".to_string() } else { stem }; + let Some(path) = pick(filter, &[ext], &format!("{stem}.{ext}")) else { return Ok(Value::Null) }; + let mut p = params.as_object().cloned().unwrap_or_default(); + p.insert("path".into(), json!(path)); + let r = self.session.execute(id, Value::Object(p)).map_err(|e| e.to_string()); + self.ui.status = match &r { + Ok(_) => format!("Exported {path}"), + Err(e) => e.clone(), + }; + r + } + /// Import dropped files. fn handle_drops(&mut self, ctx: &egui::Context) { let dropped = ctx.input(|i| i.raw.dropped_files.clone()); @@ -1022,7 +1089,7 @@ impl FilmcraftApp { } } if !paths.is_empty() { - let _ = self.session.execute("file.import", json!({"paths": paths})); + let _ = self.session.execute("file.import", json!({"paths": paths, "bin": self.import_bin().0})); } } @@ -1235,10 +1302,12 @@ impl FilmcraftApp { let t = self.tokens; let full = ui.max_rect(); ui.painter().rect_filled(full, 0.0, t.app_bg); - let header_h = 38.0; + let header_h = if self.ui.show_header { 38.0 } else { 0.0 }; let header = egui::Rect::from_min_size(full.min, egui::vec2(full.width(), header_h)); - header::show(self, ui, header); - let status_h = 20.0; + if self.ui.show_header { + header::show(self, ui, header); + } + let status_h = if self.ui.show_status_bar { 20.0 } else { 0.0 }; let body = egui::Rect::from_min_max(egui::pos2(full.min.x + 1.0, header.max.y + 1.0), egui::pos2(full.max.x - 1.0, full.max.y - status_h - 2.0)); match self.ui.mode { state::Mode::Edit => self.dock_area(ui, body), @@ -1246,9 +1315,14 @@ impl FilmcraftApp { state::Mode::Export => panels::export_mode::show(self, ui, body), } panels::dialogs::show(self, &ctx); + if !self.ui.show_status_bar { + // no bar to draw the job in, but a finished preview render still plays + self.watch_jobs(ui.ctx()); + return; + } // Status / hint bar let sb = egui::Rect::from_min_max(egui::pos2(full.min.x, full.max.y - status_h), full.max); - ui.painter().rect_filled(sb, 0.0, egui::Color32::from_rgb(0x1c, 0x1c, 0x1c)); + ui.painter().rect_filled(sb, 0.0, t.header_bg); let now = ui.input(|i| i.time); if self.ui.status != self.status_seen.0 { self.status_seen = (self.ui.status.clone(), now); @@ -1268,9 +1342,9 @@ impl FilmcraftApp { self.job_status(ui, sb, &t); } - /// Right side of the status bar: the running job (export / render previews) with a progress - /// bar and a cancel button; plays the rendered range when a preview render completes. - fn job_status(&mut self, ui: &mut egui::Ui, sb: egui::Rect, t: &Tokens) { + /// The running job (export / render previews), if any; plays the rendered range when a preview + /// render completes. Runs every frame, with or without the status bar. + fn watch_jobs(&mut self, ctx: &egui::Context) -> Option { use std::sync::atomic::Ordering; let running = self.session.jobs.iter().rev().find(|j| !j.progress.finished.load(Ordering::Relaxed)).cloned(); // Play after rendering previews. @@ -1285,12 +1359,20 @@ impl FilmcraftApp { self.play(1.0); } } - let Some(job) = running else { return }; + let job = running?; if job.label.starts_with("Rendering ") && !job.label.contains("audio") && self.watched_render.is_none_or(|w| w.0 != job.id) { let from = self.session.active_sequence().and_then(|q| q.mark_in).unwrap_or(Tick::ZERO); self.watched_render = Some((job.id, from)); } - ui.ctx().request_repaint_after(std::time::Duration::from_millis(150)); + ctx.request_repaint_after(std::time::Duration::from_millis(150)); + Some(job) + } + + /// Right side of the status bar: the running job (export / render previews) with a progress + /// bar and a cancel button. + fn job_status(&mut self, ui: &mut egui::Ui, sb: egui::Rect, t: &Tokens) { + use std::sync::atomic::Ordering; + let Some(job) = self.watch_jobs(ui.ctx()) else { return }; let f = job.progress.fraction().clamp(0.0, 1.0); let left = job.progress.eta().map(panels::left_text).unwrap_or_default(); let cancel = egui::Rect::from_center_size(egui::pos2(sb.max.x - 14.0, sb.center().y), egui::vec2(14.0, 14.0)); @@ -1321,8 +1403,10 @@ impl FilmcraftApp { /// Contextual hint for the status bar (Premiere shows tool/gesture hints here). fn hint_text(&self) -> String { + let mac = cfg!(target_os = "macos"); match self.ui.tool { - state::Tool::Selection => "Click to select, or click in empty space and drag to marquee select. Use Shift, Opt, and Cmd for other options.", + state::Tool::Selection if mac => "Click to select, or click in empty space and drag to marquee select. Use Shift, Opt, and Cmd for other options.", + state::Tool::Selection => "Click to select, or click in empty space and drag to marquee select. Use Shift, Alt, and Ctrl for other options.", state::Tool::TrackSelectForward => "Click to select all clips to the right in all tracks. Shift-click for a single track.", state::Tool::TrackSelectBackward => "Click to select all clips to the left in all tracks. Shift-click for a single track.", state::Tool::Ripple => "Drag an edit point to ripple trim; later clips move to keep the gap closed.", @@ -1333,7 +1417,8 @@ impl FilmcraftApp { state::Tool::Slip => "Drag a clip to slip its source in/out without moving it.", state::Tool::Slide => "Drag a clip to slide it between its neighbours.", state::Tool::Hand => "Drag to scroll the timeline.", - state::Tool::Zoom => "Click to zoom in; Opt-click to zoom out.", + state::Tool::Zoom if mac => "Click to zoom in; Opt-click to zoom out.", + state::Tool::Zoom => "Click to zoom in; Alt-click to zoom out.", _ => "", } .to_string() diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index a02ecbf9..c787d39f 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -310,6 +310,12 @@ pub fn invoke(app: &mut FilmcraftApp, ctx: &egui::Context, id: &str, params: Val filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; return crate::panels::interchange_export::open(app, ctx, id); } + // File ▸ Export ▸ Media… opens the Export mode, like ⌘M (#382); with a path it exports directly. + "file.exportMedia" if params.get("path").is_none() => { + filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; + app.ui.mode = Mode::Export; + return Ok(json!({"mode": "export"})); + } // Audio Gain from the menu or G opens the dialog; with params it applies directly. "clip.audioGain" if params.as_object().is_none_or(|m| m.is_empty()) => { filmcraft_engine::find_command("clip.audioGain").map_or(Ok(()), |c| (c.enabled)(&app.session))?; @@ -351,6 +357,13 @@ pub fn invoke(app: &mut FilmcraftApp, ctx: &egui::Context, id: &str, params: Val app.dialog = Some(crate::Dialog::DeleteTracks); return Ok(json!({"dialog": "deleteTracks"})); } + // Sequence Settings… from the menu opens the dialog; with params the engine command applies + // them directly. + "sequence.settings" if params.as_object().is_none_or(|m| m.is_empty()) => { + filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; + crate::panels::sequence_settings::open(app); + return Ok(json!({"dialog": "sequenceSettings"})); + } "sequence.colorSettings" if params.as_object().is_none_or(|m| m.is_empty()) => { filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; crate::panels::color_dialogs::open_sequence(app); @@ -388,6 +401,7 @@ pub fn invoke(app: &mut FilmcraftApp, ctx: &egui::Context, id: &str, params: Val || (id == "file.save" && params.get("path").is_none() && app.session.path.is_none()) || (matches!(id, "captions.import" | "captions.export") && params.get("path").is_none()) || (id == "graphics.newFromFile" && params.get("path").is_none()) + || (crate::EXPORT_SAVE_DIALOGS.iter().any(|(c, ..)| *c == id) && params.get("path").is_none()) { return app.file_dialog(id, ¶ms); } @@ -502,16 +516,22 @@ pub fn shortcut_text(s: &str) -> String { Chord::parse(s).map(|c| c.display(Platform::current())).unwrap_or_else(|_| s.to_string()) } -/// Parse "Cmd+Shift+K" into modifiers + key. +/// Parse "Cmd+Shift+K" into the modifiers a chord requires + its key. Off macOS the Control key +/// is the primary modifier, so `Ctrl` and `Cmd` are one key and both mean `command` (the engine's +/// `Chord::effective` says the same): a physical Ctrl press carries `command` there, and egui's +/// matching asks only for what the pattern names, so a `Ctrl+…` chord matches a `Cmd+…` binding +/// and the other way round (#245). On a Mac they stay two keys. pub fn parse_shortcut(s: &str) -> Option<(egui::Modifiers, egui::Key)> { let mut m = egui::Modifiers::NONE; let mut key = None; let parts: Vec<&str> = if s == "+" { vec!["+"] } else { s.split('+').collect() }; + let mac = cfg!(target_os = "macos"); for p in parts { match p { "Cmd" => m.command = true, "Shift" => m.shift = true, "Alt" => m.alt = true, + "Ctrl" if !mac => m.command = true, "Ctrl" => m.ctrl = true, k => { key = match k { @@ -638,3 +658,33 @@ fn menu_entry(ui: &mut egui::Ui, it: &MenuItem) -> bool { } r.clicked() } + +#[cfg(test)] +mod parse_shortcut_tests { + use super::parse_shortcut; + + /// Off macOS `Ctrl` and `Cmd` are the same key, so either spelling parses to the modifiers a + /// physical Ctrl press carries and matches a binding written the other way (#245). + #[test] + fn ctrl_and_cmd_are_one_key_off_macos() { + let (ctrl, k) = parse_shortcut("Ctrl+Z").unwrap(); + let (cmd, k2) = parse_shortcut("Cmd+Z").unwrap(); + assert_eq!((k, k2), (egui::Key::Z, egui::Key::Z)); + if cfg!(target_os = "macos") { + assert_eq!((ctrl.ctrl, ctrl.command), (true, false)); + assert_eq!((cmd.ctrl, cmd.command), (false, true)); + assert!(!ctrl.matches_logically(cmd) && !cmd.matches_logically(ctrl), "two different keys on a Mac"); + } else { + assert_eq!((ctrl.ctrl, ctrl.command), (false, true)); + assert_eq!(ctrl, cmd, "one key off a Mac"); + // what egui-winit reports for the physical key, and a bare `command` as tests send it + let physical = egui::Modifiers { ctrl: true, command: true, ..Default::default() }; + assert!(physical.matches_logically(cmd) && physical.matches_logically(ctrl)); + assert!(egui::Modifiers::COMMAND.matches_logically(ctrl)); + } + assert!(!parse_shortcut("Z").unwrap().0.matches_logically(cmd), "a bare key never stands in for the chord"); + let (m, k) = parse_shortcut("Cmd+Shift+K").unwrap(); + assert!(m.command && m.shift && !m.alt && k == egui::Key::K); + assert_eq!(parse_shortcut("+").map(|(_, k)| k), Some(egui::Key::Plus)); + } +} diff --git a/crates/ui-egui/src/panels/clip_dialogs.rs b/crates/ui-egui/src/panels/clip_dialogs.rs index a205e1ce..2f5bba59 100644 --- a/crates/ui-egui/src/panels/clip_dialogs.rs +++ b/crates/ui-egui/src/panels/clip_dialogs.rs @@ -593,10 +593,8 @@ pub fn show(app: &mut FilmcraftApp, ctx: &egui::Context) { fn finish(d: &ClipDialogDraft) -> Value { let mut p = d.params.clone(); match d.command.as_str() { - "file.newOfflineFile" => { - if p["name"].as_str().is_some_and(str::is_empty) { - p.as_object_mut().map(|m| m.remove("name")); - } + "file.newOfflineFile" if p["name"].as_str().is_some_and(str::is_empty) => { + p.as_object_mut().map(|m| m.remove("name")); } "clip.modifyTimecode" if p["reset"].as_bool() == Some(true) => { p.as_object_mut().map(|m| m.remove("timecode")); diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index 46136e79..4fc28ffc 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -65,6 +65,12 @@ pub fn show(app: &mut FilmcraftApp, ctx: &egui::Context) { } return; } + Dialog::SequenceSettings => { + if !crate::panels::sequence_settings::show(app, ctx) && app.dialog == Some(d) { + app.dialog = None; + } + return; + } Dialog::NewSequence | Dialog::Preferences | Dialog::Recovery | Dialog::RevertConfirm => {} } if !open { diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 9931aa72..600c98f2 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -12,6 +12,21 @@ use crate::icons::{self, Icon}; use crate::theme::Tokens; const ROW_H: f32 = 22.0; +/// Keyframe lane header: the time ruler (beside the Source / Sequence pills) and the clip's bar. +const RULER_H: f32 = 24.0; +const CLIP_BAR_H: f32 = 18.0; +/// Distance between the keyframe navigator's buttons (◀ ◆ ▶). +const NAV_STEP: f32 = 16.0; +/// Properties panel: the keyframe diamonds' column, from a row's right edge (the arrows sit one +/// [`NAV_STEP`] to either side of it, and a section's reset button above it). +const PROPS_NAV_X: f32 = 26.0; +/// Properties panel: where a row's value fields end, left of the navigator. +const PROPS_VALUE_R: f32 = PROPS_NAV_X + NAV_STEP + 12.0; + +/// Timeline time at which the clip shows media time `m` (where a keyframe sits in the sequence). +fn timeline_time_of(it: &TrackItem, m: Tick) -> Tick { + it.start + Tick(((m - it.source_in).0 as f64 / it.speed.abs().max(1e-6)) as i64) +} fn selected_clip(app: &FilmcraftApp) -> Option<(ClipId, TrackItem, filmcraft_project::TrackKind)> { let seq = app.session.active_sequence()?; @@ -52,22 +67,29 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { let half = (head.width() - 6.0) / 2.0; pill(ui, Rect::from_min_size(head.min, vec2(half, 24.0)), &format!("Source · {}", it.name), false); pill(ui, Rect::from_min_size(head.min + vec2(half + 6.0, 0.0), vec2(half, 24.0)), &format!("{} · {}", seq_name(app), it.name), true); - // keyframe lane header: mini ruler over the clip's duration + // keyframe lane header, as Premiere's: a time ruler over the clip's stretch of the sequence, + // which the playhead's handle rides on, and the clip's bar under it let lane = Rect::from_min_max(pos2(split + 4.0, rect.min.y + 4.0), pos2(rect.max.x - 6.0, rect.max.y - 26.0)); ui.painter().rect_filled(lane, 0.0, t.tl_bg); let ph = app.session.playhead(); let dur = it.duration.0.max(1) as f64; let lx = |tk: Tick| -> f32 { lane.min.x + (((tk - it.start).0 as f64 / dur) as f32).clamp(0.0, 1.0) * lane.width() }; - ui.painter().rect_filled(Rect::from_min_max(pos2(lane.min.x, lane.min.y + 2.0), pos2(lane.max.x, lane.min.y + 16.0)), 2.0, Color32::from_rgb(58, 58, 70)); - ui.painter().text(pos2(lane.min.x + 4.0, lane.min.y + 9.0), Align2::LEFT_CENTER, &it.name, Tokens::ui(10.0), t.text); - let scrub = Rect::from_min_max(lane.min, pos2(lane.max.x, lane.min.y + 18.0)); + let rate = seq.settings.frame_rate; + let ruler = Rect::from_min_max(lane.min, pos2(lane.max.x, lane.min.y + RULER_H)); + paint_ruler(ui.painter(), ruler, &it, rate, seq.settings.drop_frame, &t); + let bar = Rect::from_min_max(pos2(lane.min.x, ruler.max.y + 4.0), pos2(lane.max.x, ruler.max.y + 4.0 + CLIP_BAR_H)); + ui.painter().rect_filled(bar.shrink2(vec2(0.0, 2.0)), 2.0, Color32::from_rgb(58, 58, 70)); + ui.painter().with_clip_rect(bar).text(pos2(bar.min.x + 4.0, bar.center().y), Align2::LEFT_CENTER, &it.name, Tokens::ui(10.0), t.text); + // click or drag anywhere on the ruler (or the bar) to move the playhead + let scrub = Rect::from_min_max(ruler.min, bar.max); let sresp = ui.interact(scrub, egui::Id::new(("ec-scrub", clip.0)), Sense::click_and_drag()); app.auto.add("effectControls.lane", lane, "keyframe lane"); + app.auto.add("effectControls.ruler", scrub, "time ruler"); if (sresp.dragged() || sresp.clicked()) && let Some(pos) = sresp.interact_pointer_pos() { let f = ((pos.x - lane.min.x) / lane.width()).clamp(0.0, 1.0) as f64; - let tk = it.start + Tick((f * it.duration.0 as f64) as i64); + let tk = rate.snap_nearest(it.start + Tick((f * it.duration.0 as f64) as i64)); app.stop(); app.session.set_playhead(tk); } @@ -163,9 +185,15 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { } }); let _ = scroll_out; - // playhead in lane - let px = lx(ph); - ui.painter().line_segment([pos2(px, lane.min.y), pos2(px, lane.max.y)], Stroke::new(1.0, t.playhead)); + // playhead: a handle on the ruler and a line down the lane, while it is on the clip + if ph >= it.start && ph <= it.end() { + let px = lx(ph); + let (top, tip) = (ruler.max.y - 13.0, ruler.max.y); + let head = vec![pos2(px - 5.0, top), pos2(px + 5.0, top), pos2(px + 5.0, tip - 5.0), pos2(px, tip), pos2(px - 5.0, tip - 5.0)]; + ui.painter().add(egui::Shape::convex_polygon(head, t.playhead, Stroke::NONE)); + ui.painter().line_segment([pos2(px, tip), pos2(px, lane.max.y)], Stroke::new(1.0, t.playhead)); + app.auto.add("effectControls.playhead", Rect::from_min_max(pos2(px - 5.0, top), pos2(px + 5.0, tip)), "playhead"); + } // footer timecode let tc = filmcraft_time::format_time(ph, seq.settings.frame_rate, seq.settings.drop_frame, filmcraft_time::TimeDisplay::Timecode, 48000); ui.painter().text(pos2(rect.min.x + 10.0, rect.max.y - 13.0), Align2::LEFT_CENTER, tc, Tokens::mono(13.0), t.timecode); @@ -192,6 +220,137 @@ fn seq_name(app: &FilmcraftApp) -> String { app.session.state.active_sequence.and_then(|s| app.session.project.item(s)).map(|i| i.name.clone()).unwrap_or_default() } +/// The keyframe lane's time ruler: ticks and sequence timecode across the clip's stretch of the +/// timeline (the lane shows exactly the clip, so the ruler starts at the clip's start). +fn paint_ruler(p: &egui::Painter, ruler: Rect, it: &TrackItem, rate: filmcraft_time::FrameRate, drop_frame: bool, t: &Tokens) { + let p = p.with_clip_rect(ruler); + let dur = it.duration.0.max(1) as f64; + let frame_px = ruler.width() as f64 * rate.frame_duration().0.max(1) as f64 / dur; + let base = rate.timecode_base(); + let steps = [ + 1, + 2, + 5, + 10, + base / 2, + base, + base * 2, + base * 5, + base * 10, + base * 15, + base * 30, + base * 60, + base * 120, + base * 300, + base * 600, + base * 1800, + base * 3600, + ]; + // labels far enough apart to read; the small ticks divide the labelled ones evenly + let label_step = steps.iter().copied().find(|s| *s > 0 && *s as f64 * frame_px >= 80.0); + let minor = steps.iter().copied().find(|s| *s > 0 && *s as f64 * frame_px >= 8.0 && label_step.is_none_or(|l| l % s == 0)); + let (f0, f1) = (rate.frame_at(it.start), rate.frame_at(it.end())); + let base_y = ruler.max.y - 1.0; + for (step, h, labelled) in [(minor, 3.0, false), (label_step, 7.0, true)] { + let Some(step) = step else { continue }; + let mut f = f0.div_euclid(step).saturating_mul(step); + // a step is at least 8 px wide, so this covers any ruler; the cap is for damaged numbers + for _ in 0..4096 { + if f > f1 { + break; + } + let x = ruler.min.x + ((rate.tick_of(f) - it.start).0 as f64 / dur) as f32 * ruler.width(); + p.line_segment([pos2(x, base_y - h), pos2(x, base_y)], Stroke::new(1.0, t.tl_ruler_tick)); + if labelled { + let label = filmcraft_time::format_time(rate.tick_of(f), rate, drop_frame, filmcraft_time::TimeDisplay::Timecode, 48000); + p.text(pos2(x, ruler.min.y + 7.0), Align2::CENTER_CENTER, label, Tokens::ui(10.0), t.tl_ruler_text); + } + f = f.saturating_add(step); + } + } +} + +/// Premiere's keyframe navigator, centred on `at`: a diamond that adds a keyframe at the playhead +/// or removes the one there (filled while the playhead is on a keyframe) and, once the parameter +/// is animated, arrows to the previous and the next keyframe. `param` is `None` while the effect +/// is not on the clip yet. Automation ids: `.addKeyframe`, `.prevKeyframe`, `.nextKeyframe`. +/// Returns whether the diamond was clicked; the arrows queue their own `playhead.set`. +#[allow(clippy::too_many_arguments)] +fn keyframe_nav( + app: &mut FilmcraftApp, + ui: &mut egui::Ui, + at: Pos2, + param: Option<&filmcraft_project::Param>, + mt: Tick, + it: &TrackItem, + id: egui::Id, + auto: &str, + tip: &str, + actions: &mut Vec<(String, Value)>, +) -> bool { + let t = app.tokens; + if let Some(param) = param.filter(|p| p.is_animated()) { + for (d, target, name, tip) in + [(-1.0, param.prev_keyframe(mt), "prevKeyframe", "Go to previous keyframe"), (1.0, param.next_keyframe(mt), "nextKeyframe", "Go to next keyframe")] + { + let r = Rect::from_center_size(at + vec2(d * NAV_STEP, 0.0), vec2(12.0, 14.0)); + let resp = ui.interact(r, id.with(name), Sense::click()).on_hover_text(tip); + let c = r.center(); + let pts = vec![c + vec2(-3.0 * d, -4.0), c + vec2(-3.0 * d, 4.0), c + vec2(3.0 * d, 0.0)]; + ui.painter().add(egui::Shape::convex_polygon(pts, if target.is_some() { t.text } else { t.text_faint }, Stroke::NONE)); + app.auto.add(&format!("{auto}.{name}"), r, tip); + if resp.clicked() + && let Some(k) = target + { + actions.push(("playhead.set".into(), json!({"time": timeline_time_of(it, k).0}))); + } + } + } + let r = Rect::from_center_size(at, vec2(13.0, 13.0)); + let resp = ui.interact(r.expand(1.5), id.with("key"), Sense::click()).on_hover_text(tip); + if param.is_some_and(|p| p.keyframes.iter().any(|k| k.time == mt)) { + icons::paint(ui.painter(), r, Icon::Keyframe, t.hot_text); + } else { + let s = 4.0; + let outline = vec![at + vec2(0.0, -s), at + vec2(s, 0.0), at + vec2(0.0, s), at + vec2(-s, 0.0)]; + ui.painter().add(egui::Shape::closed_line(outline, Stroke::new(1.2, if resp.hovered() { t.text } else { t.text_dim }))); + } + app.auto.add(&format!("{auto}.addKeyframe"), r, tip); + resp.clicked() +} + +/// A Properties row's keyframe navigator for `effect`.`param`, with Premiere's tooltips. The +/// diamond is `effects.addKeyframe`: it adds a keyframe at the playhead (turning animation on) or +/// removes the one there, so the value falls back to what the remaining keyframes give. An effect +/// the clip does not have yet (Crop) is applied first. +#[allow(clippy::too_many_arguments)] +fn properties_nav( + app: &mut FilmcraftApp, + ui: &mut egui::Ui, + row: Rect, + clip: ClipId, + it: &TrackItem, + mt: Tick, + effect: &str, + param: &str, + actions: &mut Vec<(String, Value)>, +) { + let prm = it.effect(effect).and_then(|e| e.param(param)); + let tip = match prm { + Some(p) if p.keyframes.iter().any(|k| k.time == mt) => "Remove keyframe", + Some(p) if p.is_animated() => "Add keyframe", + _ => "Turn on animation and add keyframe", + }; + let at = pos2(row.max.x - PROPS_NAV_X, row.center().y); + let id = egui::Id::new(("props-nav", effect, param, clip.0)); + if keyframe_nav(app, ui, at, prm, mt, it, id, &format!("properties.{effect}.{param}"), tip, actions) { + if it.effect(effect).is_none() { + actions.push(("effects.apply".into(), json!({"clips": [clip.0], "effect": effect}))); + } + actions.push(("effects.addKeyframe".into(), json!({"clip": clip.0, "effect": effect, "param": param}))); + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn param_row( app: &mut FilmcraftApp, @@ -337,41 +496,11 @@ pub(crate) fn param_row( // keyframe navigator ◀ ◆ ▶ (when animated) let eff_json = json!(idx); if pd.animatable && param.is_animated() { - let nx = r.max.x - 58.0; - let cy = r.center().y; - let at_key = param.keyframes.iter().any(|k| k.time == mt); - let prev = param.prev_keyframe(mt); - let next = param.next_keyframe(mt); - let to_tl = |k: Tick| it.start + Tick(((k - it.source_in).0 as f64 / it.speed.abs().max(1e-6)) as i64); - let pr = Rect::from_center_size(pos2(nx, cy), vec2(12.0, 14.0)); - let kr = Rect::from_center_size(pos2(nx + 16.0, cy), vec2(12.0, 12.0)); - let nr = Rect::from_center_size(pos2(nx + 32.0, cy), vec2(12.0, 14.0)); - let arrow = |p: &egui::Painter, r: Rect, left: bool, on: bool| { - let c = r.center(); - let pts = if left { - vec![c + vec2(3.0, -4.0), c + vec2(3.0, 4.0), c + vec2(-3.0, 0.0)] - } else { - vec![c + vec2(-3.0, -4.0), c + vec2(-3.0, 4.0), c + vec2(3.0, 0.0)] - }; - p.add(egui::Shape::convex_polygon(pts, if on { t.text } else { t.text_faint }, Stroke::NONE)); - }; - arrow(ui.painter(), pr, true, prev.is_some()); - arrow(ui.painter(), nr, false, next.is_some()); - icons::paint(ui.painter(), kr, Icon::Keyframe, if at_key { t.hot_text } else { t.text_dim }); - if ui.interact(pr, egui::Id::new(("kprev", clip.0, idx, pkey)), Sense::click()).clicked() - && let Some(k) = prev - { - actions.push(("playhead.set".into(), json!({"time": to_tl(k).0}))); - } - if ui.interact(nr, egui::Id::new(("knext", clip.0, idx, pkey)), Sense::click()).clicked() - && let Some(k) = next - { - actions.push(("playhead.set".into(), json!({"time": to_tl(k).0}))); - } - if ui.interact(kr, egui::Id::new(("kadd", clip.0, idx, pkey)), Sense::click()).on_hover_text("Add/Remove Keyframe").clicked() { + let at = pos2(r.max.x - 42.0, r.center().y); + let auto = format!("effectControls.{}.{}", e.effect, pkey); + if keyframe_nav(app, ui, at, Some(param), mt, it, egui::Id::new(("knav", clip.0, idx, pkey)), &auto, "Add/Remove Keyframe", actions) { actions.push(("effects.addKeyframe".into(), with_mask(json!({"clip": clip.0, "effect": eff_json, "param": pd.id})))); } - app.auto.add(&format!("effectControls.{}.{}.addKeyframe", e.effect, pkey), kr, "Add/Remove Keyframe"); } // keyframes in the lane: draggable diamonds; right-click for interpolation if param.is_animated() { @@ -563,7 +692,7 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { ); ui.painter().text(pos2(r.min.x + 18.0, r.center().y), Align2::LEFT_CENTER, name, Tokens::semibold(13.0), t.text); if let Some((idx, c)) = reset { - let rr = Rect::from_center_size(pos2(r.max.x - 10.0, r.center().y), vec2(14.0, 14.0)); + let rr = Rect::from_center_size(pos2(r.max.x - PROPS_NAV_X, r.center().y), vec2(14.0, 14.0)); icons::paint(ui.painter(), rr, Icon::Reset, t.text_dim); if ui.interact(rr, egui::Id::new(("props-reset", name, c)), Sense::click()).clicked() { actions.push(("effects.reset".into(), json!({"clip": c, "index": idx}))); @@ -578,22 +707,8 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { } open }; - let diamond = |ui: &mut egui::Ui, r: Rect, animated: bool, id: egui::Id| -> bool { - let dr = Rect::from_center_size(pos2(r.max.x - 10.0, r.center().y), vec2(10.0, 10.0)); - let resp = ui.interact(dr.expand(3.0), id, Sense::click()); - if animated { - icons::paint(ui.painter(), dr, Icon::Keyframe, t.hot_text); - } else { - let c = dr.center(); - let s = 4.5; - ui.painter() - .add(egui::Shape::closed_line(vec![c + vec2(0.0, -s), c + vec2(s, 0.0), c + vec2(0.0, s), c + vec2(-s, 0.0)], Stroke::new(1.2, t.text_dim))); - } - resp.clicked() - }; let eff_idx = |id: &str| it.effects.iter().position(|e| e.effect == id); let val = |eid: &str, p: &str| it.effect(eid).and_then(|e| e.param(p)).map(|p| p.value_at(mt)); - let anim = |eid: &str, p: &str| it.effect(eid).and_then(|e| e.param(p)).is_some_and(|p| p.is_animated()); if kind == filmcraft_project::TrackKind::Video { if section(&mut bui, app, "Transform", eff_idx("motion").map(|i| (i, clip.0)), &mut actions) { for (label, p, unit, speed, range) in [ @@ -605,7 +720,7 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = row(&mut bui, label); let mut vui = bui.new_child( egui::UiBuilder::new() - .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - 24.0, r.max.y - 4.0))) + .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - PROPS_VALUE_R, r.max.y - 4.0))) .layout(egui::Layout::left_to_right(egui::Align::Center)), ); match val("motion", p) { @@ -629,14 +744,12 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { } _ => {} } - if diamond(&mut bui, r, anim("motion", p), egui::Id::new(("pd", p, clip.0))) { - actions.push(("effects.toggleAnimation".into(), json!({"clip": clip.0, "effect": "motion", "param": p}))); - } + properties_nav(app, &mut bui, r, clip, &it, mt, "motion", p, &mut actions); } let r = row(&mut bui, "Opacity"); let mut vui = bui.new_child( egui::UiBuilder::new() - .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - 24.0, r.max.y - 4.0))) + .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - PROPS_VALUE_R, r.max.y - 4.0))) .layout(egui::Layout::left_to_right(egui::Align::Center)), ); if let Some(ParamValue::Float(v)) = val("opacity", "opacity") { @@ -645,9 +758,7 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("effects.setParam".into(), json!({"clip": clip.0, "effect": "opacity", "param": "opacity", "value": nv}))); } } - if diamond(&mut bui, r, anim("opacity", "opacity"), egui::Id::new(("pd-op", clip.0))) { - actions.push(("effects.toggleAnimation".into(), json!({"clip": clip.0, "effect": "opacity", "param": "opacity"}))); - } + properties_nav(app, &mut bui, r, clip, &it, mt, "opacity", "opacity", &mut actions); } let crop = eff_idx("crop"); if section(&mut bui, app, "Crop", crop.map(|i| (i, clip.0)), &mut actions) { @@ -655,7 +766,7 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = row(&mut bui, label); let mut vui = bui.new_child( egui::UiBuilder::new() - .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - 24.0, r.max.y - 4.0))) + .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - PROPS_VALUE_R, r.max.y - 4.0))) .layout(egui::Layout::left_to_right(egui::Align::Center)), ); let v = val("crop", p).and_then(|v| v.as_f64()).unwrap_or(0.0); @@ -666,14 +777,14 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { } actions.push(("effects.setParam".into(), json!({"clip": clip.0, "effect": "crop", "param": p, "value": nv}))); } - diamond(&mut bui, r, anim("crop", p), egui::Id::new(("pd-crop", p, clip.0))); + properties_nav(app, &mut bui, r, clip, &it, mt, "crop", p, &mut actions); } } } else if section(&mut bui, app, "Audio", eff_idx("volume").map(|i| (i, clip.0)), &mut actions) { let r = row(&mut bui, "Level"); let mut vui = bui.new_child( egui::UiBuilder::new() - .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - 24.0, r.max.y - 4.0))) + .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - PROPS_VALUE_R, r.max.y - 4.0))) .layout(egui::Layout::left_to_right(egui::Align::Center)), ); let v = val("volume", "level").and_then(|v| v.as_f64()).unwrap_or(0.0); @@ -681,13 +792,11 @@ pub fn properties_panel(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { if let Some(nv) = nv { actions.push(("effects.setParam".into(), json!({"clip": clip.0, "effect": "volume", "param": "level", "value": nv}))); } - if diamond(&mut bui, r, anim("volume", "level"), egui::Id::new(("pd-vol", clip.0))) { - actions.push(("effects.toggleAnimation".into(), json!({"clip": clip.0, "effect": "volume", "param": "level"}))); - } + properties_nav(app, &mut bui, r, clip, &it, mt, "volume", "level", &mut actions); let r = row(&mut bui, "Pan"); let mut vui = bui.new_child( egui::UiBuilder::new() - .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - 24.0, r.max.y - 4.0))) + .max_rect(Rect::from_min_max(pos2(r.min.x + 150.0, r.min.y + 4.0), pos2(r.max.x - PROPS_VALUE_R, r.max.y - 4.0))) .layout(egui::Layout::left_to_right(egui::Align::Center)), ); let v = val("panner", "balance").and_then(|v| v.as_f64()).unwrap_or(0.0); @@ -938,3 +1047,97 @@ mod drag_undo_tests { assert_eq!(opacity(&app), start, "and then the first"); } } + +#[cfg(test)] +mod lane_tests { + use super::*; + + /// A context with the app's fonts (the panels use the semibold family). + fn context() -> egui::Context { + let ctx = egui::Context::default(); + crate::theme::install(&ctx, &Tokens::for_kind(Default::default())); + ctx + } + + /// A project file is only checked for overflow, so a damaged one can carry a clip with no + /// (or a negative) duration. Selecting it must still draw both panels, ruler, playhead and + /// keyframe navigators included. + #[test] + fn a_clip_without_a_duration_still_draws() { + let mut s = filmcraft_engine::Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let clip = s.active_sequence().unwrap().video_tracks[0].items[0].id; + s.execute("timeline.select", json!({"clips": [clip.0]})).unwrap(); + s.execute("effects.toggleAnimation", json!({"clip": clip.0, "effect": "motion", "param": "scale"})).unwrap(); + let seq = s.state.active_sequence.unwrap(); + let mut app = crate::FilmcraftApp::new(s); + let ctx = context(); + for duration in [0, -5 * filmcraft_time::TICKS_PER_SECOND] { + let project = std::sync::Arc::make_mut(&mut app.session.project); + project.sequence_mut(seq).unwrap().find_item_mut(clip).unwrap().1.duration = Tick(duration); + let rect = Rect::from_min_size(pos2(0.0, 0.0), vec2(700.0, 500.0)); + let mut out = ctx.run_ui(egui::RawInput::default(), |ui| { + show(&mut app, ui, rect); + properties_panel(&mut app, ui, rect); + }); + out.textures_delta.clear(); + } + } + + /// The ruler takes its numbers from the project (clip times, the sequence's frame rate) and + /// the panel's size. Anything a project that opens can carry (times within `Tick::MIN..=MAX`, + /// a rate within `SequenceSettings::validate`, or an unset one) must not make it panic or + /// tick forever. + #[test] + fn ruler_survives_damaged_numbers() { + let mut s = filmcraft_engine::Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let mut it = s.active_sequence().unwrap().video_tracks[0].items[0].clone(); + let ctx = context(); + let t = Tokens::for_kind(Default::default()); + let big = i64::from(u32::MAX); + let rates = [(24000, 1001), (30, 1), (1000, 1), (1, 1), (1, big), (big, big), (0, 0), (-25, 1)]; + let mut out = ctx.run_ui(egui::RawInput::default(), |ui| { + for width in [300.0, 0.0, -40.0, 0.001, 1.0e9, f32::NAN, f32::INFINITY] { + for duration in [1, 0, -7, filmcraft_time::TICKS_PER_SECOND, Tick::MAX.0 / 2] { + for start in [0, Tick::MIN.0, Tick::MAX.0 / 2] { + for (num, den) in rates { + it.start = Tick(start); + it.duration = Tick(duration); + let ruler = Rect::from_min_size(pos2(10.0, 10.0), vec2(width, RULER_H)); + paint_ruler(ui.painter(), ruler, &it, filmcraft_time::FrameRate { num, den }, false, &t); + } + } + } + } + }); + out.textures_delta.clear(); + } + + /// A five-second lane 300 px wide is labelled every two seconds, in sequence timecode. + #[test] + fn ruler_labels_sequence_timecode() { + let mut s = filmcraft_engine::Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let mut it = s.active_sequence().unwrap().video_tracks[0].items[0].clone(); + let rate = filmcraft_time::FrameRate { num: 24, den: 1 }; + it.start = rate.tick_of(24 * 60); + it.duration = rate.tick_of(24 * 5); + let ctx = context(); + let t = Tokens::for_kind(Default::default()); + let mut out = ctx.run_ui(egui::RawInput::default(), |ui| { + paint_ruler(ui.painter(), Rect::from_min_size(pos2(0.0, 0.0), vec2(300.0, RULER_H)), &it, rate, false, &t); + }); + out.textures_delta.clear(); + fn texts(s: egui::Shape, out: &mut Vec) { + match s { + egui::Shape::Vec(v) => v.into_iter().for_each(|s| texts(s, out)), + egui::Shape::Text(t) => out.push(t.galley.text().to_string()), + _ => {} + } + } + let mut labels = Vec::new(); + out.shapes.into_iter().for_each(|c| texts(c.shape, &mut labels)); + assert_eq!(labels, ["00:01:00:00", "00:01:02:00", "00:01:04:00"]); + } +} diff --git a/crates/ui-egui/src/panels/export_mode.rs b/crates/ui-egui/src/panels/export_mode.rs index 23d3d7e9..811cacf5 100644 --- a/crates/ui-egui/src/panels/export_mode.rs +++ b/crates/ui-egui/src/panels/export_mode.rs @@ -129,7 +129,7 @@ pub fn apply_preset(app: &mut FilmcraftApp, name: &str) -> bool { /// Full output path of the current settings. pub fn output_path(ex: &ExportUi) -> String { let name = if ex.file_name.trim().is_empty() { "Export" } else { ex.file_name.trim() }; - let dir = expand_home(ex.location.trim()); + let dir = filmcraft_engine::export_tools::expand_home(ex.location.trim()); let file = format!("{name}.{}", ex.settings.extension()); if dir.is_empty() { file } else { std::path::Path::new(&dir).join(file).to_string_lossy().to_string() } } @@ -598,6 +598,11 @@ fn video_section(ui: &mut egui::Ui, reg: &mut Reg, s: &mut ExportSettings, t: &T } }); } + if matches!(s.format, Format::PngSequence | Format::TiffSequence) { + row(ui, t, "Alpha", |ui| { + check(ui, reg, "export.video.alpha", &mut s.alpha, "Include Alpha Channel"); + }); + } match s.video_format() { f @ (Format::H264 | Format::Hevc) => { let hevc = f == Format::Hevc; @@ -1288,10 +1293,3 @@ fn list_row(ui: &mut egui::Ui, t: &Tokens, selected: bool, text: &str, width: f3 ui.painter().text(pos2(r.min.x + 6.0, r.center().y), Align2::LEFT_CENTER, text, Tokens::ui(12.5), if selected { Color32::WHITE } else { t.text }); resp } - -fn expand_home(p: &str) -> String { - match (p.strip_prefix("~/"), std::env::var("HOME")) { - (Some(rest), Ok(h)) => format!("{h}/{rest}"), - _ => p.to_string(), - } -} diff --git a/crates/ui-egui/src/panels/masks.rs b/crates/ui-egui/src/panels/masks.rs index 86893f61..c7a95937 100644 --- a/crates/ui-egui/src/panels/masks.rs +++ b/crates/ui-egui/src/panels/masks.rs @@ -359,9 +359,11 @@ pub fn monitor_overlay(app: &mut FilmcraftApp, ui: &mut egui::Ui, pic: Rect, fra let selected = sel_v == Some(i); painter.rect_filled(vr, 0.0, if selected { col } else { Color32::WHITE }); painter.rect_stroke(vr, 0.0, Stroke::new(1.0, col), egui::StrokeKind::Middle); - let r = ui - .interact(vr.expand(3.0), egui::Id::new(("mask-vertex", i)), Sense::click_and_drag()) - .on_hover_text("Drag: move · Alt-click: smooth/corner · Cmd-click: delete"); + let r = ui.interact(vr.expand(3.0), egui::Id::new(("mask-vertex", i)), Sense::click_and_drag()).on_hover_text(if cfg!(target_os = "macos") { + "Drag: move · Alt-click: smooth/corner · Cmd-click: delete" + } else { + "Drag: move · Alt-click: smooth/corner · Ctrl-click: delete" + }); app.auto.add(&format!("program.mask.vertex.{i}"), vr, "Mask vertex"); if r.drag_started() { ui.data_mut(|d| d.insert_temp(sel_v_id, i)); diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index f5510dd4..6033fab5 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -36,6 +36,7 @@ pub mod project_views; pub mod reference; pub mod remix; pub mod scopes; +pub mod sequence_settings; pub mod settings; pub mod shortcuts_dialog; pub mod text; @@ -43,6 +44,7 @@ pub mod timecode; pub mod timeline; pub mod timeline_automation; pub mod timeline_captions; +pub mod timeline_hit; pub mod tools; pub mod trim_monitor; pub mod voiceover; diff --git a/crates/ui-egui/src/panels/monitor.rs b/crates/ui-egui/src/panels/monitor.rs index d98b9b4d..95ff8f7f 100644 --- a/crates/ui-egui/src/panels/monitor.rs +++ b/crates/ui-egui/src/panels/monitor.rs @@ -3,7 +3,9 @@ //! Frames come from the background [`FrameServer`](crate::frames::FrameServer) at the *smaller* of //! the playback resolution and the on-screen resolution (so a small monitor never pays for 4K). //! While playing we prefetch the next frames in priority order; while scrubbing the exact frame is -//! requested first and the nearest cached frame is shown until it arrives (no black flashes). +//! requested first and the nearest cached frame is shown until it arrives (no black flashes), or +//! failing that the frame asked for a refresh earlier, so the picture follows a scrub or a value +//! that is dragged instead of waiting for the mouse to rest. use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use filmcraft_project::ItemKind; @@ -175,18 +177,26 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) } else { app.frames.request(key, rate.tick_of(frame), scale, &project, 0); } + let asked = asked_before(app, prefix, key, playing); let tex_name = if channel { format!("monitor-{prefix}-{display:?}") } else { format!("monitor-{prefix}") }; - let (shown, exact) = if use_gpu { - let exact = app.frames.get_plan(&key).map(|p| (key, p)); - let is_exact = exact.is_some(); - let tex = match exact.or_else(|| app.frames.nearest_plan(key, 6)) { - Some((k, plan)) => app.gpu_present(k, &plan).map(|(id, _)| id), - None => app.gpu.as_ref().and_then(|g| g.texture), - }; - (tex, is_exact) + let (shown, on_screen) = if use_gpu { + let found = app + .frames + .get_plan(&key) + .map(|p| (key, p)) + .or_else(|| app.frames.nearest_plan(key, 6)) + .or_else(|| asked.iter().find_map(|k| app.frames.get_plan(k).map(|p| (*k, p)))); + match found { + Some((k, plan)) => (app.gpu_present(k, &plan).map(|(id, _)| id), Some(k)), + None => (app.gpu.as_ref().and_then(|g| g.texture), app.gpu.as_ref().and_then(|g| g.last_key)), + } } else { - cpu_texture(app, &ctx, &tex_name, key, display) + cpu_texture(app, &ctx, &tex_name, key, &asked, display) }; + let exact = on_screen == Some(key); + if which == Which::Program { + app.program_shown = on_screen; + } if !playing && !exact { app.monitor_inexact = true; } @@ -207,7 +217,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) app.frames.request(rkey, rate.tick_of(rf), scale, &project, 1); let rpic = fit(ra, frame_size.0 as f32, frame_size.1 as f32); ui.painter().rect_filled(rpic, 0.0, t.monitor_bg); - if let (Some(tex), _) = cpu_texture(app, &ctx, &format!("monitor-{prefix}-ref"), rkey, DisplayMode::Composite) { + if let (Some(tex), _) = cpu_texture(app, &ctx, &format!("monitor-{prefix}-ref"), rkey, &[], DisplayMode::Composite) { ui.painter().image(tex, rpic, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); } app.auto.add("program.compare.reference", rpic, "reference frame"); @@ -342,20 +352,63 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) transport(app, ui, row3, which); } -/// A monitor texture from the CPU frame path: the exact frame, else the nearest cached one, with -/// the display mode's channel mapping. Returns (texture, exact). -fn cpu_texture(app: &mut FilmcraftApp, ctx: &egui::Context, name: &str, key: FrameKey, mode: DisplayMode) -> (Option, bool) { - let upload = |app: &mut FilmcraftApp, k: FrameKey, img: &crate::frames::Rgba| { - if mode.is_channel() { app.texture_for_mapped(ctx, name, k, img, |i| monitor_view::channel_view(i, mode)) } else { app.texture_for(ctx, name, k, img) } - }; - if let Some(img) = app.frames.get(&key) { - return (Some(upload(app, key, &img)), true); +/// How many of a paused monitor's latest requests are remembered ([`asked_before`]): about a +/// second of refreshes, so a frame that took several refreshes to render is still known when it +/// is ready. +const ASKED_KEYS: usize = 64; + +/// Note that the monitor `prefix` asks for `key`, and return the frames it asked for on the passes +/// before, newest first: the stand-ins for `key` while it renders. +/// +/// A value that is dragged asks for a new revision on every pass, and a scrub for a new frame, so +/// the frame asked for is never the one that is ready; the one asked for a pass earlier usually +/// is, and showing it keeps the picture one pass behind the mouse instead of frozen until the +/// mouse rests. Playback has its own schedule and remembers nothing (what was on screen before +/// Play is no stand-in for where it stopped). +fn asked_before(app: &mut FilmcraftApp, prefix: &str, key: FrameKey, playing: bool) -> Vec { + let asked = app.monitor_asked.entry(prefix.to_string()).or_default(); + if playing { + asked.clear(); + return Vec::new(); + } + // another sequence or clip: its pictures are not this one's + asked.retain(|k| k.target == key.target && *k != key); + let before: Vec = asked.iter().rev().copied().collect(); + asked.push_back(key); + while asked.len() > ASKED_KEYS { + asked.pop_front(); + } + before +} + +/// A monitor texture from the CPU frame path: the exact frame, else the nearest cached one before +/// it, else the newest ready one of `asked` (see [`asked_before`]), with the display mode's +/// channel mapping. Returns the texture and the frame it shows. +fn cpu_texture( + app: &mut FilmcraftApp, + ctx: &egui::Context, + name: &str, + key: FrameKey, + asked: &[FrameKey], + mode: DisplayMode, +) -> (Option, Option) { + let found = app + .frames + .get(&key) + .map(|img| (key, img)) + .or_else(|| (1..=6).map(|d| FrameKey { frame: key.frame.saturating_sub(d), ..key }).find_map(|k| app.frames.get(&k).map(|img| (k, img)))) + .or_else(|| asked.iter().find_map(|k| app.frames.get(k).map(|img| (*k, img)))); + match found { + Some((k, img)) => { + let tex = if mode.is_channel() { + app.texture_for_mapped(ctx, name, k, &img, |i| monitor_view::channel_view(i, mode)) + } else { + app.texture_for(ctx, name, k, &img) + }; + (Some(tex), Some(k)) + } + None => (app.texture_existing(name).map(|(id, _)| id), app.texture_key(name)), } - let tex = match app.frames.nearest(key.target, key.frame, key.size, key.revision, 6) { - Some(img) => Some(upload(app, FrameKey { frame: key.frame - 1, ..key }, &img)), - None => app.texture_existing(name).map(|(id, _)| id), - }; - (tex, false) } pub fn quantize_scale(s: f32) -> f32 { diff --git a/crates/ui-egui/src/panels/sequence_settings.rs b/crates/ui-egui/src/panels/sequence_settings.rs new file mode 100644 index 00000000..06000805 --- /dev/null +++ b/crates/ui-egui/src/panels/sequence_settings.rs @@ -0,0 +1,480 @@ +//! Sequence ▸ Sequence Settings…, laid out as Premiere Pro's dialog: General, Color Management and +//! VR Properties tabs, Cancel / OK. +//! +//! What it changes (one `sequence.settings` command, one undo step): Timebase, Frame Size (with +//! "Scale motion effects proportionally when changing frame size", on by default), drop-frame or +//! non-drop-frame timecode for the NTSC rates, audio Channel Format and Sample Rate, Maximum Render +//! Quality, and the Color Management tab (working colour space, wide gamut, auto tone map). +//! +//! Premiere settings FilmCraft has no equivalent for yet are shown greyed out with their current +//! value and a "not supported yet" hint, rather than editable and silently ignored: Editing Mode, +//! Pixel Aspect Ratio (stored, not rendered), Fields, the Frames and Feet + Frames display +//! formats, Number of Channels, audio Display Format, the Video Previews format, codec and size, +//! Maximum Bit Depth, and the VR Properties tab. Composite in Linear Color is shown on and greyed: +//! FilmCraft always composites in linear light. +//! +//! Automation ids (`sequenceSettings.` + …): `tab.general`, `tab.color`, `tab.vr`; General: +//! `editingMode`, `timebase` (+ `timebase.option./` while open), `width`, `height`, +//! `aspect`, `scaleMotion`, `par`, `fields`, `videoDisplay` (+ `videoDisplay.option.`), +//! `channelFormat` (+ `channelFormat.option.`), `channels`, `sampleRate` +//! (+ `sampleRate.option.`), `audioDisplay`, `previewFormat`, `previewCodec`, `previewWidth`, +//! `previewHeight`, `maxBitDepth`, `maxRenderQuality`, `linearColor`; Color Management: +//! `workingSpace` (+ `workingSpace.option.`), `wideGamut`, `autoToneMap`; VR Properties: +//! `vr.projection`, `vr.layout`, `vr.horizontal`, `vr.vertical`; `ok`, `cancel`. + +use filmcraft_color::WorkingSpace; +use filmcraft_project::{AudioChannels, SequenceSettings}; +use filmcraft_time::FrameRate; +use serde_json::{Map, Value, json}; + +use crate::FilmcraftApp; +use crate::state::SequenceSettingsDraft; + +const NOT_YET: &str = "Not supported in FilmCraft yet"; +const LINEAR: &str = "FilmCraft always composites in linear light"; + +/// The Timebase list (frames per second as num/den), Premiere's common rates. +const TIMEBASES: [(i64, i64); 15] = [ + (10, 1), + (12, 1), + (25, 2), + (15, 1), + (24_000, 1001), + (24, 1), + (25, 1), + (30_000, 1001), + (30, 1), + (48, 1), + (50, 1), + (60_000, 1001), + (60, 1), + (120_000, 1001), + (120, 1), +]; +const SAMPLE_RATES: [u32; 5] = [32_000, 44_100, 48_000, 88_200, 96_000]; +const MIXES: [&str; 4] = ["Stereo", "Mono", "5.1", "Adaptive"]; + +fn mix_name(c: AudioChannels) -> &'static str { + match c { + AudioChannels::Mono => "Mono", + AudioChannels::Stereo => "Stereo", + AudioChannels::Surround51 => "5.1", + AudioChannels::Adaptive => "Adaptive", + } +} + +/// Fill the draft from the active sequence and open the dialog on the General tab. +pub fn open(app: &mut FilmcraftApp) { + let Some(q) = app.session.active_sequence() else { return }; + let st = &q.settings; + app.ui.sequence_settings = SequenceSettingsDraft { + tab: "general".into(), + fps_num: st.frame_rate.num, + fps_den: st.frame_rate.den, + width: st.width, + height: st.height, + scale_motion: true, + drop_frame: st.drop_frame && st.frame_rate.supports_drop_frame(), + mix: mix_name(st.audio_master).into(), + sample_rate: st.sample_rate, + max_render_quality: st.max_render_quality, + working_space: st.color.working.id().into(), + wide_gamut: st.color.wide_gamut, + auto_tone_map: st.color.auto_tone_map, + }; + app.dialog = Some(crate::Dialog::SequenceSettings); +} + +fn rate(d: &SequenceSettingsDraft) -> FrameRate { + FrameRate { num: d.fps_num, den: d.fps_den }.sane() +} + +/// "16:9" for a frame size, or "1.90:1" when the reduced ratio has no small integer form. +fn aspect(w: u32, h: u32) -> String { + let g = gcd(w, h).max(1); + let (a, b) = (w / g, h / g); + if a <= 64 && b <= 64 { + format!("{a}:{b}") + } else if h > 0 { + format!("{:.2}:1", f64::from(w) / f64::from(h)) + } else { + String::new() + } +} + +fn gcd(mut a: u32, mut b: u32) -> u32 { + while b != 0 { + (a, b) = (b, a % b); + } + a +} + +fn timebase_label(r: FrameRate) -> String { + format!("{} frames/second", r.label()) +} + +/// The Video Display Format choices for a timebase: drop-frame / non-drop-frame timecode for the +/// NTSC rates, plain timecode otherwise (`(id, label)`). +fn timecode_formats(r: FrameRate) -> Vec<(&'static str, String)> { + if r.supports_drop_frame() { + vec![("df", format!("{} fps Drop-Frame Timecode", r.label())), ("ndf", format!("{} fps Non Drop-Frame Timecode", r.label()))] + } else { + vec![("tc", format!("{} fps Timecode", r.label()))] + } +} + +fn par_label(par: (u32, u32)) -> String { + if par.0 == par.1 { + "Square Pixels (1.0)".into() + } else if par.1 > 0 { + format!("Custom ({:.4})", f64::from(par.0) / f64::from(par.1)) + } else { + "Custom".into() + } +} + +/// The parameters for `sequence.settings` that differ from `cur`; empty when nothing changed. +fn changes(d: &SequenceSettingsDraft, cur: &SequenceSettings) -> Map { + let mut p = Map::new(); + let r = rate(d); + if r != cur.frame_rate { + p.insert("fps".into(), json!(r.as_f64())); + } + if (d.width, d.height) != (cur.width, cur.height) { + p.insert("width".into(), json!(d.width)); + p.insert("height".into(), json!(d.height)); + p.insert("scaleMotion".into(), json!(d.scale_motion)); + } + let df = d.drop_frame && r.supports_drop_frame(); + if df != (cur.drop_frame && cur.frame_rate.supports_drop_frame()) { + p.insert("dropFrame".into(), json!(df)); + } + if d.mix != mix_name(cur.audio_master) { + p.insert("mix".into(), json!(d.mix)); + } + if d.sample_rate != cur.sample_rate { + p.insert("sampleRate".into(), json!(d.sample_rate)); + } + if d.max_render_quality != cur.max_render_quality { + p.insert("maxRenderQuality".into(), json!(d.max_render_quality)); + } + if d.working_space != cur.color.working.id() { + p.insert("workingSpace".into(), json!(d.working_space)); + } + if d.wide_gamut != cur.color.wide_gamut { + p.insert("wideGamut".into(), json!(d.wide_gamut)); + } + if d.auto_tone_map != cur.color.auto_tone_map { + p.insert("autoToneMap".into(), json!(d.auto_tone_map)); + } + p +} + +type Elems = Vec<(String, egui::Rect, String)>; + +/// One labelled row: the label right-aligned in a `label_w` column, then the control. +fn row(ui: &mut egui::Ui, label_w: f32, label: &str, enabled: bool, add: impl FnOnce(&mut egui::Ui) -> R) -> R { + ui.horizontal(|ui| { + let h = ui.spacing().interact_size.y; + ui.allocate_ui_with_layout(egui::vec2(label_w, h), egui::Layout::right_to_left(egui::Align::Center), |ui| { + ui.add_enabled(enabled, egui::Label::new(label)); + }); + add(ui) + }) + .inner +} + +/// A greyed-out list showing `value`, with the "not supported yet" hint (or `hint`). +fn fixed_combo(ui: &mut egui::Ui, elems: &mut Elems, id: &str, value: &str, width: f32, hint: &str) { + let r = ui + .add_enabled_ui(false, |ui| egui::ComboBox::from_id_salt(("seq-settings-fixed", id)).selected_text(value).width(width).show_ui(ui, |_| {}).response) + .inner + .on_disabled_hover_text(hint); + elems.push((format!("sequenceSettings.{id}"), r.rect, value.to_string())); +} + +/// A greyed-out read-only value (a number field in Premiere), with the hint. +fn fixed_value(ui: &mut egui::Ui, elems: &mut Elems, id: &str, value: &str) { + let r = ui.add_enabled(false, egui::Button::new(value).min_size(egui::vec2(64.0, 0.0))).on_disabled_hover_text(NOT_YET); + elems.push((format!("sequenceSettings.{id}"), r.rect, value.to_string())); +} + +fn section(ui: &mut egui::Ui, title: &str, add: impl FnOnce(&mut egui::Ui)) { + ui.add_space(6.0); + ui.label(egui::RichText::new(title).strong()); + ui.group(|ui| { + ui.set_width(ui.available_width()); + add(ui); + }); +} + +fn general(ui: &mut egui::Ui, d: &mut SequenceSettingsDraft, cur: &SequenceSettings, audio_samples: bool, elems: &mut Elems) { + let font = egui::TextStyle::Body.resolve(ui.style()); + let label_w = ["Preview File Format:", "Pixel Aspect Ratio:", "Number of Channels:"] + .iter() + .map(|l| ui.painter().layout_no_wrap(l.to_string(), font.clone(), egui::Color32::WHITE).size().x) + .fold(0.0, f32::max); + let list_w = 260.0; + row(ui, label_w, "Editing Mode:", false, |ui| fixed_combo(ui, elems, "editingMode", "Custom", list_w, NOT_YET)); + row(ui, label_w, "Timebase:", true, |ui| { + let r = rate(d); + let mut choices: Vec<(i64, i64)> = TIMEBASES.to_vec(); + if !choices.contains(&(r.num, r.den)) { + choices.insert(0, (r.num, r.den)); + } + let shown = timebase_label(r); + // no row limit of our own: the list grows to egui's popup limit (`Spacing::default_area_size`, + // 400 px), so every rate up to 119.88 shows without scrolling (the default 200 px stopped + // before 29.97) + let resp = egui::ComboBox::from_id_salt("seq-settings-timebase").selected_text(&shown).width(list_w).height(f32::INFINITY).show_ui(ui, |ui| { + for (num, den) in choices { + let label = timebase_label(FrameRate { num, den }); + let o = ui.selectable_label((d.fps_num, d.fps_den) == (num, den), &label); + // a new timebase starts on its usual timecode (drop-frame for the NTSC rates); + // picking the current one again keeps the Display Format chosen for it + if o.clicked() && (d.fps_num, d.fps_den) != (num, den) { + d.fps_num = num; + d.fps_den = den; + d.drop_frame = FrameRate { num, den }.supports_drop_frame(); + } + elems.push((format!("sequenceSettings.timebase.option.{num}/{den}"), o.rect, label)); + } + }); + elems.push(("sequenceSettings.timebase".into(), resp.response.rect, shown)); + }); + section(ui, "Video", |ui| { + row(ui, label_w, "Frame Size:", true, |ui| { + let w = ui.add(egui::DragValue::new(&mut d.width).range(1..=filmcraft_project::MAX_FRAME_SIDE).speed(1.0)); + elems.push(("sequenceSettings.width".into(), w.rect, d.width.to_string())); + ui.label("horizontal"); + let h = ui.add(egui::DragValue::new(&mut d.height).range(1..=filmcraft_project::MAX_FRAME_SIDE).speed(1.0)); + elems.push(("sequenceSettings.height".into(), h.rect, d.height.to_string())); + ui.label("vertical"); + let a = aspect(d.width, d.height); + let ar = ui.label(&a); + elems.push(("sequenceSettings.aspect".into(), ar.rect, a)); + }); + let r = ui.checkbox(&mut d.scale_motion, "Scale motion effects proportionally when changing frame size"); + elems.push(("sequenceSettings.scaleMotion".into(), r.rect, d.scale_motion.to_string())); + row(ui, label_w, "Pixel Aspect Ratio:", false, |ui| fixed_combo(ui, elems, "par", &par_label(cur.par), list_w, NOT_YET)); + row(ui, label_w, "Fields:", false, |ui| fixed_combo(ui, elems, "fields", "No Fields (Progressive Scan)", list_w, NOT_YET)); + row(ui, label_w, "Display Format:", true, |ui| { + let r = rate(d); + let formats = timecode_formats(r); + let chosen = if r.supports_drop_frame() { if d.drop_frame { "df" } else { "ndf" } } else { "tc" }; + let shown = formats.iter().find(|(id, _)| *id == chosen).map(|(_, l)| l.clone()).unwrap_or_default(); + let resp = egui::ComboBox::from_id_salt("seq-settings-video-display").selected_text(&shown).width(list_w).show_ui(ui, |ui| { + for (id, label) in &formats { + let o = ui.selectable_label(*id == chosen, label); + if o.clicked() { + d.drop_frame = *id == "df"; + } + elems.push((format!("sequenceSettings.videoDisplay.option.{id}"), o.rect, label.clone())); + } + for label in ["Feet + Frames 16mm", "Feet + Frames 35mm", "Frames"] { + ui.add_enabled(false, egui::Button::selectable(false, label)).on_disabled_hover_text(NOT_YET); + } + }); + elems.push(("sequenceSettings.videoDisplay".into(), resp.response.rect, shown)); + }); + }); + section(ui, "Audio", |ui| { + row(ui, label_w, "Channel Format:", true, |ui| { + let shown = d.mix.clone(); + let resp = egui::ComboBox::from_id_salt("seq-settings-mix").selected_text(&shown).width(120.0).show_ui(ui, |ui| { + for m in MIXES { + let o = ui.selectable_value(&mut d.mix, m.to_string(), m); + elems.push((format!("sequenceSettings.channelFormat.option.{m}"), o.rect, m.to_string())); + } + }); + elems.push(("sequenceSettings.channelFormat".into(), resp.response.rect, shown)); + ui.add_enabled(false, egui::Label::new("Number of Channels:")); + let n = match d.mix.as_str() { + "Mono" => "1", + "5.1" => "6", + "Adaptive" => "—", + _ => "2", + }; + fixed_combo(ui, elems, "channels", n, 60.0, NOT_YET); + }); + row(ui, label_w, "Sample Rate:", true, |ui| { + let mut rates = SAMPLE_RATES.to_vec(); + if !rates.contains(&d.sample_rate) { + rates.insert(0, d.sample_rate); + } + let shown = format!("{} Hz", d.sample_rate); + let resp = egui::ComboBox::from_id_salt("seq-settings-sample-rate").selected_text(&shown).width(list_w).show_ui(ui, |ui| { + for hz in rates { + let label = format!("{hz} Hz"); + let o = ui.selectable_value(&mut d.sample_rate, hz, &label); + elems.push((format!("sequenceSettings.sampleRate.option.{hz}"), o.rect, label)); + } + }); + elems.push(("sequenceSettings.sampleRate".into(), resp.response.rect, shown)); + }); + let shown = if audio_samples { "Audio Samples" } else { "Milliseconds" }; + row(ui, label_w, "Display Format:", false, |ui| fixed_combo(ui, elems, "audioDisplay", shown, list_w, NOT_YET)); + }); + section(ui, "Video Previews", |ui| { + row(ui, label_w, "Preview File Format:", false, |ui| { + fixed_combo(ui, elems, "previewFormat", "QuickTime", list_w, NOT_YET); + ui.add_enabled(false, egui::Button::new("Configure…")).on_disabled_hover_text(NOT_YET); + }); + row(ui, label_w, "Codec:", false, |ui| fixed_combo(ui, elems, "previewCodec", &cur.preview_codec, list_w, NOT_YET)); + row(ui, label_w, "Width:", false, |ui| fixed_value(ui, elems, "previewWidth", &d.width.to_string())); + row(ui, label_w, "Height:", false, |ui| { + fixed_value(ui, elems, "previewHeight", &d.height.to_string()); + ui.add_enabled(false, egui::Button::new("Reset")).on_disabled_hover_text(NOT_YET); + }); + ui.horizontal(|ui| { + let mut bit_depth = cur.max_bit_depth; + let r = ui.add_enabled(false, egui::Checkbox::new(&mut bit_depth, "Maximum Bit Depth")).on_disabled_hover_text(NOT_YET); + elems.push(("sequenceSettings.maxBitDepth".into(), r.rect, bit_depth.to_string())); + let r = ui.checkbox(&mut d.max_render_quality, "Maximum Render Quality"); + elems.push(("sequenceSettings.maxRenderQuality".into(), r.rect, d.max_render_quality.to_string())); + }); + let mut linear = true; + let r = ui.add_enabled(false, egui::Checkbox::new(&mut linear, "Composite in Linear Color")).on_disabled_hover_text(LINEAR); + elems.push(("sequenceSettings.linearColor".into(), r.rect, "true".into())); + }); +} + +fn color(ui: &mut egui::Ui, d: &mut SequenceSettingsDraft, elems: &mut Elems) { + let font = egui::TextStyle::Body.resolve(ui.style()); + let label_w = ui.painter().layout_no_wrap("Working Color Space:".to_string(), font, egui::Color32::WHITE).size().x; + row(ui, label_w, "Working Color Space:", true, |ui| { + let shown = WorkingSpace::parse(&d.working_space).map_or(d.working_space.clone(), |w| w.label().to_string()); + let resp = egui::ComboBox::from_id_salt("seq-settings-working-space").selected_text(&shown).width(220.0).show_ui(ui, |ui| { + for w in WorkingSpace::ALL { + let o = ui.selectable_value(&mut d.working_space, w.id().to_string(), w.label()); + elems.push((format!("sequenceSettings.workingSpace.option.{}", w.id()), o.rect, w.label().to_string())); + } + }); + elems.push(("sequenceSettings.workingSpace".into(), resp.response.rect, shown)); + }); + ui.add_space(4.0); + let r = ui.checkbox(&mut d.wide_gamut, "Wide gamut color (composite in BT.2020)"); + elems.push(("sequenceSettings.wideGamut".into(), r.rect, d.wide_gamut.to_string())); + let r = ui.checkbox(&mut d.auto_tone_map, "Auto Tone Map Media (HDR and log into SDR)"); + elems.push(("sequenceSettings.autoToneMap".into(), r.rect, d.auto_tone_map.to_string())); +} + +fn vr(ui: &mut egui::Ui, elems: &mut Elems) { + let font = egui::TextStyle::Body.resolve(ui.style()); + let label_w = ui.painter().layout_no_wrap("Captured View:".to_string(), font, egui::Color32::WHITE).size().x; + row(ui, label_w, "Projection:", false, |ui| fixed_combo(ui, elems, "vr.projection", "None", 200.0, NOT_YET)); + row(ui, label_w, "Layout:", false, |ui| fixed_combo(ui, elems, "vr.layout", "Monoscopic", 200.0, NOT_YET)); + row(ui, label_w, "Captured View:", false, |ui| { + fixed_value(ui, elems, "vr.horizontal", "0°"); + ui.add_enabled(false, egui::Label::new("Horizontal")); + fixed_value(ui, elems, "vr.vertical", "0°"); + ui.add_enabled(false, egui::Label::new("Vertical")); + }); +} + +/// Draw the dialog. Returns whether it stays open. +pub fn show(app: &mut FilmcraftApp, ctx: &egui::Context) -> bool { + let Some(q) = app.session.active_sequence() else { return false }; + let cur = q.settings.clone(); + let audio_samples = app.session.project.settings.audio_display_samples; + let accent = app.tokens.accent; + let mut d = app.ui.sequence_settings.clone(); + let mut keep = true; + let mut apply = false; + let mut elems: Elems = Vec::new(); + let max_h = ctx.content_rect().height() * 0.8; + egui::Window::new("Sequence Settings").collapsible(false).resizable(false).default_width(620.0).anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]).show( + ctx, + |ui| { + ui.set_width(620.0); + ui.horizontal(|ui| { + for (id, label) in [("general", "General"), ("color", "Color Management"), ("vr", "VR Properties")] { + let r = ui.selectable_label(d.tab == id, label); + if r.clicked() { + d.tab = id.to_string(); + } + elems.push((format!("sequenceSettings.tab.{id}"), r.rect, label.to_string())); + } + }); + ui.separator(); + egui::ScrollArea::vertical().max_height(max_h).auto_shrink([false, true]).show(ui, |ui| match d.tab.as_str() { + "color" => color(ui, &mut d, &mut elems), + "vr" => vr(ui, &mut elems), + _ => general(ui, &mut d, &cur, audio_samples, &mut elems), + }); + ui.add_space(10.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let o = ui.add(egui::Button::new(egui::RichText::new("OK").color(egui::Color32::WHITE)).fill(accent)); + elems.push(("sequenceSettings.ok".into(), o.rect, "OK".into())); + if o.clicked() { + apply = true; + } + let c = ui.button("Cancel"); + elems.push(("sequenceSettings.cancel".into(), c.rect, "Cancel".into())); + if c.clicked() { + keep = false; + } + }); + }, + ); + for (id, r, l) in elems { + app.auto.add(&id, r, &l); + } + if ctx.input(|i| i.key_pressed(egui::Key::Escape)) { + keep = false; + } + if ctx.input(|i| i.key_pressed(egui::Key::Enter)) && !ctx.egui_wants_keyboard_input() { + apply = true; + } + if apply { + let p = changes(&d, &cur); + // nothing changed: OK closes the dialog and adds no undo step + if !p.is_empty() + && let Err(e) = app.session.execute("sequence.settings", Value::Object(p)) + { + app.ui.status = e.to_string(); + } + keep = false; + } + app.ui.sequence_settings = d; + keep +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aspect_labels() { + assert_eq!(aspect(1920, 1080), "16:9"); + assert_eq!(aspect(640, 360), "16:9"); + assert_eq!(aspect(1080, 1920), "9:16"); + assert_eq!(aspect(1440, 1080), "4:3"); + assert_eq!(aspect(4096, 2160), "1.90:1"); + } + + #[test] + fn display_formats_follow_the_timebase() { + assert_eq!(timecode_formats(FrameRate::FPS_23_976), vec![("tc", "23.976 fps Timecode".to_string())]); + let ntsc = timecode_formats(FrameRate::FPS_29_97); + assert_eq!(ntsc.iter().map(|(id, _)| *id).collect::>(), vec!["df", "ndf"]); + assert_eq!(ntsc[0].1, "29.97 fps Drop-Frame Timecode"); + } + + #[test] + fn only_changed_settings_are_sent() { + let cur = SequenceSettings::default(); + let mut d = SequenceSettingsDraft::default(); + assert!(changes(&d, &cur).is_empty(), "the defaults match: nothing to send"); + d.width = 1280; + d.height = 720; + d.max_render_quality = true; + let p = changes(&d, &cur); + assert_eq!(Value::Object(p), json!({"width":1280,"height":720,"scaleMotion":true,"maxRenderQuality":true})); + // drop-frame only travels for a rate that has it + let mut d = SequenceSettingsDraft { drop_frame: true, ..Default::default() }; + assert!(changes(&d, &cur).is_empty()); + (d.fps_num, d.fps_den) = (30_000, 1001); + assert_eq!(Value::Object(changes(&d, &cur)), json!({"fps":30_000.0/1001.0,"dropFrame":true})); + } +} diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 0172243c..1bfbaedd 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -22,6 +22,9 @@ use crate::icons::{self, Icon}; use crate::state::{TimelineView, Tool}; use crate::theme::Tokens; +use super::timeline_hit::{EdgeKind, Grab, edge_geometry, grab_at}; +pub use super::timeline_hit::{Hit, ROLL_PX, hit, selection_trim_kind}; + const TOP_H: f32 = 58.0; // timecode + toolbar (left) / ruler (right) const RULER_H: f32 = 44.0; const SCROLLBAR_H: f32 = 17.0; @@ -33,6 +36,8 @@ const SNAP_PX: f32 = 9.0; #[derive(Default)] pub struct TlState { pub drag: Option, + /// What the primary button went down on in the Timeline (#259); see [`Pressed`]. + pub pressed: Option, /// Last layout (for hit-testing from the control channel). pub layout: Option, /// Snap indicator x (screen) this frame. @@ -63,6 +68,18 @@ impl TlState { } } +/// What was under the pointer when the primary button went down in the Timeline, worked out on +/// that frame (#259). egui reports a click when the button comes up, by when it has forgotten the +/// press point, and a drag only once the pointer has moved past its 6 pt drag threshold, nearly +/// the 7 px edge zone. Both act on this instead, so a press acts on what it pressed. +#[derive(Clone, Copy, Debug)] +pub struct Pressed { + pub pos: Pos2, + /// The time under the pointer. + pub tick: Tick, + pub grab: Grab, +} + #[derive(Clone, Debug)] pub enum Drag { Scrub, @@ -78,11 +95,16 @@ pub enum Drag { edge: filmcraft_edit::Edge, mode: filmcraft_edit::TrimMode, delta: Tick, + /// The time under the pointer when the button went down: the edge moves by how far the + /// pointer has moved since (#259). + from: Tick, }, Roll { left: ClipId, right: ClipId, delta: Tick, + /// As in [`Drag::Trim`]. + from: Tick, }, Slip { clip: ClipId, @@ -96,11 +118,15 @@ pub enum Drag { clip: ClipId, edge: filmcraft_edit::Edge, delta: Tick, + /// As in [`Drag::Trim`]. + from: Tick, }, /// Remix tool: drag a music clip's Out edge; on release `clip.remix` re-plans it to the new duration. Remix { clip: ClipId, delta: Tick, + /// As in [`Drag::Trim`]. + from: Tick, }, Pan { last: Pos2, @@ -1371,37 +1397,6 @@ fn vertical_scrollbar(ui: &mut egui::Ui, bar: Rect, value: &mut f32, max: f32, i let _ = t; } -/// How close (px) to a cut the Selection tool rolls instead of rippling when Settings ▸ Trim ▸ -/// "Allow Selection tool to choose Roll and Ripple trims without modifier key" is on. -pub const ROLL_PX: f32 = 2.5; - -/// The trim the Selection tool starts at an edit point: Cmd+Shift = roll, Cmd = ripple. With the -/// Trim setting on (`no_modifier`), no modifier is needed: right on the cut (when a clip is on the -/// other side) rolls, elsewhere on the edge ripples. Otherwise a plain drag is a regular trim. -pub fn selection_trim_kind(no_modifier: bool, cmd: bool, shift: bool, dist_px: f32, has_neighbour: bool) -> &'static str { - if cmd && shift { - "roll" - } else if cmd { - "ripple" - } else if no_modifier { - if has_neighbour && dist_px <= ROLL_PX { "roll" } else { "ripple" } - } else { - "trim" - } -} - -/// Distance (px) from `x` to the cut at `clip`'s `edge`, and whether a clip touches that cut on -/// the other side. -fn edge_geometry(seq: &Sequence, layout: &Layout, track: filmcraft_project::TrackId, clip: ClipId, edge: filmcraft_edit::Edge, x: f32) -> (f32, bool) { - let Some(tr) = seq.track(track) else { return (f32::MAX, false) }; - let Some(it) = tr.item(clip) else { return (f32::MAX, false) }; - let (cut, neighbour) = match edge { - filmcraft_edit::Edge::Out => (it.end(), tr.items.iter().any(|x| x.start == it.end())), - filmcraft_edit::Edge::In => (it.start, tr.items.iter().any(|x| x.end() == it.start)), - }; - ((layout.x_of(cut) - x).abs(), neighbour) -} - /// Snap `t` to nearby candidates (edits, playhead, markers, in/out). Returns snapped tick. fn snap(app: &mut FilmcraftApp, seq: &Sequence, layout: &Layout, t: Tick, exclude: &[ClipId]) -> Tick { snap_to(app, seq, layout, t, exclude, true) @@ -1450,59 +1445,21 @@ fn snap_to(app: &mut FilmcraftApp, seq: &Sequence, layout: &Layout, t: Tick, exc } } -/// What is at a screen point. -#[derive(Clone, Debug)] -pub enum Hit { - Ruler, - Clip { track: TrackId, clip: ClipId, edge: Option }, - Transition { track: TrackId, id: filmcraft_project::TransitionId }, - Empty { track: TrackId }, - None, -} - -pub fn hit(seq: &Sequence, layout: &Layout, pos: Pos2) -> Hit { - if layout.ruler.contains(pos) { - return Hit::Ruler; - } - if !layout.content.contains(pos) { - return Hit::None; - } - let Some(row) = layout.row_at(pos.y) else { return Hit::None }; - let Some(tr) = seq.track(row.track) else { return Hit::None }; - for trn in &tr.transitions { - let x0 = layout.x_of(trn.start); - let x1 = layout.x_of(trn.end()); - if pos.x >= x0 && pos.x <= x1 && pos.y > row.rect.min.y + 17.0 { - return Hit::Transition { track: row.track, id: trn.id }; - } - } - let t = layout.tick_at(pos.x); - let edge_px = 7.0f32; - // prefer edges - for it in &tr.items { - let x0 = layout.x_of(it.start); - let x1 = layout.x_of(it.end()); - let w = x1 - x0; - let e = edge_px.min(w / 3.0); - if (pos.x - x0).abs() <= e && pos.x >= x0 - e { - return Hit::Clip { track: row.track, clip: it.id, edge: Some(filmcraft_edit::Edge::In) }; - } - if (pos.x - x1).abs() <= e && pos.x <= x1 + e { - return Hit::Clip { track: row.track, clip: it.id, edge: Some(filmcraft_edit::Edge::Out) }; - } - } - if let Some(it) = tr.item_at(t) { - return Hit::Clip { track: row.track, clip: it.id, edge: None }; - } - Hit::Empty { track: row.track } -} - -pub fn hit_json(app: &FilmcraftApp, pos: Pos2) -> Value { +/// What the Timeline shows at `pos` for the control channel's `ui.timeline.hit`, with `kind`: the +/// trim a press of the current tool with `mods` starts there (null when it would not trim). +pub fn hit_json(app: &FilmcraftApp, pos: Pos2, mods: egui::Modifiers) -> Value { let (Some(layout), Some(seq)) = (app.tl.layout.as_ref(), app.session.active_sequence()) else { return json!({"hit": "none"}) }; let t = layout.tick_at(pos.x); - match hit(seq, layout, pos) { + let grab = grab_at(seq, layout, pos, app.ui.tool, mods, app.session.prefs.trim.selection_tool_roll_ripple); + let kind = match grab { + Grab::Edge { kind, .. } => json!(kind.name()), + Grab::Other(_) => Value::Null, + }; + match grab.hit() { Hit::Ruler => json!({"hit": "ruler", "time": t.0}), - Hit::Clip { track, clip, edge } => json!({"hit": "clip", "track": track.0, "clip": clip.0, "edge": edge.map(|e| format!("{e:?}")), "time": t.0}), + Hit::Clip { track, clip, edge } => { + json!({"hit": "clip", "track": track.0, "clip": clip.0, "edge": edge.map(|e| format!("{e:?}")), "kind": kind, "time": t.0}) + } Hit::Transition { track, id } => json!({"hit": "transition", "track": track.0, "transition": id.0}), Hit::Empty { track } => json!({"hit": "empty", "track": track.0, "time": t.0}), Hit::None => json!({"hit": "none"}), @@ -1535,12 +1492,12 @@ fn preview_drag(app: &FilmcraftApp, seq: &Sequence, _layout: &Layout, out: &mut } } } - Drag::Remix { clip, delta } => { + Drag::Remix { clip, delta, .. } => { if let Some((_, it)) = seq.find_item(*clip) { out.insert(*clip, (it.start, it.duration + *delta, None)); } } - Drag::Trim { clip, edge, delta, .. } | Drag::Stretch { clip, edge, delta } => { + Drag::Trim { clip, edge, delta, .. } | Drag::Stretch { clip, edge, delta, .. } => { let ids = filmcraft_engine::commands::with_links(&app.session, &[*clip]); for c in ids { if let Some((_, it)) = seq.find_item(c) { @@ -1552,7 +1509,7 @@ fn preview_drag(app: &FilmcraftApp, seq: &Sequence, _layout: &Layout, out: &mut } } } - Drag::Roll { left, right, delta } => { + Drag::Roll { left, right, delta, .. } => { if let Some((_, l)) = seq.find_item(*left) { out.insert(*left, (l.start, l.duration + *delta, None)); } @@ -1712,6 +1669,18 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & let mods = ctx.input(|i| i.modifiers); let tool = app.ui.tool; let rate = seq.settings.frame_rate; + let roll_ripple = app.session.prefs.trim.selection_tool_roll_ripple; + + // ---- what the button went down on (#259): kept from the press frame until it comes up + if ctx.input(|i| !i.pointer.any_down() && !i.pointer.any_released()) { + app.tl.pressed = None; + } + if resp.is_pointer_button_down_on() + && ctx.input(|i| i.pointer.primary_pressed()) + && let Some(p) = ctx.input(|i| i.pointer.press_origin()) + { + app.tl.pressed = Some(Pressed { pos: p, tick: layout.tick_at(p.x), grab: grab_at(seq, layout, p, tool, mods, roll_ripple) }); + } // ---- wheel, as in Premiere Pro on macOS (checked in 26.5.2): // wheel the tracks under the pointer, up and down @@ -1753,26 +1722,28 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } } - // ---- cursor feedback + // ---- cursor feedback: what a press here would grab (#259) if app.tl.drag.is_none() && let Some(p) = pos && area.contains(p) { - let h = hit(seq, layout, p); - let cur = match (tool, &h) { - (Tool::Selection, Hit::Clip { edge: Some(_), .. }) => CursorIcon::ResizeColumn, - (Tool::Ripple | Tool::Rolling | Tool::RateStretch, Hit::Clip { edge: Some(_), .. }) => CursorIcon::ResizeColumn, - (Tool::Remix, Hit::Clip { edge: Some(filmcraft_edit::Edge::Out), .. }) => CursorIcon::ResizeColumn, - (Tool::Razor, Hit::Clip { .. }) => CursorIcon::Crosshair, - (Tool::Slip | Tool::Slide, Hit::Clip { .. }) => CursorIcon::ResizeHorizontal, + let g = grab_at(seq, layout, p, tool, mods, roll_ripple); + let cur = match (tool, g) { + (_, Grab::Edge { .. }) => CursorIcon::ResizeColumn, + (Tool::Razor, Grab::Other(Hit::Clip { .. })) => CursorIcon::Crosshair, + (Tool::Slip | Tool::Slide, Grab::Other(Hit::Clip { .. })) => CursorIcon::ResizeHorizontal, (Tool::Hand, _) => CursorIcon::Grab, (Tool::Zoom, _) => CursorIcon::ZoomIn, _ => CursorIcon::Default, }; ctx.set_cursor_icon(cur); + // the edge a press here would grab, and the trim it starts (#259) + if let Grab::Edge { track, clip, edge, kind } = g { + super::trim_monitor::paint_hover_bracket(ui.painter(), seq, layout, track, clip, edge, kind); + } // razor preview line if tool == Tool::Razor - && let Hit::Clip { track, .. } = h + && let Grab::Other(Hit::Clip { track, .. }) = g && let Some(row) = layout.rows.iter().find(|r| r.track == track) { let t = snap(app, seq, layout, rate.snap_nearest(layout.tick_at(p.x)), &[]); @@ -1781,13 +1752,19 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } } - // ---- press + // ---- press: act on what was under the pointer when the button went down (#259) if resp.drag_started() || (resp.clicked() && app.tl.drag.is_none()) { - let Some(p) = resp.interact_pointer_pos() else { return }; - let h = hit(seq, layout, p); - let t = layout.tick_at(p.x); - let started = match (tool, h.clone()) { - (_, Hit::Ruler) => Some(Drag::Scrub), + let pressed = match app.tl.pressed.take() { + Some(pr) => pr, + // a press and release in the same frame is never stored: it happened where egui says + None => { + let Some(p) = resp.interact_pointer_pos() else { return }; + Pressed { pos: p, tick: layout.tick_at(p.x), grab: grab_at(seq, layout, p, tool, mods, roll_ripple) } + } + }; + let (p, t) = (pressed.pos, pressed.tick); + let started = match (tool, pressed.grab) { + (_, Grab::Other(Hit::Ruler)) => Some(Drag::Scrub), (Tool::Hand, _) => Some(Drag::Pan { last: p }), (Tool::Zoom, _) => { if resp.clicked() { @@ -1797,7 +1774,7 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } None } - (Tool::Razor, Hit::Clip { clip, .. }) => { + (Tool::Razor, Grab::Other(Hit::Clip { clip, .. })) => { if resp.clicked() || resp.drag_started() { let tt = snap(app, seq, layout, rate.snap_nearest(t), &[]); let r = if mods.shift { @@ -1811,7 +1788,7 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } None } - (Tool::TrackSelectForward | Tool::TrackSelectBackward, Hit::Clip { track, .. } | Hit::Empty { track }) => { + (Tool::TrackSelectForward | Tool::TrackSelectBackward, Grab::Other(Hit::Clip { track, .. } | Hit::Empty { track })) => { let fwd = tool == Tool::TrackSelectForward; let ids: Vec = seq .all_tracks() @@ -1823,14 +1800,14 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & let _ = app.session.execute("timeline.select", json!({"clips": ids})); None } - (Tool::Selection | Tool::Ripple | Tool::Rolling, Hit::Clip { clip, edge: Some(edge), track }) if resp.clicked() => { + (Tool::Selection | Tool::Ripple | Tool::Rolling, Grab::Edge { clip, edge, track, .. }) if resp.clicked() => { // click an edge = select it as an edit point (trim mode); Shift adds let kind = match tool { Tool::Rolling => "roll", Tool::Ripple => "ripple", _ => { let (dist, neighbour) = edge_geometry(seq, layout, track, clip, edge, p.x); - selection_trim_kind(app.session.prefs.trim.selection_tool_roll_ripple, mods.command, mods.shift, dist, neighbour) + selection_trim_kind(roll_ripple, mods.command, mods.shift, dist, neighbour) } }; let e = if edge == filmcraft_edit::Edge::In { "in" } else { "out" }; @@ -1840,36 +1817,16 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } None } - (Tool::Selection | Tool::Ripple | Tool::Rolling | Tool::RateStretch, Hit::Clip { clip, edge: Some(edge), track }) => { - let Some(tr) = seq.track(track) else { return }; - let sel_kind = if tool == Tool::Selection { - let (dist, neighbour) = edge_geometry(seq, layout, track, clip, edge, p.x); - selection_trim_kind(app.session.prefs.trim.selection_tool_roll_ripple, mods.command, mods.shift, dist, neighbour) - } else { - "" - }; - let mode = if tool == Tool::Ripple || sel_kind == "ripple" { filmcraft_edit::TrimMode::Ripple } else { filmcraft_edit::TrimMode::Regular }; - if tool == Tool::Rolling || sel_kind == "roll" { - // roll the cut between this and its neighbour - let Some(it) = tr.item(clip) else { return }; - let (l, r) = match edge { - filmcraft_edit::Edge::Out => (Some(clip), tr.items.iter().find(|x| x.start == it.end()).map(|x| x.id)), - filmcraft_edit::Edge::In => (tr.items.iter().find(|x| x.end() == it.start).map(|x| x.id), Some(clip)), - }; - match (l, r) { - (Some(left), Some(right)) => Some(Drag::Roll { left, right, delta: Tick::ZERO }), - _ => Some(Drag::Trim { clip, edge, mode, delta: Tick::ZERO }), - } - } else if tool == Tool::RateStretch { - Some(Drag::Stretch { clip, edge, delta: Tick::ZERO }) - } else { - Some(Drag::Trim { clip, edge, mode, delta: Tick::ZERO }) - } - } - (Tool::Remix, Hit::Clip { clip, edge: Some(filmcraft_edit::Edge::Out), .. }) => Some(Drag::Remix { clip, delta: Tick::ZERO }), - (Tool::Slip, Hit::Clip { clip, .. }) => Some(Drag::Slip { clip, delta: Tick::ZERO }), - (Tool::Slide, Hit::Clip { clip, .. }) => Some(Drag::Slide { clip, delta: Tick::ZERO }), - (_, Hit::Clip { clip, track, .. }) => { + (_, Grab::Edge { clip, edge, kind, .. }) => Some(match kind { + EdgeKind::Trim => Drag::Trim { clip, edge, mode: filmcraft_edit::TrimMode::Regular, delta: Tick::ZERO, from: t }, + EdgeKind::Ripple => Drag::Trim { clip, edge, mode: filmcraft_edit::TrimMode::Ripple, delta: Tick::ZERO, from: t }, + EdgeKind::Roll { left, right } => Drag::Roll { left, right, delta: Tick::ZERO, from: t }, + EdgeKind::Stretch => Drag::Stretch { clip, edge, delta: Tick::ZERO, from: t }, + EdgeKind::Remix => Drag::Remix { clip, delta: Tick::ZERO, from: t }, + }), + (Tool::Slip, Grab::Other(Hit::Clip { clip, .. })) => Some(Drag::Slip { clip, delta: Tick::ZERO }), + (Tool::Slide, Grab::Other(Hit::Clip { clip, .. })) => Some(Drag::Slide { clip, delta: Tick::ZERO }), + (_, Grab::Other(Hit::Clip { clip, track, .. })) => { // select (shift toggles; alt selects one side of a link) let sel = &app.session.state.selection; if mods.shift { @@ -1888,8 +1845,8 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & None } } - (_, Hit::Transition { .. }) => None, - (_, Hit::Empty { .. }) => { + (_, Grab::Other(Hit::Transition { .. })) => None, + (_, Grab::Other(Hit::Empty { .. })) => { if resp.drag_started() { Some(Drag::Marquee { start: p }) } else { @@ -1898,7 +1855,7 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & None } } - (_, Hit::None) => None, + (_, Grab::Other(Hit::None)) => None, }; if let Some(d) = started { if matches!(d, Drag::Scrub) { @@ -1955,24 +1912,23 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & }; Some(Drag::Move { clips, grab_tick, start_track, offset, track_delta }) } - Drag::Trim { clip, edge, mode, .. } => seq.find_item(clip).map(|(_, it)| { + Drag::Trim { clip, edge, mode, from, .. } => seq.find_item(clip).map(|(_, it)| { let base = if edge == filmcraft_edit::Edge::In { it.start } else { it.end() }; - let target = snap(app, seq, layout, rate.snap_nearest(t_here), &[clip]); - let delta = target - base; - Drag::Trim { clip, edge, mode, delta } + let target = snap(app, seq, layout, rate.snap_nearest(base + (t_here - from)), &[clip]); + Drag::Trim { clip, edge, mode, delta: target - base, from } }), - Drag::Remix { clip, .. } => seq.find_item(clip).map(|(_, it)| { - let target = rate.snap_nearest(t_here).max(it.start + rate.frame_duration()); - Drag::Remix { clip, delta: target - it.end() } + Drag::Remix { clip, from, .. } => seq.find_item(clip).map(|(_, it)| { + let target = rate.snap_nearest(it.end() + (t_here - from)).max(it.start + rate.frame_duration()); + Drag::Remix { clip, delta: target - it.end(), from } }), - Drag::Stretch { clip, edge, .. } => seq.find_item(clip).map(|(_, it)| { + Drag::Stretch { clip, edge, from, .. } => seq.find_item(clip).map(|(_, it)| { let base = if edge == filmcraft_edit::Edge::In { it.start } else { it.end() }; - let target = snap(app, seq, layout, rate.snap_nearest(t_here), &[clip]); - Drag::Stretch { clip, edge, delta: target - base } + let target = snap(app, seq, layout, rate.snap_nearest(base + (t_here - from)), &[clip]); + Drag::Stretch { clip, edge, delta: target - base, from } }), - Drag::Roll { left, right, .. } => seq.find_item(left).map(|(_, l)| { - let target = snap(app, seq, layout, rate.snap_nearest(t_here), &[left, right]); - Drag::Roll { left, right, delta: target - l.end() } + Drag::Roll { left, right, from, .. } => seq.find_item(left).map(|(_, l)| { + let target = snap(app, seq, layout, rate.snap_nearest(l.end() + (t_here - from)), &[left, right]); + Drag::Roll { left, right, delta: target - l.end(), from } }), Drag::Slip { clip, .. } => { let start = resp.interact_pointer_pos().map(|_| ()).and(ctx.input(|i| i.pointer.press_origin())); @@ -2017,16 +1973,16 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & .collect(); Some(app.session.execute("timeline.move", json!({"moves": moves, "insert": mods.command, "linked": false}))) } - Drag::Trim { clip, edge, mode, delta } if delta != Tick::ZERO => Some(app.session.execute( + Drag::Trim { clip, edge, mode, delta, .. } if delta != Tick::ZERO => Some(app.session.execute( "timeline.trim", json!({"clip": clip.0, "edge": if edge == filmcraft_edit::Edge::In {"in"} else {"out"}, "mode": if mode == filmcraft_edit::TrimMode::Ripple {"ripple"} else {"regular"}, "delta": delta.0}), )), - Drag::Stretch { clip, edge, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.rateStretch", json!({"clip": clip.0, "edge": if edge == filmcraft_edit::Edge::In {"in"} else {"out"}, "delta": delta.0}))), - Drag::Remix { clip, delta } if delta != Tick::ZERO => { + Drag::Stretch { clip, edge, delta, .. } if delta != Tick::ZERO => Some(app.session.execute("timeline.rateStretch", json!({"clip": clip.0, "edge": if edge == filmcraft_edit::Edge::In {"in"} else {"out"}, "delta": delta.0}))), + Drag::Remix { clip, delta, .. } if delta != Tick::ZERO => { let d = seq.find_item(clip).map(|(_, it)| it.duration + delta).unwrap_or(Tick::ZERO); Some(app.session.execute("clip.remix", json!({"clip": clip.0, "duration": d.0}))) } - Drag::Roll { left, right, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.roll", json!({"left": left.0, "right": right.0, "delta": delta.0}))), + Drag::Roll { left, right, delta, .. } if delta != Tick::ZERO => Some(app.session.execute("timeline.roll", json!({"left": left.0, "right": right.0, "delta": delta.0}))), Drag::Slip { clip, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.slip", json!({"clip": clip.0, "delta": delta.0}))), Drag::Slide { clip, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.slide", json!({"clip": clip.0, "delta": delta.0}))), Drag::Marquee { start } => { @@ -2066,12 +2022,16 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } // ---- context menu on clips (right-clicking an unselected clip selects it first) + // the clip right-clicked, remembered while its menu is open: Unlink leaves only it selected + let menu_clip_id = egui::Id::new("timeline.clipMenu.clip"); if resp.secondary_clicked() && let Some(p) = resp.interact_pointer_pos() && let Hit::Clip { clip, .. } = hit(seq, layout, p) - && !app.session.state.selection.contains(&clip) { - let _ = app.session.execute("timeline.select", json!({"clips": [clip.0]})); + ctx.data_mut(|d| d.insert_temp(menu_clip_id, clip)); + if !app.session.state.selection.contains(&clip) { + let _ = app.session.execute("timeline.select", json!({"clips": [clip.0]})); + } } resp.context_menu(|ui| { ui.set_min_width(220.0); @@ -2108,8 +2068,18 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & let r = ui.add_enabled(!sel.is_empty() && app.session.is_enabled(cmd), egui::Button::new(label)); app.auto.add(&format!("timeline.clipMenu.{cmd}"), r.rect, label); if r.clicked() { - if let Err(e) = crate::menus::invoke(app, &ctx, cmd, json!({})) { - app.ui.status = e; + match crate::menus::invoke(app, &ctx, cmd, json!({})) { + // Unlink (#220): the clips were selected together because they were linked; + // keep only the one right-clicked, so it can be dragged away from the others + Ok(v) if cmd == "clip.link" && v["linked"] == json!(false) => { + if let Some(clip) = ctx.data(|d| d.get_temp::(menu_clip_id)) + && app.session.state.selection.contains(&clip) + { + let _ = app.session.execute("timeline.select", json!({"clips": [clip.0]})); + } + } + Ok(_) => {} + Err(e) => app.ui.status = e, } ui.close(); } diff --git a/crates/ui-egui/src/panels/timeline_captions.rs b/crates/ui-egui/src/panels/timeline_captions.rs index b154f554..bbb73c21 100644 --- a/crates/ui-egui/src/panels/timeline_captions.rs +++ b/crates/ui-egui/src/panels/timeline_captions.rs @@ -213,8 +213,10 @@ fn interact_block( if tr.locked { return; } + // decide on what was under the pointer when the button went down, not where egui recognised + // the drag (6 pt later, past the 5 px edge zones) (#259) if resp.drag_started() - && let Some(p) = resp.interact_pointer_pos() + && let Some(p) = ctx.input(|i| i.pointer.press_origin()).or(resp.interact_pointer_pos()) { let g = match gesture_at(p.x) { Gesture::Move => 0u8, diff --git a/crates/ui-egui/src/panels/timeline_hit.rs b/crates/ui-egui/src/panels/timeline_hit.rs new file mode 100644 index 00000000..f8bb64c2 --- /dev/null +++ b/crates/ui-egui/src/panels/timeline_hit.rs @@ -0,0 +1,434 @@ +//! What a press on the Timeline grabs (#259). +//! +//! Pure geometry and rules, shared by the cursor, the hover bracket, the press handler and the +//! control channel's `ui.timeline.hit`, so what the pointer shows is what a press does. + +use std::cmp::Ordering; + +use egui::{Modifiers, Pos2}; +use filmcraft_edit::Edge; +use filmcraft_project::{ClipId, Sequence, Track, TrackId, TransitionId}; + +use super::timeline::Layout; +use crate::state::Tool; + +/// How close (px) to a cut the Selection tool rolls instead of rippling when Settings ▸ Trim ▸ +/// "Allow Selection tool to choose Roll and Ripple trims without modifier key" is on. +pub const ROLL_PX: f32 = 2.5; + +/// Widest an edge zone gets (px), on each side of a clip edge. +pub const EDGE_PX: f32 = 7.0; + +/// The trim the Selection tool starts at an edit point: Cmd+Shift = roll, Cmd = ripple. With the +/// Trim setting on (`no_modifier`), no modifier is needed: right on the cut (when a clip is on the +/// other side) rolls, elsewhere on the edge ripples. Otherwise a plain drag is a regular trim. +pub fn selection_trim_kind(no_modifier: bool, cmd: bool, shift: bool, dist_px: f32, has_neighbour: bool) -> &'static str { + if cmd && shift { + "roll" + } else if cmd { + "ripple" + } else if no_modifier { + if has_neighbour && dist_px <= ROLL_PX { "roll" } else { "ripple" } + } else { + "trim" + } +} + +/// Distance (px) from `x` to the cut at `clip`'s `edge`, and whether a clip touches that cut on +/// the other side. +pub(crate) fn edge_geometry(seq: &Sequence, layout: &Layout, track: TrackId, clip: ClipId, edge: Edge, x: f32) -> (f32, bool) { + let Some(tr) = seq.track(track) else { return (f32::MAX, false) }; + let Some(it) = tr.item(clip) else { return (f32::MAX, false) }; + let (cut, neighbour) = match edge { + Edge::Out => (it.end(), tr.items.iter().any(|x| x.start == it.end())), + Edge::In => (it.start, tr.items.iter().any(|x| x.end() == it.start)), + }; + ((layout.x_of(cut) - x).abs(), neighbour) +} + +/// What is at a screen point. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Hit { + Ruler, + Clip { track: TrackId, clip: ClipId, edge: Option }, + Transition { track: TrackId, id: TransitionId }, + Empty { track: TrackId }, + None, +} + +pub fn hit(seq: &Sequence, layout: &Layout, pos: Pos2) -> Hit { + if layout.ruler.contains(pos) { + return Hit::Ruler; + } + if !layout.content.contains(pos) { + return Hit::None; + } + let Some(row) = layout.row_at(pos.y) else { return Hit::None }; + let Some(tr) = seq.track(row.track) else { return Hit::None }; + for trn in &tr.transitions { + let x0 = layout.x_of(trn.start); + let x1 = layout.x_of(trn.end()); + if pos.x >= x0 && pos.x <= x1 && pos.y > row.rect.min.y + 17.0 { + return Hit::Transition { track: row.track, id: trn.id }; + } + } + if let Some((clip, edge)) = edge_at(tr, layout, pos.x) { + return Hit::Clip { track: row.track, clip, edge: Some(edge) }; + } + match tr.item_at(layout.tick_at(pos.x)) { + Some(it) => Hit::Clip { track: row.track, clip: it.id, edge: None }, + None => Hit::Empty { track: row.track }, + } +} + +/// A clip edge within reach of the pointer. +#[derive(Clone, Copy)] +struct Reach { + clip: ClipId, + edge: Edge, + /// The pointer is inside the clip (or exactly on the edge). + inside: bool, + /// How far the pointer is from the edge (px). + dist: f32, +} + +impl Reach { + /// Inside a clip beats outside one (so at a cut the side of the cut picks the clip), then the + /// nearer edge wins; an exact tie (right on a cut, or mid-way across a narrow gap) goes to the + /// later clip's In edge. + fn beats(&self, other: &Reach) -> bool { + if self.inside != other.inside { + return self.inside; + } + match self.dist.partial_cmp(&other.dist) { + Some(Ordering::Less) => true, + Some(Ordering::Greater) => false, + _ => self.edge == Edge::In && other.edge != Edge::In, + } + } +} + +/// The edge zone (px, on each side of an edge) of a clip `w` px wide on screen: a third of the +/// clip, at most [`EDGE_PX`], so the middle third of a narrow clip still moves it. `None` when the +/// clip has no usable width (zoomed far out, or a broken zoom): it has no edges then. +fn zone(w: f32) -> Option { + if w.is_finite() && w > 0.0 { Some(EDGE_PX.min(w / 3.0)) } else { None } +} + +/// The clip edge on `tr` a pointer at `x` grabs (#259), whatever the order of `tr.items`. +fn edge_at(tr: &Track, layout: &Layout, x: f32) -> Option<(ClipId, Edge)> { + let mut best: Option = None; + for it in &tr.items { + let (x0, x1) = (layout.x_of(it.start), layout.x_of(it.end())); + let Some(e) = zone(x1 - x0) else { continue }; + // the pointer's distance into the clip from each of its edges + for (edge, d) in [(Edge::In, x - x0), (Edge::Out, x1 - x)] { + if !(-e..=e).contains(&d) { + continue; + } + let r = Reach { clip: it.id, edge, inside: d >= 0.0, dist: d.abs() }; + if best.is_none_or(|b| r.beats(&b)) { + best = Some(r); + } + } + } + best.map(|r| (r.clip, r.edge)) +} + +/// The trim a press on a clip edge starts. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum EdgeKind { + /// Regular trim: the edge moves, opening or covering a gap. + Trim, + /// Ripple trim: the edge moves and later clips follow. + Ripple, + /// Roll the cut between `left` and `right`. + Roll { left: ClipId, right: ClipId }, + /// Rate Stretch: the clip's speed changes to fill its new length. + Stretch, + /// Remix: a music clip is re-planned to its new length (Out edges only). + Remix, +} + +impl EdgeKind { + /// Its name on the control channel (`ui.timeline.hit` → `kind`). + pub fn name(self) -> &'static str { + match self { + EdgeKind::Trim => "trim", + EdgeKind::Ripple => "ripple", + EdgeKind::Roll { .. } => "roll", + EdgeKind::Stretch => "rateStretch", + EdgeKind::Remix => "remix", + } + } +} + +/// What a press of a tool at a point grabs: a clip edge and the trim it starts, or whatever else +/// is there. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Grab { + Edge { + track: TrackId, + clip: ClipId, + edge: Edge, + kind: EdgeKind, + }, + /// Not an edge this tool trims: a clip body, a transition, an empty track, the ruler or + /// nothing. A clip's `edge` may still be set, for tools that do something else there (Razor, + /// Slip…). + Other(Hit), +} + +impl Grab { + /// What is under the pointer, whatever the tool does with it. + pub fn hit(self) -> Hit { + match self { + Grab::Edge { track, clip, edge, .. } => Hit::Clip { track, clip, edge: Some(edge) }, + Grab::Other(h) => h, + } + } +} + +/// What a press of `tool` with `mods` at `pos` grabs (#259): the one place that decides which +/// tools trim at a clip edge and which trim they start. `roll_ripple` is Settings ▸ Trim ▸ "Allow +/// Selection tool to choose Roll and Ripple trims without modifier key". +pub fn grab_at(seq: &Sequence, layout: &Layout, pos: Pos2, tool: Tool, mods: Modifiers, roll_ripple: bool) -> Grab { + let h = hit(seq, layout, pos); + let Hit::Clip { track, clip, edge: Some(edge) } = h else { return Grab::Other(h) }; + let kind = match tool { + Tool::Selection => { + let (dist, neighbour) = edge_geometry(seq, layout, track, clip, edge, pos.x); + match selection_trim_kind(roll_ripple, mods.command, mods.shift, dist, neighbour) { + "roll" => roll(seq, track, clip, edge), + "ripple" => EdgeKind::Ripple, + _ => EdgeKind::Trim, + } + } + Tool::Ripple => EdgeKind::Ripple, + Tool::Rolling => roll(seq, track, clip, edge), + Tool::RateStretch => EdgeKind::Stretch, + Tool::Remix if edge == Edge::Out => EdgeKind::Remix, + _ => return Grab::Other(h), + }; + Grab::Edge { track, clip, edge, kind } +} + +/// A roll of the cut at `clip`'s `edge`, or a regular trim when no clip touches that cut on the +/// other side (there is nothing to roll against). +fn roll(seq: &Sequence, track: TrackId, clip: ClipId, edge: Edge) -> EdgeKind { + let Some(tr) = seq.track(track) else { return EdgeKind::Trim }; + let Some(it) = tr.item(clip) else { return EdgeKind::Trim }; + let pair = match edge { + Edge::Out => tr.items.iter().find(|x| x.start == it.end()).map(|r| (clip, r.id)), + Edge::In => tr.items.iter().find(|x| x.end() == it.start).map(|l| (l.id, clip)), + }; + match pair { + Some((left, right)) => EdgeKind::Roll { left, right }, + None => EdgeKind::Trim, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use egui::{Rect, pos2}; + use filmcraft_project::{ItemId, Label, Project, SequenceSettings, TrackItem, TrackKind}; + use filmcraft_time::{FrameRate, Tick}; + + use crate::panels::timeline::Row; + + /// The row's top. Points are hit-tested 10 px below it, above where transitions are hit. + const TOP: f32 = 100.0; + + /// A clip on screen from `x0` to `x1` px (the layout is 100 px per second). + fn clip(id: u64, x0: f64, x1: f64) -> TrackItem { + TrackItem { + id: ClipId(id), + item: ItemId(1), + name: format!("c{id}"), + label: Label::Iris, + start: Tick::from_seconds_f64(x0 / 100.0), + duration: Tick::from_seconds_f64((x1 - x0) / 100.0), + source_in: Tick::ZERO, + speed: 1.0, + reverse: false, + enabled: true, + link: None, + group: None, + effects: Vec::new(), + markers: Vec::new(), + gain_db: 0.0, + frame_hold: None, + scale_to_frame: false, + essential: None, + multicam: None, + time_interpolation: Default::default(), + hold_filters: false, + field_options: None, + source_channels: Vec::new(), + graphic: None, + } + } + + /// A: 100–300, B: 300–500 (sharing a cut with A), C: 504–700 (after a 4 px gap), D: 800–809 + /// (9 px wide, so its zones are 3 px). + fn clips() -> Vec { + vec![clip(1, 100.0, 300.0), clip(2, 300.0, 500.0), clip(3, 504.0, 700.0), clip(4, 800.0, 809.0)] + } + + /// One video track holding `items`, laid out at 100 px/s from x = 0. + fn fixture(items: Vec) -> (Sequence, Layout, TrackId) { + let mut p = Project::new("t"); + let s = p.new_sequence("s", SequenceSettings { frame_rate: FrameRate::FPS_24, ..Default::default() }, 1, 0, None); + let mut seq = p.sequence(s).cloned().expect("sequence"); + let track = seq.video_tracks[0].id; + seq.video_tracks[0].items = items; + let row = Rect::from_min_max(pos2(0.0, TOP), pos2(2000.0, TOP + 60.0)); + let layout = Layout { + content: row, + ruler: Rect::from_min_max(pos2(0.0, 0.0), pos2(2000.0, 40.0)), + rows: vec![Row { track, kind: TrackKind::Video, index: 0, rect: row }], + pps: 100.0, + scroll: 0.0, + split_y: TOP + 60.0, + }; + (seq, layout, track) + } + + fn at(x: f32) -> Pos2 { + pos2(x, TOP + 10.0) + } + + fn edge_hit(track: TrackId, clip: u64, edge: Edge) -> Hit { + Hit::Clip { track, clip: ClipId(clip), edge: Some(edge) } + } + + #[test] + fn the_side_of_a_cut_picks_the_clip() { + let (seq, l, tr) = fixture(clips()); + assert_eq!(hit(&seq, &l, at(297.0)), edge_hit(tr, 1, Edge::Out)); + assert_eq!(hit(&seq, &l, at(303.0)), edge_hit(tr, 2, Edge::In)); + // right on the cut: the later clip + assert_eq!(hit(&seq, &l, at(300.0)), edge_hit(tr, 2, Edge::In)); + } + + #[test] + fn a_narrow_gap_splits_at_its_middle() { + let (seq, l, tr) = fixture(clips()); + assert_eq!(hit(&seq, &l, at(501.0)), edge_hit(tr, 2, Edge::Out)); + assert_eq!(hit(&seq, &l, at(503.0)), edge_hit(tr, 3, Edge::In)); + // exactly mid-way: the later clip + assert_eq!(hit(&seq, &l, at(502.0)), edge_hit(tr, 3, Edge::In)); + } + + #[test] + fn free_edges_reach_into_the_gap_beside_them() { + let (seq, l, tr) = fixture(clips()); + assert_eq!(hit(&seq, &l, at(95.0)), edge_hit(tr, 1, Edge::In)); + assert_eq!(hit(&seq, &l, at(705.0)), edge_hit(tr, 3, Edge::Out)); + assert_eq!(hit(&seq, &l, at(92.0)), Hit::Empty { track: tr }); + assert_eq!(hit(&seq, &l, at(200.0)), Hit::Clip { track: tr, clip: ClipId(1), edge: None }); + } + + #[test] + fn edges_do_not_depend_on_the_order_of_the_items() { + let mut reversed = clips(); + reversed.reverse(); + let (a, la, _) = fixture(clips()); + let (b, lb, _) = fixture(reversed); + // edge points only: finding the clip under a body point relies on the items being sorted + for x in [297.0, 300.0, 303.0, 501.0, 502.0, 503.0, 95.0, 705.0, 802.0, 807.0] { + assert_eq!(hit(&a, &la, at(x)), hit(&b, &lb, at(x)), "x = {x}"); + } + } + + #[test] + fn a_narrow_clip_keeps_its_middle_third() { + let (seq, l, tr) = fixture(clips()); + assert_eq!(hit(&seq, &l, at(802.0)), edge_hit(tr, 4, Edge::In)); + assert_eq!(hit(&seq, &l, at(807.0)), edge_hit(tr, 4, Edge::Out)); + assert_eq!(hit(&seq, &l, at(804.5)), Hit::Clip { track: tr, clip: ClipId(4), edge: None }); + // 4 px outside it is beyond its 3 px zone + assert_eq!(hit(&seq, &l, at(796.0)), Hit::Empty { track: tr }); + } + + #[test] + fn hostile_points_and_zooms_never_panic_or_grab_an_edge() { + let (seq, l, tr) = fixture(vec![clip(5, 1000.0, 1000.0)]); + // a zero-length clip has no edges, even right on it + assert_eq!(hit(&seq, &l, at(1000.0)), Hit::Empty { track: tr }); + for p in [pos2(f32::NAN, TOP + 10.0), pos2(f32::INFINITY, TOP + 10.0), pos2(f32::NEG_INFINITY, TOP + 10.0), pos2(300.0, f32::NAN), pos2(300.0, 5000.0)] + { + assert_eq!(hit(&seq, &l, p), Hit::None, "{p:?}"); + } + let (seq, l, _) = fixture(clips()); + assert_eq!(hit(&seq, &l, pos2(300.0, 20.0)), Hit::Ruler); + for pps in [0.0, f64::NAN, f64::INFINITY] { + let broken = Layout { pps, ..l.clone() }; + assert!(!matches!(hit(&seq, &broken, at(300.0)), Hit::Clip { edge: Some(_), .. }), "pps {pps}"); + } + } + + use egui::Modifiers; + + use crate::state::Tool; + + const ROLL_AB: EdgeKind = EdgeKind::Roll { left: ClipId(1), right: ClipId(2) }; + + fn grab(seq: &Sequence, l: &Layout, tool: Tool, x: f32, mods: Modifiers, roll_ripple: bool) -> Grab { + grab_at(seq, l, at(x), tool, mods, roll_ripple) + } + + fn edge_grab(track: TrackId, clip: u64, edge: Edge, kind: EdgeKind) -> Grab { + Grab::Edge { track, clip: ClipId(clip), edge, kind } + } + + #[test] + fn selection_tool_trims_by_modifier_and_setting() { + let (seq, l, tr) = fixture(clips()); + let (none, cmd, cmd_shift) = (Modifiers::NONE, Modifiers::COMMAND, Modifiers::COMMAND | Modifiers::SHIFT); + let sel = |x, mods, pref| grab(&seq, &l, Tool::Selection, x, mods, pref); + assert_eq!(sel(297.0, none, false), edge_grab(tr, 1, Edge::Out, EdgeKind::Trim)); + assert_eq!(sel(303.0, cmd, false), edge_grab(tr, 2, Edge::In, EdgeKind::Ripple)); + assert_eq!(sel(303.0, cmd_shift, false), edge_grab(tr, 2, Edge::In, ROLL_AB)); + // Settings ▸ Trim: right on the cut rolls, elsewhere on the edge ripples + assert_eq!(sel(298.0, none, true), edge_grab(tr, 1, Edge::Out, ROLL_AB)); + assert_eq!(sel(302.0, none, true), edge_grab(tr, 2, Edge::In, ROLL_AB)); + assert_eq!(sel(305.0, none, true), edge_grab(tr, 2, Edge::In, EdgeKind::Ripple)); + // a roll with no clip on the other side of the cut is a regular trim + assert_eq!(sel(102.0, cmd_shift, false), edge_grab(tr, 1, Edge::In, EdgeKind::Trim)); + // mid-clip: the clip itself + assert_eq!(sel(200.0, none, false), Grab::Other(Hit::Clip { track: tr, clip: ClipId(1), edge: None })); + } + + #[test] + fn which_tools_trim_at_an_edge() { + let (seq, l, tr) = fixture(clips()); + let g = |tool, x| grab(&seq, &l, tool, x, Modifiers::NONE, false); + assert_eq!(g(Tool::Ripple, 297.0), edge_grab(tr, 1, Edge::Out, EdgeKind::Ripple)); + assert_eq!(g(Tool::Rolling, 303.0), edge_grab(tr, 2, Edge::In, ROLL_AB)); + assert_eq!(g(Tool::Rolling, 102.0), edge_grab(tr, 1, Edge::In, EdgeKind::Trim)); + assert_eq!(g(Tool::RateStretch, 297.0), edge_grab(tr, 1, Edge::Out, EdgeKind::Stretch)); + assert_eq!(g(Tool::Remix, 297.0), edge_grab(tr, 1, Edge::Out, EdgeKind::Remix)); + // Remix has Out edges only + assert_eq!(g(Tool::Remix, 303.0), Grab::Other(edge_hit(tr, 2, Edge::In))); + for tool in [Tool::Razor, Tool::Slip, Tool::Slide, Tool::Hand, Tool::Zoom, Tool::TrackSelectForward] { + assert_eq!(g(tool, 297.0), Grab::Other(edge_hit(tr, 1, Edge::Out)), "{tool:?}"); + } + } + + #[test] + fn edge_kind_names_for_the_control_channel() { + assert_eq!( + [EdgeKind::Trim, EdgeKind::Ripple, ROLL_AB, EdgeKind::Stretch, EdgeKind::Remix].map(EdgeKind::name), + ["trim", "ripple", "roll", "rateStretch", "remix"] + ); + } + + #[test] + fn a_grab_reports_what_is_under_it() { + let (seq, l, tr) = fixture(clips()); + assert_eq!(grab(&seq, &l, Tool::Selection, 303.0, Modifiers::NONE, false).hit(), edge_hit(tr, 2, Edge::In)); + assert_eq!(grab(&seq, &l, Tool::Razor, 303.0, Modifiers::NONE, false).hit(), edge_hit(tr, 2, Edge::In)); + } +} diff --git a/crates/ui-egui/src/panels/trim_monitor.rs b/crates/ui-egui/src/panels/trim_monitor.rs index 4347bf6f..cee0b505 100644 --- a/crates/ui-egui/src/panels/trim_monitor.rs +++ b/crates/ui-egui/src/panels/trim_monitor.rs @@ -14,7 +14,8 @@ //! `trimMonitor.playAround`, `trimMonitor.exit`. use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; -use filmcraft_project::{ItemKind, Sequence, Track}; +use filmcraft_edit::Edge; +use filmcraft_project::{ClipId, ItemKind, Sequence, Track, TrackId}; use filmcraft_time::{Tick, TimeDisplay, format_time}; use serde_json::{Value, json}; @@ -22,6 +23,7 @@ use crate::FilmcraftApp; use crate::frames::{FrameKey, Target}; use crate::icons::{self, Icon}; use crate::panels::timeline::{Layout, Row}; +use crate::panels::timeline_hit::EdgeKind; use crate::theme::Tokens; /// Ripple edit points are red, roll / regular trims yellow (as in Premiere's timeline). @@ -79,19 +81,40 @@ pub fn paint_edit_points(app: &FilmcraftApp, p: &egui::Painter, seq: &Sequence, let col = if ep.kind == TrimKind::Ripple { RIPPLE_RED } else { ROLL_YELLOW }; for (clip, out) in marks { let Some(it) = tr.item(clip) else { continue }; - let x = layout.x_of(if out { it.end() } else { it.start }); - let (y0, y1) = (row.rect.min.y + 2.0, row.rect.max.y - 2.0); - let dir = if out { -1.0 } else { 1.0 }; - let xi = x + dir * 1.5; - let w = 6.0 * dir; - let s = Stroke::new(3.0, col); - p.line_segment([pos2(xi, y0), pos2(xi, y1)], s); - p.line_segment([pos2(xi, y0 + 1.0), pos2(xi + w, y0 + 1.0)], s); - p.line_segment([pos2(xi, y1 - 1.0), pos2(xi + w, y1 - 1.0)], s); + paint_bracket(p, layout.x_of(if out { it.end() } else { it.start }), row, out, col); } } } +/// A trim bracket at `x` on `row`, its arms pointing into the clip (left for an Out edge). +pub fn paint_bracket(p: &egui::Painter, x: f32, row: &Row, out: bool, col: Color32) { + let (y0, y1) = (row.rect.min.y + 2.0, row.rect.max.y - 2.0); + let dir = if out { -1.0 } else { 1.0 }; + let xi = x + dir * 1.5; + let w = 6.0 * dir; + let s = Stroke::new(3.0, col); + p.line_segment([pos2(xi, y0), pos2(xi, y1)], s); + p.line_segment([pos2(xi, y0 + 1.0), pos2(xi + w, y0 + 1.0)], s); + p.line_segment([pos2(xi, y1 - 1.0), pos2(xi + w, y1 - 1.0)], s); +} + +/// The bracket on the edge a press would grab (#259): an edit point's bracket at half strength, +/// red for a ripple and yellow otherwise; a roll marks both sides of the cut. +pub fn paint_hover_bracket(p: &egui::Painter, seq: &Sequence, layout: &Layout, track: TrackId, clip: ClipId, edge: Edge, kind: EdgeKind) { + let Some(row) = layout.rows.iter().find(|r| r.track == track) else { return }; + let Some(tr) = seq.track(track) else { return }; + let full = if kind == EdgeKind::Ripple { RIPPLE_RED } else { ROLL_YELLOW }; + let col = full.gamma_multiply(0.5); + let marks = match kind { + EdgeKind::Roll { left, right } => vec![(left, true), (right, false)], + _ => vec![(clip, edge == Edge::Out)], + }; + for (c, out) in marks { + let Some(it) = tr.item(c) else { continue }; + paint_bracket(p, layout.x_of(if out { it.end() } else { it.start }), row, out, col); + } +} + /// Fit a w×h picture into `area`. fn fit(area: Rect, w: f32, h: f32) -> Rect { let s = (area.width() / w.max(1.0)).min(area.height() / h.max(1.0)); diff --git a/crates/ui-egui/src/perf.rs b/crates/ui-egui/src/perf.rs index 79df3c52..b5b44f1b 100644 --- a/crates/ui-egui/src/perf.rs +++ b/crates/ui-egui/src/perf.rs @@ -31,6 +31,13 @@ pub fn stats(app: &FilmcraftApp) -> Value { frames["renderCostMs"] = json!(app.frames.render_cost() * 1e3); frames["cache"] = json!({"images": images, "imageMB": image_bytes as f64 / 1e6, "plans": plans, "planMB": plan_bytes as f64 / 1e6}); v["frames"] = frames; + // the Program monitor's picture against what is due: how far a drag or a scrub is ahead of it + v["monitor"] = json!({ + "frame": app.program_picture().map(|k| k.frame), + "revision": app.program_picture().map(|k| k.revision), + "projectRevision": app.session.revision, + "playheadFrame": app.session.active_sequence().map(|q| q.settings.frame_rate.frame_at(app.session.playhead())), + }); v["ui"] = json!({"fps": app.fps, "frameMs": if app.fps > 0.0 { 1000.0 / app.fps as f64 } else { 0.0 }}); v["process"] = json!({"cpuS": crate::frames::process_cpu_time().map(|d| d.as_secs_f64())}); v diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 0bdfcda8..6e4780d3 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -384,6 +384,12 @@ pub struct UiState { /// Collapsed effect sections in Effect Controls ("clip:index"). pub collapsed_fx: Vec, pub show_menu_bar: bool, + /// The header bar (Home, Import, Edit, Export, workspaces). An app that embeds FilmCraft can hide it. + #[serde(default = "shown")] + pub show_header: bool, + /// The status bar (hints, messages, job progress). An app that embeds FilmCraft can hide it. + #[serde(default = "shown")] + pub show_status_bar: bool, pub dark: bool, /// Lumetri scopes visible in the Program monitor area. pub show_scopes: bool, @@ -432,6 +438,9 @@ pub struct UiState { /// Delete Tracks dialog draft. #[serde(default)] pub delete_tracks: DeleteTracksDraft, + /// Sequence Settings dialog draft. + #[serde(default)] + pub sequence_settings: SequenceSettingsDraft, /// On-monitor text editing (Type tool / double-click on a text layer). #[serde(default)] pub gfx_edit: Option, @@ -735,6 +744,49 @@ impl Default for AddTracksDraft { } } +/// The Sequence Settings dialog (Sequence ▸ Sequence Settings…), filled from the active sequence +/// when it opens (`panels::sequence_settings::open`). `tab`: `general`, `color` or `vr`; the +/// timebase is `fps_num`/`fps_den`; `mix`: `Stereo`, `Mono`, `5.1` or `Adaptive`; `working_space`: +/// a [`filmcraft_color::WorkingSpace`] id. `scale_motion` is "Scale motion effects proportionally +/// when changing frame size", on by default as in Premiere. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct SequenceSettingsDraft { + pub tab: String, + pub fps_num: i64, + pub fps_den: i64, + pub width: u32, + pub height: u32, + pub scale_motion: bool, + pub drop_frame: bool, + pub mix: String, + pub sample_rate: u32, + pub max_render_quality: bool, + pub working_space: String, + pub wide_gamut: bool, + pub auto_tone_map: bool, +} + +impl Default for SequenceSettingsDraft { + fn default() -> Self { + Self { + tab: "general".into(), + fps_num: 24_000, + fps_den: 1001, + width: 1920, + height: 1080, + scale_motion: true, + drop_frame: false, + mix: "Stereo".into(), + sample_rate: 48_000, + max_render_quality: false, + working_space: "rec709".into(), + wide_gamut: false, + auto_tone_map: true, + } + } +} + /// The Delete Tracks dialog (Sequence ▸ Delete Tracks…): per kind, whether to delete and which /// track (`"empty"` = All Empty Tracks, or a track name such as `"V2"`). #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -776,6 +828,10 @@ pub struct GfxEdit { pub anchor: usize, } +fn shown() -> bool { + true +} + fn captions_tab() -> String { "Captions".into() } @@ -803,6 +859,8 @@ impl Default for UiState { lumetri_grid_folder: None, collapsed_fx: vec![], show_menu_bar: true, + show_header: true, + show_status_bar: true, dark: true, show_scopes: false, status: String::new(), @@ -820,6 +878,7 @@ impl Default for UiState { audio_gain: AudioGainDraft::default(), add_tracks: AddTracksDraft::default(), delete_tracks: DeleteTracksDraft::default(), + sequence_settings: SequenceSettingsDraft::default(), gfx_edit: None, pen_points: vec![], link_media: None, diff --git a/crates/ui-egui/tests/chrome_ui.rs b/crates/ui-egui/tests/chrome_ui.rs new file mode 100644 index 00000000..c5efaea0 --- /dev/null +++ b/crates/ui-egui/tests/chrome_ui.rs @@ -0,0 +1,68 @@ +//! Headless UI tests of the window chrome flags: `UiState::show_header` and +//! `UiState::show_status_bar` default to on, and an app that embeds FilmCraft can turn them off +//! to give the panels the whole window. + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::state::UiState; +use serde_json::json; + +const W: f32 = 1600.0; +const H: f32 = 980.0; + +fn harness(show_header: bool, show_status_bar: bool) -> Harness<'static, FilmcraftApp> { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let mut app = FilmcraftApp::new(s); + app.ui.show_header = show_header; + app.ui.show_status_bar = show_status_bar; + let mut harness = Harness::builder().with_size(egui::vec2(W, H)).with_max_steps(10_000).build_eframe(move |_cc| app); + for _ in 0..4 { + harness.step(); + } + harness +} + +/// Top and bottom edge of every registered element outside the header. +fn body_extent(h: &Harness<'static, FilmcraftApp>) -> (f32, f32) { + let els = h.state().auto.query(""); + let body: Vec<_> = els.iter().filter(|e| !e.id.starts_with("header.")).collect(); + assert!(!body.is_empty(), "no panel elements registered"); + let top = body.iter().map(|e| e.rect[1]).fold(f32::INFINITY, f32::min); + let bottom = body.iter().map(|e| e.rect[1] + e.rect[3]).fold(f32::NEG_INFINITY, f32::max); + (top, bottom) +} + +#[test] +fn header_and_status_bar_are_shown_by_default() { + assert!(UiState::default().show_header); + assert!(UiState::default().show_status_bar); + let h = harness(true, true); + assert!(!h.state().auto.query("header.").is_empty()); + let (top, bottom) = body_extent(&h); + assert!(top >= 38.0, "panels start under the header: {top}"); + assert!(bottom <= H - 20.0, "panels end above the status bar: {bottom}"); + assert!(h.state().ui_error.is_none(), "{:?}", h.state().ui_error); +} + +#[test] +fn hidden_header_and_status_bar_give_the_panels_the_window() { + let h = harness(false, false); + assert!(h.state().auto.query("header.").is_empty(), "header drawn while hidden"); + let (top, bottom) = body_extent(&h); + assert!(top < 38.0, "panels start at the top of the window: {top}"); + assert!(bottom > H - 20.0, "panels reach the bottom of the window: {bottom}"); + assert!(h.state().ui_error.is_none(), "{:?}", h.state().ui_error); +} + +#[test] +fn saved_ui_state_without_the_flags_shows_the_chrome() { + let mut v = serde_json::to_value(UiState::default()).unwrap(); + let obj = v.as_object_mut().unwrap(); + assert!(obj.remove("show_header").is_some()); + assert!(obj.remove("show_status_bar").is_some()); + let ui: UiState = serde_json::from_value(v).unwrap(); + assert!(ui.show_header); + assert!(ui.show_status_bar); +} diff --git a/crates/ui-egui/tests/export_menu_ui.rs b/crates/ui-egui/tests/export_menu_ui.rs new file mode 100644 index 00000000..4c1ef8cc --- /dev/null +++ b/crates/ui-egui/tests/export_menu_ui.rs @@ -0,0 +1,144 @@ +//! Headless UI tests of File ▸ Export from the menus (#382): a menu entry carries no parameters, so +//! the entries whose command needs a `path` ask the host's save panel for it, and Media… opens the +//! Export mode like ⌘M, instead of failing with a missing `path`. + +use std::sync::mpsc::{Sender, channel}; +use std::sync::{Arc, Mutex}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use filmcraft_ui_egui::state::Mode; +use serde_json::{Value, json}; + +/// What the save panel was asked: filter label, extensions, suggested name. +type Asked = Arc, String)>>>; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, +} + +impl Driver { + /// `save_to: None` is a cancelled save panel. + fn new(session: Session, save_to: Option, asked: Asked) -> Self { + let (tx, rx) = channel(); + let mut app = FilmcraftApp::new(session).with_control(rx); + app.hooks.pick_save_as = Some(Box::new(move |filter, exts, suggested| { + asked.lock().unwrap().push((filter.to_string(), exts.iter().map(|e| e.to_string()).collect(), suggested.to_string())); + save_to.as_ref().map(|d| d.join(suggested).to_string_lossy().into_owned()) + })); + let harness = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000).build_eframe(move |_cc| app); + let mut d = Driver { harness, tx }; + d.frames(4); + d + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + + fn call(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(v) = reply.try_recv() { + return v; + } + } + panic!("no reply to {method} {params}"); + } + + fn ok(&mut self, method: &str, params: Value) -> Value { + let v = self.call(method, params.clone()); + assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}"); + v["result"].clone() + } +} + +/// The demo project (sequences, clips) with one clip selected in the Project panel. +fn session() -> Session { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let clip = s.project.items.iter().find(|(_, it)| it.name.ends_with(".mp4")).map(|(id, _)| id.0).unwrap(); + s.execute("project.select", json!({"items": [clip]})).unwrap(); + s +} + +fn tmp(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("fc-export-menu-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// Every File ▸ Export entry whose command needs a path, and the extension it saves. +const SAVED: [(&str, &str); 6] = [ + ("file.exportEdl", "edl"), + ("file.exportFcp7Xml", "xml"), + ("file.exportFcpxml", "fcpxml"), + ("file.exportOtio", "otio"), + ("file.exportAle", "ale"), + ("file.exportSelectionProject", "fcproj"), +]; + +#[test] +fn file_export_entries_ask_for_the_destination() { + let dir = tmp("save"); + let asked = Asked::default(); + let mut d = Driver::new(session(), Some(dir.clone()), asked.clone()); + let menu = d.ok("ui.menu.list", json!({})); + for (id, ext) in SAVED { + let entry = menu.as_array().unwrap().iter().find(|i| i["id"] == id).unwrap_or_else(|| panic!("{id} missing from the menus")); + assert_eq!(entry["path"], json!(["File", "Export"]), "{id}"); + let r = d.ok("ui.menu.invoke", json!({"id": id})); + d.frames(2); + let (_, exts, suggested) = asked.lock().unwrap().last().cloned().unwrap_or_else(|| panic!("{id} did not ask where to save")); + assert_eq!(exts, [ext], "{id}"); + assert!(suggested.ends_with(&format!(".{ext}")) && suggested.len() > ext.len() + 1, "{id}: {suggested}"); + let written = dir.join(&suggested); + assert!(written.metadata().is_ok_and(|m| m.len() > 0), "{id} wrote nothing at {} ({r})", written.display()); + } + assert_eq!(asked.lock().unwrap().len(), SAVED.len()); +} + +#[test] +fn cancelling_the_save_panel_writes_nothing_and_reports_no_error() { + let asked = Asked::default(); + let mut d = Driver::new(session(), None, asked.clone()); + for (id, _) in SAVED { + assert_eq!(d.ok("ui.menu.invoke", json!({"id": id})), Value::Null, "{id}"); + } + assert_eq!(asked.lock().unwrap().len(), SAVED.len()); + assert!(!d.harness.state().ui.status.contains("path"), "{}", d.harness.state().ui.status); +} + +#[test] +fn media_from_the_menu_opens_the_export_mode() { + let mut d = Driver::new(session(), None, Asked::default()); + assert_eq!(d.harness.state().ui.mode, Mode::Edit); + d.ok("ui.menu.invoke", json!({"id": "file.exportMedia"})); + d.frames(2); + assert_eq!(d.harness.state().ui.mode, Mode::Export); + // with a path it still exports directly, as scripts and the CLI use it + let out = tmp("media").join("direct.wav"); + let r = d.ok("ui.menu.invoke", json!({"id": "file.exportMedia", "params": {"path": out.to_string_lossy(), "format": "wav", "wait": true}})); + assert!(out.metadata().is_ok_and(|m| m.len() > 0), "{r}"); +} + +#[test] +fn file_export_entries_need_an_open_sequence() { + let mut d = Driver::new(Session::default(), Some(tmp("noseq")), Asked::default()); + for id in ["file.exportMedia", "file.exportEdl", "file.exportFcp7Xml", "file.exportFcpxml", "file.exportOtio"] { + assert_eq!(d.call("ui.menu.invoke", json!({"id": id}))["ok"], json!(false), "{id}"); + assert_eq!(d.harness.state().ui.mode, Mode::Edit, "{id}"); + } +} diff --git a/crates/ui-egui/tests/keyboard_ui.rs b/crates/ui-egui/tests/keyboard_ui.rs index 67020b7f..03660632 100644 --- a/crates/ui-egui/tests/keyboard_ui.rs +++ b/crates/ui-egui/tests/keyboard_ui.rs @@ -179,3 +179,62 @@ fn new_keys_work_in_the_app() { d.key("Cmd+Shift+0"); assert_eq!(d.app().ui.program.zoom, None); } + +/// Off macOS the menus show `Ctrl+…`, so that is what agents send: a `Ctrl` chord through +/// `ui.key` (and `ctrl` in `ui.click` modifiers) must act like the physical key, which egui-winit +/// reports as `ctrl` + `command`. Bindings are written `Cmd+…`, so this failed to match (#245). +#[cfg(not(target_os = "macos"))] +#[test] +fn ctrl_chords_fire_shortcuts_off_macos() { + let mut d = Driver::new(); + let seq = d.exec("sequence.inspect", json!({})); + let second = &seq["video"][0]["items"].as_array().unwrap()[1]; + let (clip, start) = (second["clip"].as_u64().unwrap(), second["start"].as_i64().unwrap()); + let start_of = |d: &mut Driver| { + d.exec("sequence.inspect", json!({}))["video"][0]["items"] + .as_array() + .unwrap() + .iter() + .find(|i| i["clip"].as_u64() == Some(clip)) + .map(|i| i["start"].as_i64().unwrap()) + .unwrap() + }; + d.exec("playhead.set", json!({"time": start + 1_000_000_000})); + d.focus("Timeline"); + d.key("D"); + assert!(d.app().session.state.selection.iter().any(|c| c.0 == clip)); + // Ctrl+Right nudges one frame right, like Cmd+Right; Ctrl+Z undoes it + let rate = d.app().session.sequence_rate(); + d.key("Ctrl+Right"); + assert_eq!(start_of(&mut d), start + rate.tick_of(1).0, "Ctrl+Right nudges"); + d.key("Ctrl+Z"); + assert_eq!(start_of(&mut d), start, "Ctrl+Z undoes"); + // the same chord spelt with the modifier flag + d.ok("ui.key", json!({"key": "Right", "ctrl": true})); + d.frames(3); + assert_eq!(start_of(&mut d), start + rate.tick_of(1).0, "`ctrl: true` nudges"); + d.ok("ui.key", json!({"key": "Z", "ctrl": true})); + d.frames(3); + assert_eq!(start_of(&mut d), start, "`ctrl: true` undoes"); + // and the bare key still does not + d.key("Z"); + assert_eq!(start_of(&mut d), start); +} + +#[test] +fn delete_key_clears_the_selected_timeline_clip() { + // #243: on Windows and Linux keyboards Delete is the forward-delete key; with the Timeline + // focused it ran Project ▸ Clear (nothing selected there), so the clip stayed. + let mut d = Driver::new(); + let items = |d: &mut Driver| d.exec("sequence.inspect", json!({}))["video"][0]["items"].as_array().unwrap().clone(); + let second = items(&mut d)[1].clone(); + let (clip, start) = (second["clip"].as_u64().unwrap(), second["start"].as_i64().unwrap()); + d.exec("playhead.set", json!({"time": start + 1_000_000_000})); + d.focus("Timeline"); + d.key("D"); + assert!(d.app().session.state.selection.iter().any(|c| c.0 == clip), "D selects the clip under the playhead"); + d.key("Delete"); + assert!(!items(&mut d).iter().any(|i| i["clip"].as_u64() == Some(clip)), "Delete removed the selected clip"); + d.key("Cmd+Z"); + assert!(items(&mut d).iter().any(|i| i["clip"].as_u64() == Some(clip)), "and undo brings it back"); +} diff --git a/crates/ui-egui/tests/keyframes_ui.rs b/crates/ui-egui/tests/keyframes_ui.rs new file mode 100644 index 00000000..630a8cea --- /dev/null +++ b/crates/ui-egui/tests/keyframes_ui.rs @@ -0,0 +1,227 @@ +//! Keyframes in the Properties and Effect Controls panels, driven headless through the control +//! channel: the Properties panel's keyframe navigator (◀ ◆ ▶) adds and removes the keyframe at the +//! playhead and steps between keyframes, as Premiere's does, and the Effect Controls time ruler +//! moves the playhead (click, or drag the playhead's handle). +//! +//! With `FILMCRAFT_UI_SHOTS=

` the tests also render the UI with wgpu and write PNGs there. + +use std::sync::mpsc::{Sender, channel}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_project::ClipId; +use filmcraft_time::Tick; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use serde_json::{Value, json}; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, + shots: Option, +} + +impl Driver { + /// The demo project with its first video clip selected and both panels showing. + fn demo() -> (Self, Clip) { + let mut session = Session::default(); + session.execute("file.openDemoProject", json!({})).expect("demo project"); + let (tx, rx) = channel(); + let app = FilmcraftApp::new(session).with_control(rx); + let shots = std::env::var_os("FILMCRAFT_UI_SHOTS").map(std::path::PathBuf::from); + let mut b = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000); + if shots.is_some() { + b = b.wgpu().with_pixels_per_point(1.0); + } + let harness = b.build_eframe(move |_cc| app); + let mut d = Driver { harness, tx, shots }; + d.frames(4); + let seq = d.exec("sequence.inspect", json!({})); + let c = &seq["video"][0]["items"][0]; + let clip = Clip { id: c["clip"].as_u64().unwrap(), start: c["start"].as_i64().unwrap(), duration: c["duration"].as_i64().unwrap() }; + d.exec("timeline.select", json!({"clips": [clip.id]})); + for panel in ["EffectControls", "Properties"] { + d.ok("ui.panel.show", json!({"panel": panel})); + } + d.frames(3); + (d, clip) + } + fn frames(&mut self, n: usize) { + for _ in 0..n { + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + fn call(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(v) = reply.try_recv() { + return v; + } + } + panic!("no reply to {method} {params}"); + } + fn ok(&mut self, method: &str, params: Value) -> Value { + let r = self.call(method, params.clone()); + assert_eq!(r["ok"], true, "{method} {params}: {r}"); + r["result"].clone() + } + fn exec(&mut self, command: &str, params: Value) -> Value { + self.ok("engine.execute", json!({"command": command, "params": params})) + } + /// The element's rect `[x, y, w, h]`, or `None` when it is not on screen. + fn find(&mut self, id: &str) -> Option<[f64; 4]> { + self.frames(2); + let v = self.ok("ui.elements", json!({"prefix": id})); + let e = v.as_array().unwrap().iter().find(|e| e["id"] == id)?; + let r: Vec = e["rect"].as_array().unwrap().iter().map(|x| x.as_f64().unwrap()).collect(); + Some([r[0], r[1], r[2], r[3]]) + } + fn rect(&mut self, id: &str) -> [f64; 4] { + self.find(id).unwrap_or_else(|| panic!("no element {id}")) + } + fn click(&mut self, id: &str) { + self.rect(id); + self.ok("ui.click", json!({"id": id})); + self.frames(3); + } + fn playhead(&mut self) -> i64 { + self.exec("sequence.inspect", json!({}))["playhead"].as_i64().unwrap() + } + fn seek(&mut self, seconds: f64) -> i64 { + self.exec("playhead.set", json!({"seconds": seconds})); + self.frames(2); + self.playhead() + } + /// The clip's Motion ▸ Scale: its keyframe times (media time) and its value at the playhead. + fn scale(&mut self, clip: &Clip) -> (Vec, f64) { + let s = &self.harness.state().session; + let it = s.active_sequence().unwrap().find_item(ClipId(clip.id)).unwrap().1; + let p = it.effect("motion").unwrap().param("scale").unwrap(); + (p.keyframes.iter().map(|k| k.time.0).collect(), p.f64_at(it.source_time_at(s.playhead()))) + } + fn shot(&mut self, name: &str) { + let Some(dir) = self.shots.clone() else { return }; + for _ in 0..40 { + self.frames(1); + std::thread::sleep(std::time::Duration::from_millis(25)); + } + let img = self.harness.render().expect("wgpu render"); + std::fs::create_dir_all(&dir).unwrap(); + img.save(dir.join(format!("{name}.png"))).unwrap(); + } +} + +struct Clip { + id: u64, + start: i64, + duration: i64, +} + +const SCALE: &str = "properties.motion.scale"; + +/// The reported case: two Scale keyframes, the second set to 50 %. The Properties diamond removes +/// only the keyframe at the playhead, so Scale is 100 % again there (both panels read the same +/// parameter); it used to switch animation off and keep 50 %. +#[test] +fn properties_diamond_adds_and_removes_the_keyframe_at_the_playhead() { + let (mut d, clip) = Driver::demo(); + // not animated: the diamond alone + d.rect(&format!("{SCALE}.addKeyframe")); + assert!(d.find(&format!("{SCALE}.prevKeyframe")).is_none(), "no arrows before the parameter is animated"); + + let t0 = d.seek(0.5); + d.click(&format!("{SCALE}.addKeyframe")); + assert_eq!(d.scale(&clip).0.len(), 1, "the diamond turns animation on with a keyframe at the playhead"); + // animated: the arrows appear, in Properties and in Effect Controls + for panel in [SCALE, "effectControls.motion.scale"] { + d.rect(&format!("{panel}.prevKeyframe")); + d.rect(&format!("{panel}.nextKeyframe")); + } + + let t1 = d.seek(2.5); + d.click(&format!("{SCALE}.addKeyframe")); + assert_eq!(d.scale(&clip).0.len(), 2, "a second keyframe at the playhead"); + d.exec("effects.setParam", json!({"clip": clip.id, "effect": "motion", "param": "scale", "value": 50.0})); + assert_eq!(d.scale(&clip).1, 50.0); + d.shot("keyframes-two"); + + // on a keyframe: the diamond removes that keyframe only + d.click(&format!("{SCALE}.addKeyframe")); + let (keys, value) = d.scale(&clip); + assert_eq!(keys.len(), 1, "only the keyframe at the playhead went: {keys:?}"); + assert_eq!(value, 100.0, "Scale falls back to the remaining keyframe's value"); + assert_eq!(d.playhead(), t1); + d.shot("keyframes-removed"); + d.exec("edit.undo", json!({})); + let (keys, value) = d.scale(&clip); + assert_eq!((keys.len(), value), (2, 50.0), "one undo step brings the keyframe back"); + + // the arrows step between the keyframes + d.click(&format!("{SCALE}.prevKeyframe")); + assert_eq!(d.playhead(), t0, "◀ goes to the previous keyframe"); + assert_eq!(d.scale(&clip).1, 100.0); + d.click(&format!("{SCALE}.prevKeyframe")); + assert_eq!(d.playhead(), t0, "no keyframe before the first: ◀ does nothing"); + d.click(&format!("{SCALE}.nextKeyframe")); + assert_eq!(d.playhead(), t1, "▶ goes to the next keyframe"); + assert_eq!(d.scale(&clip).1, 50.0); + d.click("effectControls.motion.scale.prevKeyframe"); + assert_eq!(d.playhead(), t0, "the Effect Controls arrows do the same"); + + // removing the last keyframe ends the animation; the arrows go + d.click(&format!("{SCALE}.addKeyframe")); + d.click(&format!("{SCALE}.nextKeyframe")); + d.click(&format!("{SCALE}.addKeyframe")); + let (keys, value) = d.scale(&clip); + assert!(keys.is_empty(), "{keys:?}"); + assert_eq!(value, 50.0, "the last keyframe's value stays"); + assert!(d.find(&format!("{SCALE}.prevKeyframe")).is_none()); +} + +/// Crop is not on a clip until it is used: its diamond applies the effect, then adds the keyframe. +#[test] +fn properties_diamond_applies_crop_first() { + let (mut d, clip) = Driver::demo(); + d.seek(1.0); + d.click("properties.crop.left.addKeyframe"); + let s = &d.harness.state().session; + let it = s.active_sequence().unwrap().find_item(ClipId(clip.id)).unwrap().1; + assert_eq!(it.effect("crop").expect("crop applied").param("left").unwrap().keyframes.len(), 1); + d.rect("properties.crop.left.nextKeyframe"); +} + +/// The Effect Controls time ruler: a click moves the playhead there, the playhead's handle drags, +/// and there is no playhead while it is off the clip. +#[test] +fn effect_controls_ruler_moves_the_playhead() { + let (mut d, clip) = Driver::demo(); + let frame = Tick::from_seconds_f64(1.0 / 23.976).0; + let [x, y, w, h] = d.rect("effectControls.ruler"); + let lane = d.rect("effectControls.lane"); + assert_eq!((x, w), (lane[0], lane[2]), "the ruler spans the keyframe lane"); + let at = |f: f64| clip.start + (f * clip.duration as f64) as i64; + + d.ok("ui.click", json!({"x": x + w * 0.25, "y": y + h * 0.3})); + d.frames(2); + assert!((d.playhead() - at(0.25)).abs() <= frame, "a click on the ruler: {} vs {}", d.playhead(), at(0.25)); + let head = d.rect("effectControls.playhead"); + assert!((head[0] + head[2] / 2.0 - (x + w * 0.25)).abs() <= 2.0, "the handle sits where the playhead is: {head:?}"); + d.shot("keyframes-ruler"); + + // grab the handle and drag it right + let (hx, hy) = (head[0] + head[2] / 2.0, head[1] + head[3] / 2.0); + d.ok("ui.drag", json!({"from": {"x": hx, "y": hy}, "to": {"x": hx + w * 0.5, "y": hy}, "steps": 6})); + d.frames(2); + assert!((d.playhead() - at(0.75)).abs() <= frame, "dragging the handle: {} vs {}", d.playhead(), at(0.75)); + + // the playhead is drawn only on the clip + d.exec("playhead.set", json!({"time": clip.start + clip.duration * 2})); + assert!(d.find("effectControls.playhead").is_none(), "no handle while the playhead is past the clip"); + d.rect("effectControls.ruler"); +} diff --git a/crates/ui-egui/tests/perf_ui.rs b/crates/ui-egui/tests/perf_ui.rs index 55b52874..5b9d6c84 100644 --- a/crates/ui-egui/tests/perf_ui.rs +++ b/crates/ui-egui/tests/perf_ui.rs @@ -1,6 +1,7 @@ //! `perf.stats` through the control channel (headless app under `egui_kittest`): the control //! method and the command id (what MCP `command_run` sends) both return the engine's decode -//! counters plus playback, frame-worker and UI timings. +//! counters plus playback, frame-worker and UI timings. Also how closely the Program monitor's +//! picture follows a value that is dragged and a scrub. use std::sync::mpsc::{Sender, channel}; @@ -8,6 +9,7 @@ use egui_kittest::Harness; use filmcraft_engine::Session; use filmcraft_ui_egui::FilmcraftApp; use filmcraft_ui_egui::control::ControlRequest; +use filmcraft_ui_egui::frames::{FrameKey, Target}; use serde_json::{Value, json}; struct Driver { @@ -49,6 +51,42 @@ impl Driver { } panic!("no reply to {method}"); } + + fn exec(&mut self, command: &str, params: Value) -> Value { + self.ok("engine.execute", json!({"command": command, "params": params})) + } + + /// The frame due in the Program monitor: `shown` at the project's revision and the playhead. + fn due(&self, shown: FrameKey) -> FrameKey { + let s = &self.harness.state().session; + let frame = s.active_sequence().unwrap().settings.frame_rate.frame_at(s.playhead()); + FrameKey { frame, revision: s.revision, ..shown } + } + + /// Wait (without a UI pass) until the frame workers have `key` ready. + fn ready(&self, key: FrameKey) { + for _ in 0..2000 { + if self.harness.state().frames.is_ready(&key) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + panic!("{key:?} was never rendered"); + } + + /// The Program monitor at rest: the picture of the frame due is on screen. + fn settled(&mut self) -> FrameKey { + for _ in 0..2000 { + self.frames(1); + if let Some(k) = self.harness.state().program_picture() + && k == self.due(k) + { + return k; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + panic!("the Program monitor never showed its frame"); + } } #[test] @@ -77,3 +115,78 @@ fn perf_stats_reports_playback_frames_and_decode() { let c = d.ok("engine.execute", json!({"command": "perf.stats"})); assert!(c["frames"]["workers"].as_u64().unwrap() >= 1 && c["decode"].is_object(), "{c}"); } + +/// A value dragged in Effect Controls changes on every UI pass, so the frame the monitor asks for +/// is never ready on the pass that asks. The monitor shows the frame of the pass before (ready by +/// then) and stays one pass behind the mouse; it used to keep the picture from before the drag +/// until the mouse rested. +#[test] +fn program_monitor_follows_a_value_that_changes_every_pass() { + let mut d = Driver::demo(); + let clip = d.exec("sequence.inspect", json!({}))["video"][0]["items"][0]["clip"].as_u64().unwrap(); + d.exec("playhead.set", json!({"seconds": 1.0})); + let at_rest = d.settled(); + let mut before = at_rest; + for i in 1..=12 { + // one change and one UI pass, as a mouse move while dragging the value + d.exec("effects.setParam", json!({"clip": clip, "effect": "motion", "param": "scale", "value": 100.0 - i as f64})); + let due = d.due(at_rest); + assert_ne!(due.revision, before.revision, "each change is a new revision"); + let shown = d.harness.state().program_picture().unwrap(); + assert!(shown.revision >= before.revision, "change {i}: the picture is at most one change behind, shown {shown:?}, due {due:?}"); + d.ready(due); + before = due; + } + assert_eq!(d.settled().revision, before.revision, "at rest the picture is the frame due"); +} + +/// The same for a scrub, backwards: every pass asks for an earlier frame, and no cached frame +/// lies before it. +#[test] +fn program_monitor_follows_a_backward_scrub() { + let mut d = Driver::demo(); + d.exec("playhead.set", json!({"frame": 60})); + let at_rest = d.settled(); + let mut before = at_rest; + for i in 1..=12 { + d.exec("playhead.set", json!({"frame": 60 - i * 3})); + let due = d.due(at_rest); + assert_eq!(due.frame, 60 - i * 3); + let shown = d.harness.state().program_picture().unwrap(); + assert!(shown.frame <= before.frame, "step {i}: the picture is at most one step behind, shown {shown:?}, due {due:?}"); + d.ready(due); + before = due; + } + assert_eq!(d.settled().frame, before.frame); +} + +/// A frame being rendered is kept when the same frame is asked for at a newer revision: frames +/// that take longer than a refresh (CPU effects) were each cancelled by the next step of a drag, +/// so none was ever shown until the mouse rested. +#[test] +fn a_newer_revision_keeps_the_frame_being_rendered() { + let d = Driver::demo(); + let app = d.harness.state(); + let idle = || { + for _ in 0..200_000 { + if app.frames.queue_len() == 0 { + return; + } + std::thread::sleep(std::time::Duration::from_micros(50)); + } + panic!("the frame queue never emptied"); + }; + // the clip with Lumetri Color at full resolution: tens of milliseconds on the CPU + let seq = app.session.state.active_sequence.unwrap(); + let rate = app.session.active_sequence().unwrap().settings.frame_rate; + let frame = rate.frame_at(filmcraft_time::Tick::from_seconds_f64(15.0)); + let key = |revision| FrameKey { target: Target::Sequence(seq), frame, size: 1000, revision, draft: false }; + let project = app.session.project.clone(); + idle(); + app.frames.request(key(9001), rate.tick_of(frame), 1.0, &project, 0); + idle(); // a worker has taken it + app.frames.request(key(9002), rate.tick_of(frame), 1.0, &project, 0); + // both arrive, in either order: the one that was rendering was not cancelled for the newer + d.ready(key(9002)); + d.ready(key(9001)); +} diff --git a/crates/ui-egui/tests/project_panel_ui.rs b/crates/ui-egui/tests/project_panel_ui.rs index 6188afea..bb504f32 100644 --- a/crates/ui-egui/tests/project_panel_ui.rs +++ b/crates/ui-egui/tests/project_panel_ui.rs @@ -492,6 +492,34 @@ fn media_dir(tag: &str) -> std::path::PathBuf { dir } +#[test] +fn file_dialog_import_uses_shown_bin_unless_explicit() { + let dir = media_dir("import-bin"); + let path = dir.join("a-interview.wav").to_string_lossy().into_owned(); + let mut app = FilmcraftApp::new(Session::default()); + app.hooks.pick_files = Some(Box::new(move |_| vec![path.clone()])); + let parent = app.session.execute("file.newBin", json!({"name": "Footage"})).unwrap()["bin"].as_u64().unwrap(); + let child = app.session.execute("file.newBin", json!({"name": "Selects", "parent": parent})).unwrap()["bin"].as_u64().unwrap(); + let root = app.session.project.root.id.0; + app.ui.project_panel.tabs.push(filmcraft_ui_egui::panels::project::BinTab { bin: parent, ..Default::default() }); + for (shown, tab, params, expected) in [ + (None, None, json!({}), root), + (Some(child), None, json!({}), child), + (Some(child), Some(0), json!({}), parent), + (Some(child), None, json!({"bin": null}), root), + (Some(parent), None, json!({"bin": child}), child), + ] { + app.ui.project_panel.bin = shown; + app.ui.project_panel.active_tab = tab; + let result = app.file_dialog("file.import", ¶ms).unwrap(); + assert!(result["errors"].as_array().unwrap().is_empty(), "{result}"); + let item = ItemId(result["items"][0].as_u64().unwrap()); + let bin = app.session.project.root.find_bin(filmcraft_project::BinId(expected)).unwrap(); + assert!(bin.children.contains(&BinEntry::Item(item)), "shown={shown:?}, tab={tab:?}, params={params}: expected bin {expected}"); + } + std::fs::remove_dir_all(&dir).unwrap(); +} + #[test] fn media_browser_navigation_import_and_columns() { let dir = media_dir("nav"); diff --git a/crates/ui-egui/tests/sequence_ui.rs b/crates/ui-egui/tests/sequence_ui.rs index 807aaad5..05632817 100644 --- a/crates/ui-egui/tests/sequence_ui.rs +++ b/crates/ui-egui/tests/sequence_ui.rs @@ -1,6 +1,9 @@ //! Headless UI tests of the Sequence / Markers menu additions: the Add Tracks and Delete Tracks dialogs, the //! through-edit marks on the timeline, the Markers panel colour filter and the Shift+; gap key. //! The real `FilmcraftApp` under `egui_kittest`, driven over the control channel by automation id. +//! +//! Set `FILMCRAFT_UI_SNAPSHOT_DIR=` to also render the window offscreen with wgpu and write +//! `sequence-settings-*.png` there; without it no GPU is needed. use std::sync::mpsc::{Sender, channel}; @@ -13,6 +16,7 @@ use serde_json::{Value, json}; struct Driver { harness: Harness<'static, FilmcraftApp>, tx: Sender, + snapshots: Option, } impl Driver { @@ -21,8 +25,13 @@ impl Driver { session.execute("file.openDemoProject", json!({})).expect("demo project"); let (tx, rx) = channel(); let app = FilmcraftApp::new(session).with_control(rx); - let harness = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000).build_eframe(move |_cc| app); - let mut d = Driver { harness, tx }; + let snapshots = std::env::var_os("FILMCRAFT_UI_SNAPSHOT_DIR").map(std::path::PathBuf::from); + let mut b = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000); + if snapshots.is_some() { + b = b.wgpu(); + } + let harness = b.build_eframe(move |_cc| app); + let mut d = Driver { harness, tx, snapshots }; d.frames(4); d } @@ -80,6 +89,34 @@ impl Driver { e["label"].as_str().unwrap_or_default().to_string() } + /// With `FILMCRAFT_UI_SNAPSHOT_DIR` set, render the window offscreen and write `.png`, + /// cropped to the elements whose ids start with `prefix`. + fn snapshot(&mut self, name: &str, prefix: &str) { + let Some(dir) = self.snapshots.clone() else { return }; + let v = self.ok("ui.elements", json!({"prefix": prefix})); + let mut bb = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; + for e in v.as_array().unwrap() { + let r: Vec = e["rect"].as_array().unwrap().iter().map(|x| x.as_f64().unwrap() as f32).collect(); + bb = [bb[0].min(r[0]), bb[1].min(r[1]), bb[2].max(r[0] + r[2]), bb[3].max(r[1] + r[3])]; + } + self.frames(2); + let img = match self.harness.render() { + Ok(i) => i, + Err(e) => { + eprintln!("snapshot {name} skipped: {e}"); + return; + } + }; + let ppp = img.width() as f32 / 1600.0; + let x0 = ((bb[0] - 24.0) * ppp).max(0.0) as u32; + let y0 = ((bb[1] - 40.0) * ppp).max(0.0) as u32; + let x1 = (((bb[2] + 24.0) * ppp) as u32).min(img.width()); + let y1 = (((bb[3] + 24.0) * ppp) as u32).min(img.height()); + let img = image::imageops::crop_imm(&img, x0, y0, x1.saturating_sub(x0), y1.saturating_sub(y0)).to_image(); + std::fs::create_dir_all(&dir).unwrap(); + img.save(dir.join(format!("{name}.png"))).unwrap(); + } + fn ids(&mut self, prefix: &str) -> Vec { let v = self.ok("ui.elements", json!({"prefix": prefix})); v.as_array().unwrap().iter().filter_map(|e| e["id"].as_str().map(str::to_string)).collect() @@ -419,3 +456,104 @@ fn ctrl_x_cuts_clips() { assert!(!d.app().session.state.clipboard.is_empty()); assert!(d.app().session.active_sequence().unwrap().find_item(first.id).is_none(), "the cut clip leaves the timeline"); } + +/// Sequence ▸ Sequence Settings… opens Premiere Pro's dialog (it used to do nothing, #172): the +/// General tab shows the sequence's settings, the ones FilmCraft can't change yet are shown (greyed +/// out), and the Color Management and VR Properties tabs are there. Opening and cancelling change +/// nothing. +#[test] +fn sequence_settings_dialog_shows_the_sequence() { + let mut d = Driver::demo(); + let before = (*d.app().session.project).clone(); + let history = d.app().session.history.undo.len(); + let r = d.ok("ui.menu.invoke", json!({"id": "sequence.settings"})); + assert_eq!(r["dialog"], "sequenceSettings", "{r}"); + d.frames(3); + for (id, shown) in [ + ("sequenceSettings.editingMode", "Custom"), + ("sequenceSettings.timebase", "23.976 frames/second"), + ("sequenceSettings.width", "1920"), + ("sequenceSettings.height", "1080"), + ("sequenceSettings.aspect", "16:9"), + ("sequenceSettings.scaleMotion", "true"), + ("sequenceSettings.par", "Square Pixels (1.0)"), + ("sequenceSettings.fields", "No Fields (Progressive Scan)"), + ("sequenceSettings.videoDisplay", "23.976 fps Timecode"), + ("sequenceSettings.channelFormat", "Stereo"), + ("sequenceSettings.channels", "2"), + ("sequenceSettings.sampleRate", "48000 Hz"), + ("sequenceSettings.maxRenderQuality", "false"), + ("sequenceSettings.linearColor", "true"), + ] { + assert_eq!(d.label(id), shown, "{id}"); + } + for id in ["sequenceSettings.tab.general", "sequenceSettings.tab.color", "sequenceSettings.tab.vr", "sequenceSettings.ok", "sequenceSettings.cancel"] { + assert!(d.ids(id).iter().any(|i| i == id), "{id} missing"); + } + d.snapshot("sequence-settings-general", "sequenceSettings."); + // the timebase list offers the common rates + d.click("sequenceSettings.timebase"); + let rates = d.ids("sequenceSettings.timebase.option."); + for r in ["24000/1001", "24/1", "25/1", "30000/1001", "60/1"] { + assert!(rates.iter().any(|i| i.ends_with(r)), "{r} missing: {rates:?}"); + } + d.click("sequenceSettings.timebase.option.24000/1001"); + d.click("sequenceSettings.tab.color"); + assert_eq!(d.label("sequenceSettings.workingSpace"), "Rec. 709"); + assert_eq!(d.label("sequenceSettings.autoToneMap"), "true"); + d.snapshot("sequence-settings-color", "sequenceSettings."); + d.click("sequenceSettings.tab.vr"); + assert_eq!(d.label("sequenceSettings.vr.projection"), "None"); + d.snapshot("sequence-settings-vr", "sequenceSettings."); + d.click("sequenceSettings.cancel"); + assert!(d.ids("sequenceSettings.").is_empty(), "dialog closed"); + assert_eq!(*d.app().session.project, before); + assert_eq!(d.app().session.history.undo.len(), history, "no undo step"); + // OK without a change closes it and adds no undo step either + d.ok("ui.menu.invoke", json!({"id": "sequence.settings"})); + d.frames(3); + d.click("sequenceSettings.ok"); + assert!(d.ids("sequenceSettings.").is_empty()); + assert_eq!(d.app().session.history.undo.len(), history); +} + +/// OK applies the dialog in one undo step: a new timebase (29.97 starts on drop-frame timecode), a +/// smaller frame with the clips' Motion scaled to match, and Maximum Render Quality. +#[test] +fn sequence_settings_dialog_applies_in_one_undo_step() { + let mut d = Driver::demo(); + let before = (*d.app().session.project).clone(); + let motion = |d: &mut Driver| { + let q = d.app().session.active_sequence().unwrap(); + let m = q.video_tracks.iter().flat_map(|t| &t.items).find_map(|i| i.effect("motion")).unwrap().clone(); + match (&m.param("position").unwrap().value, &m.param("scale").unwrap().value) { + (filmcraft_engine::project::ParamValue::Vec2(p), filmcraft_engine::project::ParamValue::Float(s)) => (p.x, p.y, *s), + _ => panic!("Motion has a point and a number"), + } + }; + let (x0, y0, s0) = motion(&mut d); + d.ok("ui.menu.invoke", json!({"id": "sequence.settings"})); + // the window sizes itself over its first frames; click once it has settled + d.frames(10); + d.click("sequenceSettings.timebase"); + d.snapshot("sequence-settings-timebase-list", "sequenceSettings."); + d.click("sequenceSettings.timebase.option.30000/1001"); + assert_eq!(d.label("sequenceSettings.timebase"), "29.97 frames/second"); + assert_eq!(d.label("sequenceSettings.videoDisplay"), "29.97 fps Drop-Frame Timecode"); + // the frame-size fields are drag values; set the draft as typing would + d.app().ui.sequence_settings.width = 1280; + d.app().ui.sequence_settings.height = 720; + d.frames(2); + assert_eq!(d.label("sequenceSettings.aspect"), "16:9"); + d.click("sequenceSettings.maxRenderQuality"); + d.click("sequenceSettings.ok"); + assert!(d.ids("sequenceSettings.").is_empty(), "dialog closed"); + let st = d.app().session.active_sequence().unwrap().settings.clone(); + assert_eq!((st.width, st.height, st.frame_rate), (1280, 720, filmcraft_engine::time::FrameRate::FPS_29_97)); + assert!(st.drop_frame && st.max_render_quality); + let (x1, y1, s1) = motion(&mut d); + let k = 2.0 / 3.0; + assert!((x1 - x0 * k).abs() < 1e-9 && (y1 - y0 * k).abs() < 1e-9 && (s1 - s0 * k).abs() < 1e-9, "{x0},{y0},{s0} -> {x1},{y1},{s1}"); + d.exec("edit.undo", json!({})); + assert_eq!(*d.app().session.project, before, "one undo step for the whole dialog"); +} diff --git a/crates/ui-egui/tests/timeline_trim_ui.rs b/crates/ui-egui/tests/timeline_trim_ui.rs new file mode 100644 index 00000000..8ea07cd2 --- /dev/null +++ b/crates/ui-egui/tests/timeline_trim_ui.rs @@ -0,0 +1,300 @@ +//! Headless UI tests of trimming in the Timeline (#259): a press acts on what was under the +//! pointer when the button went down, the side of a cut picks the clip, and the cursor, +//! `ui.timeline.hit` and the press agree. The real `FilmcraftApp` under `egui_kittest` with the +//! demo project, the pointer moving a few pixels per frame as a mouse does. + +use std::sync::mpsc::{Sender, channel}; + +use egui::{CursorIcon, Pos2, pos2}; +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_project::ClipId; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use serde_json::{Value, json}; + +/// Two clips of the demo project that share a cut on V1. +const CITY: &str = "City_Night_Drive.mp4"; +const DUNES: &str = "Desert_Dunes.mp4"; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, +} + +impl Driver { + /// The demo project, snapping off so drags land exactly where the pointer goes. Frames are + /// 1/60 s apart, as on a display: at the harness's default 1/4 s, a press held for four frames + /// is already a drag by time (egui's 0.8 s click limit), wherever the pointer is. + fn demo() -> Self { + let mut session = Session::default(); + session.execute("file.openDemoProject", json!({})).expect("demo project"); + session.execute("sequence.snap", json!({"on": false})).expect("snapping off"); + let (tx, rx) = channel(); + let app = FilmcraftApp::new(session).with_control(rx); + let harness = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_step_dt(1.0 / 60.0).with_max_steps(10_000).build_eframe(move |_cc| app); + let mut d = Driver { harness, tx }; + d.frames(4); + // `ui.elements` answers once the timeline has finished fitting the sequence + d.ok("ui.elements", json!({"prefix": "timeline.clip."})); + d + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + + fn call(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(v) = reply.try_recv() { + return v; + } + } + panic!("no reply to {method} {params}"); + } + + fn ok(&mut self, method: &str, params: Value) -> Value { + let v = self.call(method, params.clone()); + assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}"); + v["result"].clone() + } + + fn exec(&mut self, command: &str, params: Value) -> Value { + self.ok("engine.execute", json!({"command": command, "params": params})) + } + + fn app(&self) -> &FilmcraftApp { + self.harness.state() + } + + /// Press at `from`, move to `to` over `steps` frames (as a mouse does), release. + fn drag(&mut self, from: Pos2, to: Pos2, steps: usize) { + let push = |d: &mut Self, e: egui::Event| d.harness.input_mut().events.push(e); + push(self, egui::Event::PointerMoved(from)); + self.frames(1); + push(self, egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + self.frames(1); + for i in 1..=steps { + push(self, egui::Event::PointerMoved(from + (to - from) * (i as f32 / steps as f32))); + self.frames(1); + } + push(self, egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + self.frames(2); + } + + /// Move the pointer to `at` with no button down, and let the Timeline react. + fn hover(&mut self, at: Pos2) { + self.harness.input_mut().events.push(egui::Event::PointerMoved(at)); + self.frames(2); + } + + /// The cursor the last frame asked for. + fn cursor(&self) -> CursorIcon { + self.harness.output().platform_output.cursor_icon + } + + /// Render the window to `/ui-screenshots/.png` (needs a GPU). + fn screenshot(&mut self, name: &str) -> std::path::PathBuf { + self.frames(2); + let img = self.harness.render().expect("render"); + let png = filmcraft_ui_egui::control::encode_png(img.as_raw(), img.width(), img.height()).expect("png"); + let dir = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("ui-screenshots"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{name}.png")); + std::fs::write(&path, png).unwrap(); + eprintln!("screenshot: {}", path.display()); + path + } + + /// The clip named `name` on V1. + fn v1_clip(&self, name: &str) -> u64 { + let seq = self.app().session.active_sequence().expect("sequence"); + seq.video_tracks[0].items.iter().find(|i| i.name == name).unwrap_or_else(|| panic!("no {name} on V1")).id.0 + } + + /// A clip's (start, end) in ticks. + fn span(&self, clip: u64) -> (i64, i64) { + let seq = self.app().session.active_sequence().expect("sequence"); + let (_, it) = seq.find_item(ClipId(clip)).expect("clip"); + (it.start.0, it.end().0) + } + + /// Screen x of a clip's In and Out edges, and a y 8 px below the top of its track: the band + /// where edges are grabbed, above the lower part of a track where a transition on a cut takes + /// the press (the demo has one on the City_Night_Drive / Desert_Dunes cut). + fn edges(&mut self, clip: u64) -> (f32, f32, f32) { + // `ui.timeline.locate` answers with points 2 px inside the edges + let i = self.ok("ui.timeline.locate", json!({"clip": clip, "edge": "in"})); + let o = self.ok("ui.timeline.locate", json!({"clip": clip, "edge": "out"})); + let f = |v: &Value, k: &str| v[k].as_f64().expect("number") as f32; + let id = format!("timeline.clip.{clip}"); + let els = self.ok("ui.elements", json!({"prefix": id})); + let el = els.as_array().expect("elements").iter().find(|e| e["id"] == id.as_str()).unwrap_or_else(|| panic!("no {id}: {els}")).clone(); + let top = el["rect"][1].as_f64().expect("rect") as f32; + (f(&i, "x") - 2.0, f(&o, "x") + 2.0, top + 8.0) + } + + /// Ticks per screen pixel, from a clip's length on screen. + fn ticks_per_px(&mut self, clip: u64) -> f64 { + let (s, e) = self.span(clip); + let (x0, x1, _) = self.edges(clip); + (e - s) as f64 / f64::from(x1 - x0) + } + + /// One frame of the sequence, in ticks. + fn frame(&self) -> i64 { + self.app().session.active_sequence().expect("sequence").settings.frame_rate.frame_duration().0 + } + + fn undo_steps(&self) -> usize { + self.app().session.history.undo.len() + } +} + +/// #259: grabbing a clip's Out edge 3 px inside and dragging inward trims it, by as far as the +/// pointer moved. The drag is recognised only once the pointer has moved past egui's 6 pt drag +/// threshold, about 10 px inside the clip and so outside the 7 px edge zone, where the clip used to +/// be selected and moved instead. +#[test] +fn dragging_an_out_edge_inward_trims_the_clip() { + let mut d = Driver::demo(); + let city = d.v1_clip(CITY); + let (x0, x1, y) = d.edges(city); + let (s0, e0) = d.span(city); + let (tpp, frame, undo0) = (d.ticks_per_px(city), d.frame(), d.undo_steps()); + let by = ((x1 - x0) / 4.0).round(); + d.drag(pos2(x1 - 3.0, y), pos2(x1 - 3.0 - by, y), by as usize); + let (s1, e1) = d.span(city); + assert_eq!(s1, s0, "the In edge stayed: trimmed, not moved"); + assert!(e1 < e0, "the Out edge moved in: {e0} → {e1}"); + let want = f64::from(by) * tpp; + assert!(((e0 - e1) as f64 - want).abs() <= frame as f64, "the edge moved {} ticks for {by} px ({want} ticks) ± a frame ({frame})", e0 - e1); + assert_eq!(d.undo_steps(), undo0 + 1, "one undo step"); +} + +/// Just right of a cut the clip on the right is grabbed: its In edge trims, and the clip on the +/// left stays as it was. The left clip's Out edge used to claim both sides of the cut. +#[test] +fn right_of_a_cut_trims_the_right_clip() { + let mut d = Driver::demo(); + let (city, dunes) = (d.v1_clip(CITY), d.v1_clip(DUNES)); + assert_eq!(d.span(city).1, d.span(dunes).0, "the demo's City_Night_Drive and Desert_Dunes share a cut"); + let (cut, _, y) = d.edges(dunes); + let (city0, (s0, e0)) = (d.span(city), d.span(dunes)); + d.drag(pos2(cut + 3.0, y), pos2(cut + 43.0, y), 40); + let (s1, e1) = d.span(dunes); + assert!(s1 > s0, "Desert_Dunes' In edge moved right: {s0} → {s1}"); + assert_eq!(e1, e0, "Desert_Dunes' Out edge stayed: trimmed, not moved"); + assert_eq!(d.span(city), city0, "City_Night_Drive is untouched"); +} + +/// A click that wobbles 5.5 px (still a click to egui, under its 6 pt limit) selects the edit point +/// that was pressed. It used to be hit-tested where the button came up, 8.5 px inside the clip and +/// outside the edge zone, and selected the clip. +#[test] +fn a_wobbly_click_on_an_edge_selects_the_edit_point() { + let mut d = Driver::demo(); + let city = d.v1_clip(CITY); + let st = &mut d.harness.state_mut().session.state; + st.selection.clear(); + st.edit_points.clear(); + let (_, x1, y) = d.edges(city); + d.drag(pos2(x1 - 3.0, y), pos2(x1 - 8.5, y), 3); + let st = &d.app().session.state; + assert!(st.edit_points.iter().any(|e| e.clip == ClipId(city) && e.out), "City_Night_Drive's Out edge is the edit point: {:?}", st.edit_points); + assert!(!st.selection.contains(&ClipId(city)), "the clip is not selected"); +} + +/// A moved clip stays under the point it was grabbed by. It used to lag behind the pointer by the +/// distance egui waits before it calls a press a drag. +#[test] +fn a_moved_clip_follows_the_pointer_from_where_it_was_grabbed() { + let mut d = Driver::demo(); + let city = d.v1_clip(CITY); + let (x0, x1, y) = d.edges(city); + let (s0, _) = d.span(city); + let (tpp, frame) = (d.ticks_per_px(city), d.frame()); + let mid = (x0 + x1) / 2.0; + d.drag(pos2(mid, y), pos2(mid + 120.0, y), 30); + let (s1, _) = d.span(city); + let want = 120.0 * tpp; + assert!(((s1 - s0) as f64 - want).abs() <= frame as f64, "moved {} ticks for 120 px ({want} ticks) ± a frame ({frame})", s1 - s0); +} + +/// The cursor, `ui.timeline.hit` and the press agree on what is under the pointer: just left of +/// a cut the left clip's Out edge, just right of it the right clip's In edge, mid-clip no edge. +/// (That a drag from those points trims those edges is tested above, from the same points.) +#[test] +fn cursor_hit_test_and_press_agree_at_a_cut() { + let mut d = Driver::demo(); + let (city, dunes) = (d.v1_clip(CITY), d.v1_clip(DUNES)); + let (x0, _, _) = d.edges(city); + let (cut, _, y) = d.edges(dunes); + let hit = |d: &mut Driver, x: f32, mods: Value| d.ok("ui.timeline.hit", json!({"x": x, "y": y, "modifiers": mods})); + for (x, clip, edge) in [(cut - 3.0, city, "Out"), (cut + 3.0, dunes, "In")] { + d.hover(pos2(x, y)); + assert_eq!(d.cursor(), CursorIcon::ResizeColumn, "trim cursor at x {x}"); + let h = hit(&mut d, x, json!({})); + assert_eq!((h["clip"].as_u64(), h["edge"].as_str(), h["kind"].as_str()), (Some(clip), Some(edge), Some("trim")), "at x {x}: {h}"); + let h = hit(&mut d, x, json!({"command": true})); + assert_eq!(h["kind"], "ripple", "Cmd at x {x}: {h}"); + } + let mid = (x0 + cut) / 2.0; + d.hover(pos2(mid, y)); + assert_eq!(d.cursor(), CursorIcon::Default, "no trim cursor mid-clip"); + let h = hit(&mut d, mid, json!({})); + assert_eq!((h["clip"].as_u64(), h["kind"].clone()), (Some(city), Value::Null), "mid-clip: {h}"); +} + +/// Renders the hover bracket at the City_Night_Drive / Desert_Dunes cut to PNGs for visual +/// review (needs a GPU): `cargo test -p filmcraft-ui-egui --test timeline_trim_ui -- --ignored +/// hover_bracket_screenshots`. Yellow on City_Night_Drive's Out edge just left of the cut and on +/// Desert_Dunes' In edge just right of it, red for the Ripple tool, both sides for the Rolling tool. +#[test] +#[ignore] +fn hover_bracket_screenshots() { + let mut d = Driver::demo(); + let dunes = d.v1_clip(DUNES); + let (cut, _, y) = d.edges(dunes); + for (tool, name, x) in + [("selection", "trim-out", cut - 3.0), ("selection", "trim-in", cut + 3.0), ("ripple", "ripple-in", cut + 3.0), ("rolling", "roll", cut + 3.0)] + { + d.ok("ui.set", json!({"tool": tool})); + d.hover(pos2(x, y)); + d.screenshot(&format!("hover-{name}")); + } +} + +/// A caption's Out edge, grabbed 2 px inside and dragged inward, trims the caption. The gesture +/// used to be decided where the drag was recognised, about 9 px inside (2 px plus egui's 6 pt drag +/// threshold), past the 5 px edge zone, so the caption moved. +#[test] +fn dragging_a_caption_edge_inward_trims_it() { + let mut d = Driver::demo(); + let id = d.exec("captions.add", json!({"seconds": 2.0, "durationSeconds": 2.0}))["caption"].as_u64().expect("caption id"); + d.frames(4); + let name = format!("timeline.caption.{id}"); + let els = d.ok("ui.elements", json!({"prefix": name})); + let el = els.as_array().expect("elements").iter().find(|e| e["id"] == name.as_str()).unwrap_or_else(|| panic!("no {name}: {els}")).clone(); + let r: Vec = el["rect"].as_array().expect("rect").iter().map(|v| v.as_f64().expect("number") as f32).collect(); + let (x1, y) = (r[0] + r[2], r[1] + r[3] / 2.0); + let span = |d: &Driver| { + let seq = d.app().session.active_sequence().expect("sequence"); + let (_, c) = seq.find_caption(ClipId(id)).expect("caption"); + (c.start.0, c.end().0) + }; + let (s0, e0) = span(&d); + d.drag(pos2(x1 - 2.0, y), pos2(x1 - 32.0, y), 30); + let (s1, e1) = span(&d); + assert_eq!(s1, s0, "the caption's start stayed: trimmed, not moved"); + assert!(e1 < e0, "the caption's end moved in: {e0} → {e1}"); +} diff --git a/crates/ui-egui/tests/unlink_ui.rs b/crates/ui-egui/tests/unlink_ui.rs new file mode 100644 index 00000000..6366cee6 --- /dev/null +++ b/crates/ui-egui/tests/unlink_ui.rs @@ -0,0 +1,114 @@ +//! Headless UI test of Unlink from the timeline clip menu (#220): after unlinking, the clip that +//! was right-clicked is the only one selected, so a drag moves it away from its former partner. + +use std::sync::mpsc::{Sender, channel}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_engine::project::{ClipId, TrackItem}; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use serde_json::{Value, json}; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, +} + +impl Driver { + fn new() -> Self { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let (tx, rx) = channel(); + let app = FilmcraftApp::new(s).with_control(rx); + let harness = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000).build_eframe(move |_cc| app); + let mut d = Driver { harness, tx }; + d.frames(4); + d + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + // the control channel's clicks and keys enter through the input hook + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + + fn ok(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(v) = reply.try_recv() { + assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}"); + return v["result"].clone(); + } + } + panic!("no reply to {method} {params}"); + } + + fn app(&mut self) -> &mut FilmcraftApp { + self.harness.state_mut() + } + + fn item(&mut self, c: ClipId) -> TrackItem { + self.app().session.active_sequence().unwrap().find_item(c).unwrap().1.clone() + } + + fn selection(&mut self) -> Vec { + let mut v = self.app().session.state.selection.clone(); + v.sort_by_key(|c| c.0); + v + } + + /// Right-click `clip` on the timeline and choose Link / Unlink from its menu. + fn menu_link(&mut self, clip: ClipId) { + self.ok("ui.click", json!({"id": format!("timeline.clip.{}", clip.0), "button": "right"})); + self.frames(3); + self.ok("ui.click", json!({"id": "timeline.clipMenu.clip.link"})); + self.frames(3); + } + + /// A linked picture on V1 and its sound. + fn linked_pair(&mut self) -> (ClipId, ClipId) { + let seq = self.app().session.active_sequence().unwrap().clone(); + let v = seq.video_tracks[0].items.iter().find(|i| i.link.is_some()).expect("a linked clip"); + let a = seq.audio_tracks.iter().flat_map(|t| &t.items).find(|i| i.link == v.link).expect("its sound"); + (v.id, a.id) + } +} + +#[test] +fn unlinked_picture_drags_away_from_its_sound() { + let mut d = Driver::new(); + let (v, a) = d.linked_pair(); + let (v0, a0) = (d.item(v).start, d.item(a).start); + d.menu_link(v); + assert_eq!((d.item(v).link, d.item(a).link), (None, None), "unlinked"); + assert_eq!(d.selection(), [v], "only the right-clicked clip stays selected"); + // dragging the picture straight away leaves the sound where it was + let id = format!("timeline.clip.{}", v.0); + d.ok("ui.drag", json!({"from": {"id": id, "fx": 0.3}, "to": {"id": id, "fx": 0.6}, "steps": 12})); + d.frames(4); + assert_ne!(d.item(v).start, v0, "the picture moved"); + assert_eq!(d.item(a).start, a0, "its sound stayed"); +} + +#[test] +fn unlinking_from_the_sound_keeps_the_sound_selected_and_link_keeps_both() { + let mut d = Driver::new(); + let (v, a) = d.linked_pair(); + d.menu_link(a); + assert_eq!(d.selection(), [a]); + // Link again: select both, link from the menu; the selection is left alone + d.ok("engine.execute", json!({"command": "timeline.select", "params": {"clips": [v.0, a.0]}})); + d.menu_link(v); + assert!(d.item(v).link.is_some() && d.item(v).link == d.item(a).link, "linked again"); + let mut both = vec![v, a]; + both.sort_by_key(|c| c.0); + assert_eq!(d.selection(), both); +} diff --git a/docs/adr/0001-platform-ffi.md b/docs/adr/0001-platform-ffi.md index d067c4b7..4c6eb7aa 100644 --- a/docs/adr/0001-platform-ffi.md +++ b/docs/adr/0001-platform-ffi.md @@ -25,8 +25,8 @@ Containment rules: 1. The crate does not use `lints.workspace = true`. Its own `[lints]` table copies the workspace lints except `unsafe_code = "deny"` (not `forbid`), and adds - `clippy::undocumented_unsafe_blocks = "deny"`. Only the FFI modules (`videotoolbox`, `media_foundation::gpu` / `media_foundation::mft`, and - `nvenc::ffi` / `nvenc::session` on Windows) carry `#[allow(unsafe_code)]`; the rest of the crate + `clippy::undocumented_unsafe_blocks = "deny"`. Only the FFI modules (`videotoolbox`, `media_foundation::gpu` / `media_foundation::mft`, + `nvenc::ffi` / `nvenc::session` on Windows, and `vaapi::va` on Linux) carry `#[allow(unsafe_code)]`; the rest of the crate (the fallback logic in `hybrid`, the decoder logic in `media_foundation`, `annexb`, `biplanar`, the encoder logic in `nvenc`) has no `unsafe`. 2. Every `unsafe` block has a `// SAFETY:` comment saying why it is sound. @@ -101,3 +101,28 @@ there will not be one (pure Rust, clean-room: x265 is GPL), so `Format::Hevc` ex OS has a hardware encoder. Choosing the format is the opt-in, `filmcraft_export::available` asks a probe the platform crate registers, and what the hardware path does not take (two-pass, odd sizes, a machine without the encoder) is an error naming the reason instead of a different encoder. + +## Addendum (2026-10-08): VA-API hardware decoding on Linux + +The Linux backend this decision named: H.264 and HEVC decoding through VA-API (`vaapi/`). The rules above +hold; VA-API differs from the other two backends in three ways. + +- **libva is loaded at run time** (`libva.so.2`, `libva-drm.so.2`, through `libloading`: ISC, already + in the lockfile through wgpu). Building needs no libva headers, and a system without libva, a DRM + render node or a working driver starts as before and decodes in software (`register()` reports + `Unavailable`). The declarations (`vaapi/ffi.rs`) are transcribed from libva's MIT-licensed + `va.h`, and `vaapi/abi_tests.rs` checks sizes, offsets, constants and bit-field positions against a + C compiler's view of it, as for NVENC. Only `vaapi::va` carries `#[allow(unsafe_code)]`; the + declarations, the H.264 front end (`vaapi::h264`) and the decoder (`VaDecoder`) are safe code. +- **Decoding is stateless:** the host parses the stream and keeps the decoded picture buffer; the GPU + only decodes slice data. FilmCraft does that host side with the software decoders' own parsers and + DPBs (`filmcraft_h264::dpb` and `filmcraft_hevc::dpb` are generic over what a picture is), so the two decoders decide alike by + construction, and the front end is tested on every OS against a stand-in for the hardware. The + one place they cannot agree is concealment: pictures predicted from frames that were never decoded + (the leading pictures of an open H.264 GOP after a seek) are concealed by both but can differ; + HEVC pictures with references that were never decoded go to the software decoder instead. +- **One callback into Rust:** libva's error messages go to our log through a callback that runs under + `catch_unwind`; its info messages are turned off. + +The fallback guarantee is unchanged: what the hardware path does not take is declined up front or, +mid-stream, handed to the software decoder by `HybridDecoder`. diff --git a/docs/agents.md b/docs/agents.md index 2f02bc91..9ad0b2bc 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -166,6 +166,9 @@ Notes: Every command is also one shell call away. Options go anywhere; output is JSON; exit status is 0 on success, 1 when a command fails and 2 on a usage error. `filmcraft-cli help` prints the reference. +If the reader of stdout closes the pipe early (`filmcraft-cli commands | head`), the CLI drops the rest +of its output but still finishes the work, saves included, and the exit status still reports +failures; any other stdout write error is reported and makes the status 1. ```sh filmcraft-cli commands razor # find ids (add --json for machine output) diff --git a/docs/architecture.md b/docs/architecture.md index e0a7885c..ede9d5e8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -72,7 +72,7 @@ and `filmcraft-cli`. | `engine` | L4 | `Session`, command registry, undo history, media pool, jobs, interchange glue | | `ui-egui` | L5 | the egui frontend: docking, panels, timeline, monitors, playback, control-channel handlers | | `automation` | L5 | MCP server (`rmcp`, stdio), headless or bridged to the running app | -| `platform` | L5 | OS media FFI only: hardware video decoding (VideoToolbox H.264 / HEVC on macOS; Media Foundation / Direct3D 11 H.264 / HEVC on Windows; a no-op elsewhere) behind `codecs::VideoDecoder`, with transparent fallback to our decoders, and hardware H.264 encoding (VideoToolbox, opt-in) and H.265 encoding (VideoToolbox, the only H.265 encoder; the format exists only where a hardware encoder does) and NVIDIA NVENC H.264 encoding (Windows, opt-in) behind `export::VideoEncoder`; H.264 declines to the built-in encoder for what the hardware does not take. The one crate allowed `unsafe` ([ADR 0001](adr/0001-platform-ffi.md), [README](../crates/platform/README.md)) | +| `platform` | L5 | OS media FFI only: hardware video decoding (VideoToolbox H.264 / HEVC on macOS; Media Foundation / Direct3D 11 H.264 / HEVC / VP9 / AV1 on Windows; VA-API H.264 / HEVC on Linux, with our own parsers and DPBs driving the stateless hardware; a no-op elsewhere) behind `codecs::VideoDecoder`, with transparent fallback to our decoders, and hardware H.264 encoding (VideoToolbox, opt-in) and H.265 encoding (VideoToolbox, the only H.265 encoder; the format exists only where a hardware encoder does) and NVIDIA NVENC H.264 encoding (Windows, opt-in) behind `export::VideoEncoder`; H.264 declines to the built-in encoder for what the hardware does not take. The one crate allowed `unsafe` ([ADR 0001](adr/0001-platform-ffi.md), [README](../crates/platform/README.md)) | | `filmcraft` | L6 | desktop binary: eframe/wgpu window, cpal audio output, file dialogs, native macOS menu, TCP control server | | `filmcraft-cli` | L6 | headless CLI: `exec`, `run`, `inspect`, `describe`, `commands`, `import`, `export`, `render`, `probe`, `mcp`; `--bridge` targets the running app | | `filmcraft-web` | L6 | the browser app (wasm32): eframe web runner on WebGPU/WebGL2, Blob-backed services, OPFS recovery, WebAudio, WebCodecs, `window.filmcraft` API ([web.md](web.md)) | @@ -450,7 +450,7 @@ Clip ▸ Remix ▸ Enable Remix / Remix Properties… / Revert Remix (`clip.remi ```text clip.essential (type + settings) ──apply()──► clip effects marked `essential`, clip gain, Volume, Panner -essentialSound.autoMatch BS.1770 integrated loudness of render::audio::clip_signal → match gain (clip gain) +essentialSound.autoMatch BS.1770 integrated loudness of render::audio::clip_signal (mono clips as one channel) → match gain (clip gain) essentialSound.generateDucking trigger clips' summed level (10 ms hops) → activity regions → Volume keyframes ``` diff --git a/docs/contributing.md b/docs/contributing.md index 5bfaf1cf..38c51d85 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -36,8 +36,13 @@ cargo run --release -p filmcraft-cli -- mcp --demo # headless MCP server on s cargo xtask web --serve 8765 # the web app on http://127.0.0.1:8765/ (docs/web.md) ``` -`FILMCRAFT_CONTROL_PORT=9876` works like `--control 9876`, and `FILMCRAFT_CPU_COMPOSITE=1` -disables the GPU compositor. +| Variable | Effect | +|---|---| +| `FILMCRAFT_CONTROL_PORT` | Same as `--control ` | +| `FILMCRAFT_CPU_COMPOSITE=1` | Disables the GPU compositor | +| `FILMCRAFT_DATA_DIR` | Same as `--data-dir `: auto-save, crash recovery, settings and logs | +| `RUST_LOG` | Log levels for standard error and the log file (see [Logs](#logs)) | + `filmcraft --help` lists the app's options. An option it does not know, or a control port that is not a number (from `--control` or from `FILMCRAFT_CONTROL_PORT`), is an error on stderr with exit code 2 instead of a window. @@ -45,6 +50,25 @@ code 2 instead of a window. Dev builds compile dependencies at `opt-level = 2` and workspace crates at `opt-level = 1`. For playback and codec speed, use `--release`. +### Logs + +The desktop app writes its `log` records to standard error and to `Logs/filmcraft.log` in the data +directory (`~/.local/share/filmcraft/Logs/` or `$XDG_DATA_HOME/filmcraft/Logs/` on Linux, +`~/Library/Application Support/FilmCraft/Logs/` on macOS, `%APPDATA%\FilmCraft\Logs\` on Windows, +or under `--data-dir` / `FILMCRAFT_DATA_DIR`), next to the crash logs; Help ▸ Reveal Log Files opens +the folder. A start from a desktop menu or the Dock has no terminal, so this file is what to attach +to a bug report: GPU compositor fallbacks, audio output problems, panicked frame jobs and container +warnings all land there. Each launch moves the previous log to `filmcraft.1.log` (and that one to +`filmcraft.2.log`), so the log of a run that crashed survives the next start. The file stops growing +at 16 MiB. `--help`, `--version` and command-line errors write no file. + +By default FilmCraft's own crates log at `info` and everything else at `warn`. `RUST_LOG` replaces +that with env_logger-style directives, for example `RUST_LOG=debug`, +`RUST_LOG=warn,filmcraft_codecs=trace` or `RUST_LOG=info,wgpu_core=warn`; a directive ending in `*` +covers every target starting with it (`filmcraft*=debug`). The logger is +`apps/filmcraft/src/logging.rs`; the realtime audio callbacks do not log. The web app logs to the +browser console instead. + ### Building with craft-fonts Font assets live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), never in this diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 6419e154..5ee02434 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -34,12 +34,12 @@ Methods (handlers in `crates/ui-egui/src/control.rs`): | `ui.click` / `ui.move` | `{id}` or `{x,y}`, `button`, `count`, `modifiers` | synthetic pointer input | | `ui.drag` | `{from, to, steps, modifiers}` | press–move–release | | `ui.scroll` | `{id|x,y, dx, dy, modifiers}` | wheel / trackpad | -| `ui.key` / `ui.type` | `{key}` (`Cmd+K`, `Space`…) / `{text}` | keyboard | -| `ui.timeline.hit` / `ui.timeline.locate` | `{x,y}` / `{clip, edge?}` | timeline hit-testing | +| `ui.key` / `ui.type` | `{key}` (`Cmd+K`, `Space`…; off macOS `Ctrl+K` is the same key) / `{text}` | keyboard | +| `ui.timeline.hit` / `ui.timeline.locate` | `{x,y,modifiers?}` / `{clip, edge?}` | timeline hit-testing. `hit` reports a clip's `edge` (`In`/`Out`) within 7 px of it (a third of a narrow clip); just left of a cut that is the left clip's Out, from the cut rightwards the right clip's In. `kind` is the trim a press of the current tool with `modifiers` starts there (`trim`, `ripple`, `roll`, `rateStretch`, `remix`), or null | | `ui.playback` | `{action: play|stop|toggle, speed?}` | | | `ui.screenshot` | `{path?, panel?}` | PNG of the window or one panel; fails after 10 s when no frame is presented (window hidden, display asleep) | | `ui.resize`, `ui.focus`, `app.quit` | | `ui.focus` is the only request that activates the app and takes keyboard focus | -| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, `draftFrames` (decoded in draft mode), `h264Threads` (frame-threading workers per H.264 decoder), `liveDecoders` (sources holding a decoder now), `cacheMB` / `cacheBudgetMB` (decoded frames all sources hold, and the shared budget), `planePoolMB` / `planesReused` (idle plane buffers of evicted frames, and planes decoded into a recycled one), decoder ms (total and per sample), `framesDecoded`, `hardware` (Settings ▸ Playback ▸ Hardware decoding: `enabled`, pictures from hardware decoders `frames` vs `softwareFrames`, `sessions`, `declined` streams, mid-stream `fallbacks`); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution, `draftDecode` (Settings ▸ Playback ▸ Draft decoding), `audio` (desktop: sound is mixed 200 ms ahead of the device; device `callbacks`, `underruns` and `missingFrames` played as silence because the mixer fell behind, `minLeadMs` the least sound left buffered); `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | +| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, `draftFrames` (decoded in draft mode), `h264Threads` (frame-threading workers per H.264 decoder), `liveDecoders` (sources holding a decoder now), `cacheMB` / `cacheBudgetMB` (decoded frames all sources hold, and the shared budget), `planePoolMB` / `planesReused` (idle plane buffers of evicted frames, and planes decoded into a recycled one), decoder ms (total and per sample), `framesDecoded`, `hardware` (Settings ▸ Playback ▸ Hardware decoding: `enabled`, pictures from hardware decoders `frames` vs `softwareFrames`, `sessions`, `declined` streams, mid-stream `fallbacks`); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution, `draftDecode` (Settings ▸ Playback ▸ Draft decoding), `audio` (desktop: sound is mixed 200 ms ahead of the device; device `callbacks`, `underruns` and `missingFrames` played as silence because the mixer fell behind, `minLeadMs` the least sound left buffered); `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `monitor`: the `frame` and `revision` of the picture the Program monitor shows against `playheadFrame` and `projectRevision` (how far a drag or a scrub is ahead of the picture); `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | **Selection or explicit targets.** A command that acts on the selection is "not available right now" when nothing is selected. When `params` name the targets under a key the command documents (`clips` / diff --git a/docs/performance.md b/docs/performance.md index 5fd62d73..e6ac4eaf 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -349,6 +349,48 @@ Software AV1 decodes 4K at 5.5 fps here, so it never plays in real time; with th plays without a drop. As on the other codecs, the hardware ignores draft mode and what is left on the CPU is the readback and plane conversion. +## Results (Linux VA-API H.264 and HEVC hardware decoding, Off → Auto) + +Same commit, Settings ▸ Playback ▸ Hardware decoding switched with the bench flag +(`cargo xtask bench --sections decode --only dec_h --repeat 3 --hw off|auto`), 2026-10-08, Intel +Core i5-13500H with Iris Xe graphics (Raptor Lake-P), 7.4 GB RAM, Intel iHD driver 26.1.2 through +libva 2.22, load average 5–15. Only H.264 goes through VA-API so far; the HEVC rows decode in +software either way and show the run-to-run spread. The decoded pictures are identical +(bit-exact parity tests, `crates/platform/tests/vaapi.rs`). + +| codec | size | CPU ms/frame Off → **Auto** | fps Off → **Auto** (best of 3) | hw frames | +|---|---|---|---|---| +| H.264 | 1080p | 103.7 → **2.3** | 133 → **263** | 360 | +| H.264 | 2160p | 416.5 → **11.4** | 32 → **56** | 216 | +| HEVC | 1080p | 63.2 → 46.5 | 99 → 131 | 0 | +| HEVC | 2160p | 349.5 → 353.7 | 30 → 35 | 0 | +| HEVC Main 10 | 2160p | 348.9 → 352.6 | 29 → 29 | 0 | + +What is left on the CPU per H.264 frame is the host side of stateless decoding (parsing, DPB, +filling the VA buffers) and the read-back: `vaGetImage` into an NV12 image and the copy into +planar Y'CbCr. One picture is decoded at a time and read back as soon as the DPB outputs it, so +4K throughput (56 fps) is bound by that round trip, not by the video engine; overlapping decode +and read-back, or zero-copy into wgpu, would raise it. + +### HEVC through VA-API (HW5 follow-up) + +Same machine and commands on 2026-10-09 with HEVC added, at a lower load (the software decoders ran +faster than in the table above; compare within a row). H.264 is unchanged; every HEVC stream now +goes through VA-API too (216–360 hardware frames, no fallbacks), bit-exact with the software decoder. + +| codec | size | CPU ms/frame Off → **Auto** | fps Off → **Auto** (best of 3) | +|---|---|---|---| +| H.264 | 1080p | 57.3 → **2.6** | 227 → **241** | +| H.264 | 2160p | 231.8 → **12.6** | 58 → **52** | +| HEVC | 1080p | 45.6 → **2.3** | 141 → **341** | +| HEVC | 2160p | 202.5 → **10.1** | 47 → **76** | +| HEVC Main 10 | 2160p | 211.6 → **18.4** | 42 → **46** | + +At this lower load the multi-threaded software decoders keep up with the one-picture-at-a-time +hardware path on throughput for H.264 and 10-bit HEVC, at 5–20 % of their CPU time; under load +(the run above) the hardware path is ahead on both. 10-bit pictures cost more to read back (P010 is +twice the bytes of NV12 and is shifted down into 16-bit planes). + ## Results (GPU1: blend modes on the GPU compositor, #30, before → after) Before = this change with the old whole-frame CPU fallback for non-Normal blend modes put back diff --git a/docs/project-files.md b/docs/project-files.md index 68980cea..b7db090a 100644 --- a/docs/project-files.md +++ b/docs/project-files.md @@ -159,7 +159,8 @@ Desktop flags: `--recover` recovers the newest candidate without asking. `--no-r without asking; the changes stay available through File ▸ Recover Unsaved Changes…. `--data-dir ` (or `FILMCRAFT_DATA_DIR`) moves the data directory. The default is `~/Library/Application Support/FilmCraft` on macOS, `%APPDATA%\FilmCraft` on Windows, and `$XDG_DATA_HOME/filmcraft` or -`~/.local/share/filmcraft` elsewhere. +`~/.local/share/filmcraft` elsewhere. A `portable.txt` file next to the executable (the Windows +portable zip ships one) makes it `data` next to the executable instead. ### Cost diff --git a/docs/releasing.md b/docs/releasing.md index 04e59e73..14e699c5 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -77,6 +77,9 @@ deliberately, in every job at once. - **Notarization:** the app is sent with `xcrun notarytool submit`, then the ticket is stapled and checked with `stapler validate`. The app goes on a DMG (`hdiutil makehybrid`), which is signed and notarized too. The universal `filmcraft-cli` is signed the same way, zipped, and the zip is notarized. + Its Finder window (background, icon size and positions) comes from + [`packaging/macos/dmg/`](../packaging/macos/dmg/README.md), and its volume is named `FilmCraft` + without the version, which the window's background needs; the DMG file name keeps the version. Locally, without certificates, the script signs ad hoc and skips notarization, which is enough to check the bundle and the DMG on your own Mac (`packaging/macos/package.sh`). @@ -104,6 +107,15 @@ nor the portable zip needs the Visual C++ redistributable. `.tar.gz` tree. The packages install both programs, the desktop entry, the AppStream metainfo, the icons and the licence files. +The Linux tarball includes `install.sh`. Extract it, open a terminal in the extracted +directory and run `./install.sh` to install both programs and their desktop integration +under `~/.local` (no administrator permissions needed). The application-menu entry uses +the installed binary's absolute path; add `~/.local/bin` to `PATH` for terminal commands. +Run the installer again to update. For a system-wide installation, use +`sudo ./install.sh --prefix /usr/local`. A custom absolute `--prefix` is also supported; +its `share` directory must be in the desktop's data search path to appear in the menu. +Prefixes containing `=`, `%` or control characters are rejected for desktop-launcher compatibility. + The jobs run on `ubuntu-22.04`, the oldest GitHub-hosted image, so the binaries only need glibc 2.35 or newer: Ubuntu 22.04+, Debian 12+, Fedora 36+ and RHEL 10. The deb (`libc6 (>= 2.35)`) and the rpm (`glibc >= 2.35`) both declare that floor, so older systems refuse the install. Moving diff --git a/docs/testing.md b/docs/testing.md index a1640bbb..bb57a849 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -20,6 +20,7 @@ FILMCRAFT_REQUIRE_ORACLES=1 cargo test --workspace # CI: missing ffmpeg fails | Monitor view / Graphics menu | `crates/ui-egui/tests/view_ui.rs`, `crates/engine/src/graphics_tests.rs` | View menu items and checkmarks, playback/paused resolution, channel / comparison / multi-camera / waveform display modes, magnification and Hand-tool panning, guides dragged from the rulers (move, lock, remove, Add Guide…, templates in the preferences), snapping a graphic to the frame centre and a guide; align to frame / as group / to selection, distribute (centres and gaps), arrange, select next/previous graphic and layer, reset parameters / duration, vertical text, New Layer from file (`FILMCRAFT_UI_SHOTS=` writes `view-*.png`) | | Settings | `crates/engine/src/settings_tests.rs`, `crates/ui-egui/tests/settings_ui.rs` | schema keys and defaults, persistence in the data directory, v1 → v2 migration and repair of bad values, validation, per-category reset; each wired setting (still / transition durations, step many, label defaults and colours, media scaling, timebase, recent projects, media cache policy, output mapping, smart quotes, auto-transcribe); the dialog: every category from its command and Cmd+,, edits by automation id, OK / Cancel / Escape / Reset…, theme, label names in Edit ▸ Label, tooltips, frame cache (`FILMCRAFT_UI_SHOTS=` writes `settings-*.png`) | | Scopes and panels (M8.9 / M12.6) | `crates/scopes/src/tests.rs`, `crates/engine/src/{scopes,panels}_tests.rs`, `crates/ui-egui/tests/panels_ui.rs` | scope maths on generated frames (flat colours → exact histogram bins, ramps → waveform row = code value, 75 % bars → parade levels and vectorscope target cells for BT.601/709/2020, HLS hue angles, YC chroma, Clamp Signal, decimation, NaN); `scopes.read` on a colour matte (exact bins, 78.43 %) and on Bars and Tone (the six targets); metadata edits (one undo step, read-only fields refused, saved in the project), the event log (failed / disabled commands, repeats, jobs started / finished / failed / cancelled); every panel from Window ▸, the scopes' wrench menu, presets, five-scope grid, typing into a Metadata field, Timecode rows and modes, Events filter / Clear All, Progress cancel, Reference Monitor park / gang / scopes (`FILMCRAFT_UI_SNAPSHOT_DIR=` writes `panels-*.png`); `perf_scopes_at_1080p` (ignored) times each scope | +| Keyframe navigator and Effect Controls ruler | `crates/engine/src/tests.rs` (`add_keyframe_toggles_the_keyframe_at_the_playhead`), `crates/ui-egui/tests/keyframes_ui.rs`, `lane_tests` in `crates/ui-egui/src/panels/effect_controls.rs` | `effects.addKeyframe` adds a keyframe at the playhead or removes the one there (the value falls back to the remaining keyframes; undo, redo, parameters that name nothing); headless: the Properties diamond turns animation on, adds and removes the keyframe at the playhead (Scale 50 % → 100 % again), its ◀ ▶ arrows and Effect Controls' step between keyframes, Crop is applied before its first keyframe; a click on the Effect Controls time ruler and a drag of the playhead's handle move the playhead, and no playhead is drawn off the clip; ruler labels are sequence timecode, and clip times, frame rates and panel widths at their limits don't panic (`FILMCRAFT_UI_SHOTS=` writes `keyframes-*.png`) | | Golden images | `crates/golden/tests/golden.rs` | CPU renders vs committed PNGs; GPU vs CPU on the same scenes (§3) | | Engine / command | `crates/engine/src/tests.rs` | run commands on the demo project, assert the sequence, undo/redo, disabled cases | | Render | `crates/render/src/tests.rs` | compositing, opacity, Motion, cross dissolve midpoint, ½-res vs full, GPU plan vs reference, audio mix, audio-effect continuity | @@ -33,6 +34,7 @@ FILMCRAFT_REQUIRE_ORACLES=1 cargo test --workspace # CI: missing ffmpeg fails | Remix | `crates/audio-dsp/src/remix.rs`, `crates/render/src/remix.rs`, `crates/engine/src/remix_tests.rs`, `crates/ui-egui/tests/remix_ui.rs` | generated rhythmic music (Rust-synthesised drums, bass, chord sections at 90–128 BPM, 22.05/44.1/48 kHz): tempo within 0.3 BPM, every beat within 10 ms (measured ≤ 8.9 ms); remix targets from 0.4× to 2.2× the source within one beat (measured ≤ 12 ms at default sliders), every cut on a detected beat, intro and outro kept, pieces ≥ 4 beats, deterministic; silence, tone and noise refused; rendering independent of request cuts, source-exact outside the 20 ms equal-power crossfades, no clicks; the engine's mix equals the source pieces (< 1e-5); undo, redo and revert exact; overlaps refused; Remix Properties dialog and Remix tool drag driven by automation id | | Oracle | `crates/*/tests/*oracle*.rs`, `conformance.rs` | compare with ffmpeg/ffprobe (§2) | | Robustness / fuzz | `*/tests/robustness.rs`, `*/tests/fuzz.rs` | seeded mutation and truncation of real and synthetic files; nothing may panic | +| Program monitor while dragging | `crates/ui-egui/tests/perf_ui.rs` | headless: a parameter changed on every UI pass (a value dragged in Effect Controls or Properties) and a backward scrub leave the Program monitor's picture at most one step behind (it shows the frame asked for the pass before while the frame due renders; it used to freeze until the mouse rested); a frame still rendering is kept when the same frame is asked for at a newer revision, so frames slower than a refresh (CPU effects) are shown too; `perf.stats` reports the picture's frame and revision against the playhead and the project | | Performance | `*/tests/perf.rs` (`#[ignore]`) | §5 | Commit `*.proptest-regressions` files so failing cases are re-run. @@ -186,10 +188,12 @@ switches, a slider drag as one undo step, section bypass, Auto-Match, ducking, B Essential Sound engine tests (`crates/engine/src/essential_sound_tests.rs`) build projects from generated speech-like and tonal WAVs: Auto-Match lands within ±0.5 LU of the target (measured: 0.000 LU, -with and without a repair/clarity chain); ducking on a dialogue + music project gives keyframes within -60 ms of the expected times (measured 20 ms) and −15.00 dB in the mix; each repair stage improves its -metric through the render path (hum −39 dB, rumble −22 dB, noise floor −13 dB, sibilance −19 dB, -reverb tail −10 dB); the mix is bit-identical however requests are cut and the WAV export equals it. +with and without a repair/clarity chain) and measures a mono clip as one channel (3.01 LU below the +same signal as dual mono, within 0.1 LU of a one-channel meter over the file); ducking on a dialogue + +music project gives keyframes within 60 ms of the expected times (measured 20 ms) and −15.00 dB in the +mix; each repair stage improves its metric through the render path (hum −39 dB, rumble −22 dB, noise +floor −13 dB, sibilance −19 dB, reverb tail −10 dB); the mix is bit-identical however requests are cut +and the WAV export equals it. `perf_full_dialogue_chain_realtime_factor` (ignored; run with `--release`) prints the realtime factor of all nine Dialogue effects on one clip (22× on one core). diff --git a/packaging/linux/install.sh b/packaging/linux/install.sh new file mode 100755 index 00000000..033fe9b7 --- /dev/null +++ b/packaging/linux/install.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Install the extracted Linux tarball. No sudo is needed for the default prefix. +set -euo pipefail + +usage() { + echo 'Usage: ./install.sh [--prefix /absolute/path]' + echo 'Installs FilmCraft and its desktop integration into ~/.local by default.' + echo 'For all users: sudo ./install.sh --prefix /usr/local' +} +fail() { echo "error: $*" >&2; exit 1; } +prefix="${HOME:?HOME is not set}/.local" +while [ "$#" -gt 0 ]; do + case "$1" in + --prefix) + [ "$#" -ge 2 ] || fail '--prefix needs an absolute path' + prefix="$2"; shift 2 ;; + -h | --help) usage; exit 0 ;; + *) usage >&2; fail "unknown argument: $1" ;; + esac +done +case "$prefix" in /*) ;; *) fail '--prefix must be an absolute path' ;; esac +prefix="$(realpath -m -- "$prefix")" +# Desktop Exec paths cannot contain '='; launchers also misinterpret '%' in executable names. +[[ "$prefix" != *'='* && "$prefix" != *'%'* && ! "$prefix" =~ [[:cntrl:]] ]] || fail 'prefix cannot contain =, % or control characters' +[ "$prefix" != / ] || fail 'use /usr/local or a user directory, not /' +source_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +case "$prefix/" in "$source_dir/"*) fail 'choose an installation prefix outside the extracted tarball' ;; esac +app_id=ai.storyteller.filmcraft +for binary in filmcraft filmcraft-cli; do + [ -f "$source_dir/bin/$binary" ] || fail "missing bin/$binary; extract the complete tarball first" +done +[ -f "$source_dir/share/applications/$app_id.desktop" ] || fail 'missing desktop entry; extract the complete tarball first' + +mkdir -p -- "$prefix/bin" "$prefix/share" +install -m755 -- "$source_dir/bin/filmcraft" "$prefix/bin/filmcraft" +install -m755 -- "$source_dir/bin/filmcraft-cli" "$prefix/bin/filmcraft-cli" +cp -R -- "$source_dir/share/." "$prefix/share/" + +# Use an absolute path so launching from the application menu does not depend on PATH. +# Escape the Exec argument, then the desktop entry's string value (two separate layers). +# https://specifications.freedesktop.org/desktop-entry/latest/exec-variables.html +exec_path="$(printf '%s' "$prefix/bin/filmcraft" | sed 's/[\\"`$]/\\&/g; s/\\/\\\\/g')" +try_path="$(printf '%s' "$prefix/bin/filmcraft" | sed 's/\\/\\\\/g')" +while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + Exec=*) printf 'Exec="%s" %%F\n' "$exec_path" ;; + TryExec=*) printf 'TryExec=%s\n' "$try_path" ;; + *) printf '%s\n' "$line" ;; + esac +done <"$source_dir/share/applications/$app_id.desktop" >"$prefix/share/applications/$app_id.desktop" + +# Cache tools are optional; installing binaries must also work on minimal systems. +if command -v update-desktop-database >/dev/null 2>&1; then + update-desktop-database -q "$prefix/share/applications" || echo 'warning: desktop cache refresh failed' >&2 +fi +if command -v update-mime-database >/dev/null 2>&1; then + update-mime-database "$prefix/share/mime" || echo 'warning: MIME cache refresh failed' >&2 +fi +if command -v gtk-update-icon-cache >/dev/null 2>&1; then + gtk-update-icon-cache -q -t -f "$prefix/share/icons/hicolor" || echo 'warning: icon cache refresh failed' >&2 +fi +printf 'Installed FilmCraft in %s\nLaunch it from the application menu or run "%s/bin/filmcraft".\n' "$prefix" "$prefix" +case ":${PATH:-}:" in + *":$prefix/bin:"*) ;; + *) printf 'For terminal commands, add %s/bin to your PATH.\n' "$prefix" ;; +esac diff --git a/packaging/linux/package.sh b/packaging/linux/package.sh index 1ee72e19..b2d00bc3 100755 --- a/packaging/linux/package.sh +++ b/packaging/linux/package.sh @@ -76,6 +76,7 @@ has() { case " $FORMATS " in *" $1 "*) return 0 ;; *) return 1 ;; esac; } if has tar; then mkdir -p "$WORK/tar" cp -R "$STAGE/usr" "$WORK/tar/$BASENAME" + install -m755 "$HERE/install.sh" "$WORK/tar/$BASENAME/install.sh" tar -C "$WORK/tar" -czf "$DIST/$BASENAME.tar.gz" "$BASENAME" echo "wrote $DIST/$BASENAME.tar.gz" fi diff --git a/packaging/linux/test_install.py b/packaging/linux/test_install.py new file mode 100644 index 00000000..ecaf2252 --- /dev/null +++ b/packaging/linux/test_install.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Exercise the tarball installer with disposable binaries and installation directories.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import time +import unittest + + +HERE = Path(__file__).resolve().parent +APP_ID = "ai.storyteller.filmcraft" + + +class InstallTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="filmcraft-install-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.bundle = self.root / "extracted bundle" + (self.bundle / "bin").mkdir(parents=True) + self.share = self.bundle / "share" + (self.share / "applications").mkdir(parents=True) + shutil.copyfile(HERE / "install.sh", self.bundle / "install.sh") + shutil.copyfile( + HERE / f"{APP_ID}.desktop", self.share / "applications" / f"{APP_ID}.desktop" + ) + for name in ("filmcraft", "filmcraft-cli"): + (self.bundle / "bin" / name).write_text( + '#!/bin/sh\nprintf \'%s\\n\' "$0" "$@" > "$(dirname "$0")/launched"\n' + ) + for path in ("doc/filmcraft/LICENSE-MIT", "icons/hicolor/test", "mime/packages/test.xml"): + target = self.share / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("fixture\n") + + def install(self, *args): + # Run outside the extracted directory to check that the script finds its own files. + return subprocess.run( + ["bash", str(self.bundle / "install.sh"), *map(str, args)], + cwd=self.root, capture_output=True, text=True, timeout=30, + ) + + def test_install_and_update_preserve_other_applications(self): + prefix = self.root / "user programs" + other = prefix / "share/applications/other.desktop" + other.parent.mkdir(parents=True) + other.write_text("unrelated\n") + for version in ("first", "updated"): + (self.bundle / "share/doc/filmcraft/LICENSE-MIT").write_text(version) + result = self.install("--prefix", prefix) + self.assertEqual(result.returncode, 0, result.stderr) + for name in ("filmcraft", "filmcraft-cli"): + self.assertTrue(os.access(prefix / "bin" / name, os.X_OK)) + self.assertEqual( + (prefix / "bin" / name).read_bytes(), (self.bundle / "bin" / name).read_bytes() + ) + self.assertEqual((prefix / "share/doc/filmcraft/LICENSE-MIT").read_text(), version) + self.assertTrue((prefix / "share/icons/hicolor/test").is_file()) + self.assertTrue((prefix / "share/mime/packages/test.xml").is_file()) + self.assertEqual(other.read_text(), "unrelated\n") + desktop = prefix / "share/applications" / f"{APP_ID}.desktop" + self.assertIn(f'Exec="{prefix}/bin/filmcraft" %F\n', desktop.read_text()) + if shutil.which("desktop-file-validate"): + subprocess.run(["desktop-file-validate", str(desktop)], check=True) + + @unittest.skipUnless(shutil.which("gio"), "gio is needed to exercise the desktop launcher") + def test_desktop_launch_escapes_paths_and_preserves_file_arguments(self): + prefix = self.root / 'programs with "quotes" $dollar `tick` \\slash & more' + result = self.install("--prefix", prefix) + self.assertEqual(result.returncode, 0, result.stderr) + document = self.root / "my film.filmcraft" + document.touch() + desktop = prefix / "share/applications" / f"{APP_ID}.desktop" + subprocess.run(["gio", "launch", str(desktop), str(document)], check=True, timeout=10) + launched = prefix / "bin/launched" + for _ in range(100): + if launched.exists() and launched.read_text().splitlines() == [ + str(prefix / "bin/filmcraft"), str(document) + ]: + break + time.sleep(0.01) + self.assertEqual(launched.read_text().splitlines(), [str(prefix / "bin/filmcraft"), str(document)]) + + def test_invalid_arguments_fail_before_writing(self): + for args in (("--prefix",), ("--prefix", "relative"), ("--prefix", "/"), ("--unknown",), + ("--prefix", "///"), ("--prefix", "/tmp/../"), ("--prefix", str(self.bundle)), + ("--prefix", str(self.bundle / "nested")), + ("--prefix", str(self.root / "bad=path")), + ("--prefix", str(self.root / "bad%path")), + ("--prefix", str(self.root / "bad\npath"))): + with self.subTest(args=args): + self.assertNotEqual(self.install(*args).returncode, 0) + + def test_incomplete_bundle_does_not_install(self): + (self.bundle / "bin/filmcraft-cli").unlink() + prefix = self.root / "destination" + result = self.install("--prefix", prefix) + self.assertNotEqual(result.returncode, 0) + self.assertIn("extract the complete tarball", result.stderr) + self.assertFalse(prefix.exists()) + + def test_help(self): + self.assertEqual(self.install("--help").returncode, 0) + + def test_release_tarball_includes_executable_installer(self): + target = self.root / "target" + binaries = target / "release" + binaries.mkdir(parents=True) + for name in ("filmcraft", "filmcraft-cli"): + binary = binaries / name + binary.write_text("#!/bin/sh\necho 'FilmCraft fixture'\n") + binary.chmod(0o755) + dist = self.root / "dist" + result = subprocess.run( + ["bash", str(HERE / "package.sh"), "--skip-build", "--formats", "tar"], + env={**os.environ, "CARGO_TARGET_DIR": str(target), "DIST": str(dist)}, + capture_output=True, text=True, timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + archives = list(dist.glob("*.tar.gz")) + self.assertEqual(len(archives), 1) + with tarfile.open(archives[0]) as archive: + installer = next(m for m in archive.getmembers() if m.name.endswith("/install.sh")) + self.assertEqual(installer.mode & 0o777, 0o755) + self.assertEqual(archive.extractfile(installer).read(), (HERE / "install.sh").read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/macos/dmg/README.md b/packaging/macos/dmg/README.md new file mode 100644 index 00000000..941880a2 --- /dev/null +++ b/packaging/macos/dmg/README.md @@ -0,0 +1,44 @@ +# macOS DMG window + +What Finder shows when the DMG opens: a 660 × 400 pt background with the app icon and the +`Applications` link side by side. `package.sh` copies these files into the image; nothing here is +generated at build time, so the DMG still builds with `hdiutil makehybrid` (no mounted device, no +Finder scripting on CI, nothing extra installed in the signing job). + +| File | What | +|---|---| +| `background.svg` | Source of the background: the app icon (`assets/app-icon/filmcraft.svg`, linked, not copied) cropped as a cover on the FilmCraft colour field (`#8b5cf6`), Ink and Paper. Its text (Inter, JetBrains Mono) is outlined, so rendering it needs no fonts. No paper grain: noise doesn't compress, and this keeps `background.tiff` small. | +| `background.tiff` | The background at 1x (660 × 400 px, 72 dpi) and 2x (1320 × 800 px, 144 dpi) in one HiDPI TIFF (16-colour palette, Deflate, sRGB). Goes to `.background/background.tiff`. | +| `dmg-layout.DS_Store` | Finder's view settings for the volume: window size, icon size 128, FilmCraft.app at (326, 205), `Applications` at (574, 205), and the background. Goes to `.DS_Store` in the image; named so it isn't mistaken for (or ignored like) a Finder-generated `.DS_Store`. | +| `generate.py` | Writes `background.tiff` and `dmg-layout.DS_Store` from the SVG and the layout above. | + +## The volume name has no version + +The mounted volume is called `FilmCraft`, not `FilmCraft `. The layout points at the background +through an alias that includes the volume name, and Finder resolves it by that name: with a +versioned name the window keeps its size and icon positions but shows no background (tested). +The DMG file name (`filmcraft--macos-.dmg`) still carries the version. + +## Rules + +- **Finder draws the icon labels in black in light and dark mode** when a window has a background, + so the area under both icons stays light (Paper). +- **Nothing goes inside the icon boxes:** artwork keeps 10 pt clear of each 128 pt icon box and of + the label strip under it. + +## Regenerate + +Edit `background.svg` (or the layout constants in `generate.py`), then run, on any OS: + +```sh +pip install 'pillow>=12' ds_store==1.3.3 mac_alias==2.2.3 +python3 packaging/macos/dmg/generate.py # needs resvg on PATH +``` + +Both outputs are byte-for-byte reproducible and carry nothing from the machine that ran it: the +SVG renders without any fonts, and `dmg-layout.DS_Store` is written from scratch, its background +alias holding only the volume name and `/.background/background.tiff`. If you add text to the SVG, +outline it (`usvg` from resvg converts text to paths). The window is 660 × 432 with Finder's 32 pt +title bar; the content area is 660 × 400. + +This follows VectorCraft's `packaging/macos/dmg/` (storytold/vectorcraft#493). diff --git a/packaging/macos/dmg/background.svg b/packaging/macos/dmg/background.svg new file mode 100644 index 00000000..abe09085 --- /dev/null +++ b/packaging/macos/dmg/background.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/macos/dmg/background.svg.attribution b/packaging/macos/dmg/background.svg.attribution new file mode 100644 index 00000000..0b3f939a --- /dev/null +++ b/packaging/macos/dmg/background.svg.attribution @@ -0,0 +1,8 @@ +asset: background.svg +title: FilmCraft macOS DMG window background +author: @XusBadia +source: original work +license: MIT OR Apache-2.0 +license-file: LICENSE-MIT, LICENSE-APACHE +added: 2026-10-08 by @XusBadia +notes: Original work for this repository, derived from the FilmCraft app icon (assets/app-icon/filmcraft.svg, referenced via , not copied). Text is outlined Inter / JetBrains Mono glyph paths. Used for the Finder window of the macOS DMG; packaging/macos/dmg/generate.py renders it to background.tiff. See packaging/macos/dmg/README.md. diff --git a/packaging/macos/dmg/background.tiff b/packaging/macos/dmg/background.tiff new file mode 100644 index 00000000..75d261ad Binary files /dev/null and b/packaging/macos/dmg/background.tiff differ diff --git a/packaging/macos/dmg/background.tiff.attribution b/packaging/macos/dmg/background.tiff.attribution new file mode 100644 index 00000000..37ed9279 --- /dev/null +++ b/packaging/macos/dmg/background.tiff.attribution @@ -0,0 +1,8 @@ +asset: background.tiff +title: FilmCraft macOS DMG window background (rendered) +author: @XusBadia +source: rendered from background.svg by generate.py +license: MIT OR Apache-2.0 +license-file: LICENSE-MIT, LICENSE-APACHE +added: 2026-10-08 by @XusBadia +notes: 1x + 2x HiDPI TIFF for the macOS DMG window; includes the app icon (see its row) and LittleCMS's built-in sRGB profile ("No copyright, use freely"). Regenerate with packaging/macos/dmg/generate.py. diff --git a/packaging/macos/dmg/dmg-layout.DS_Store b/packaging/macos/dmg/dmg-layout.DS_Store new file mode 100644 index 00000000..d6327a60 Binary files /dev/null and b/packaging/macos/dmg/dmg-layout.DS_Store differ diff --git a/packaging/macos/dmg/generate.py b/packaging/macos/dmg/generate.py new file mode 100644 index 00000000..1dde5639 --- /dev/null +++ b/packaging/macos/dmg/generate.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Regenerate background.tiff and dmg-layout.DS_Store for the macOS DMG window (see README.md here). + +Needs resvg on PATH and: + pip install 'pillow>=12' ds_store==1.3.3 mac_alias==2.2.3 + +Runs on any OS, no Mac needed. Both outputs come from this script alone, so they carry nothing +from the machine that made them: no local paths, user or disk names, dates or volume UUIDs. + + python3 packaging/macos/dmg/generate.py +""" + +import datetime +import os +import shutil +import subprocess +import sys +import tempfile + +from ds_store import DSStore +from mac_alias import Alias, TargetInfo, VolumeInfo +from PIL import Image, ImageCms + +HERE = os.path.dirname(os.path.abspath(__file__)) + +# Keep in sync with package.sh (volume name) and README.md (layout). +VOLUME = "FilmCraft" +WIDTH, HEIGHT = 660, 400 # window content, pt; the background's 1x size +TITLE_BAR = 32 +ICON_SIZE = 128 +PALETTE = 16 # colours in background.tiff +ICONS = {"FilmCraft.app": (326, 205), "Applications": (574, 205)} +# A fixed date for the alias and the colour profile. Finder never matches it against the image +# (each build is a new volume): it finds the background by volume name and path. +FIXED_DATE = datetime.datetime(2026, 10, 8, tzinfo=datetime.timezone.utc) + + +def render(svg, width, out): + # The SVG's text is already outlined and no fonts are loaded, so every machine draws the + # same pixels. + subprocess.run( + ["resvg", "--skip-system-fonts", "-w", str(width), svg, out], + check=True, + ) + with Image.open(out) as im: + return im.convert("RGB") + + +def quantize(im): + # No dithering: dither noise compresses badly and the artwork has flat fills. + return im.quantize(PALETTE, method=Image.Quantize.FASTOCTREE, dither=Image.Dither.NONE) + + +def srgb_profile(): + # LittleCMS's built-in sRGB ("No copyright, use freely"), with its creation date fixed. + icc = bytearray(ImageCms.ImageCmsProfile(ImageCms.createProfile("sRGB")).tobytes()) + d = FIXED_DATE + icc[24:36] = b"".join(v.to_bytes(2, "big") for v in (d.year, d.month, d.day, 0, 0, 0)) + if icc[84:100] != bytes(16): + sys.exit("error: the sRGB profile has an ID; recompute it after fixing the date") + return bytes(icc) + + +def write_tiff(svg, out): + """1x (72 dpi) and 2x (144 dpi) in one TIFF, as `tiffutil -cathidpicheck` writes them.""" + with tempfile.TemporaryDirectory() as tmp: + one, two = ( + quantize(render(svg, WIDTH * s, os.path.join(tmp, f"{s}x.png"))) for s in (1, 2) + ) + two.encoderinfo = {"dpi": (144, 144)} # the rest as the first page + # Opaque 16-colour palette, Deflate: the artwork is three inks plus antialiasing, and a palette + # keeps the file under ~100 KB (repository assets must stay small). As PhotoCraft's. + one.save( + out, + "TIFF", + save_all=True, + append_images=[two], + compression="tiff_adobe_deflate", # brand-ok: Pillow's name for TIFF compression 8 (Deflate) + dpi=(72, 72), + icc_profile=srgb_profile(), + ) + with Image.open(out) as im: + im.seek(1) + if im.info.get("dpi") != (144, 144): + sys.exit("error: the 2x page isn't 144 dpi (needs Pillow 12 or later)") + strips = [] + for page in (0, 1): + im.seek(page) + strips += zip(im.tag_v2[273], im.tag_v2[279]) # StripOffsets, StripByteCounts + # TIFF keeps directories and tag data on even offsets. When a strip ends on an odd one, libtiff + # leaves the padding byte after it uninitialised, so the file changed from run to run with + # identical pixels. Zero it: nothing refers to that byte. + starts = {o for o, _ in strips} + with open(out, "r+b") as f: + for o, n in strips: + if (o + n) % 2 and o + n not in starts: + f.seek(o + n) + f.write(bytes(1)) + + +def background_alias(): + # What Finder stores for /.background/background.tiff on the volume, minus anything about the + # machine (dmgbuild writes the same kind of alias from a mounted image). Disk type 5 (ejectable), + # the volume flags and the folder/file ids (25, 26) are what Finder wrote on a test image; the ids + # are only hints, as the volume's creation date. + volume = VolumeInfo(VOLUME, FIXED_DATE, b"H+", 5, 0x0D02, b"\0\0") + volume.posix_path = f"/Volumes/{VOLUME}" + target = TargetInfo(0, "background.tiff", 25, 26, FIXED_DATE, b"\0\0\0\0", b"\0\0\0\0") + target.folder_name = ".background" + target.cnid_path = [25] + target.carbon_path = f"{VOLUME}:.background:\0background.tiff" + target.posix_path = "/.background/background.tiff" + return Alias(volume=volume, target=target).to_bytes() + + +def write_ds_store(out): + if os.path.exists(out): + os.remove(out) + with DSStore.open(out, "w+") as d: + d["."]["bwsp"] = { + "ShowStatusBar": False, + "ShowToolbar": False, + "ShowTabView": False, + "ContainerShowSidebar": False, + "ShowSidebar": False, + "WindowBounds": f"{{{{200, 528}}, {{{WIDTH}, {HEIGHT + TITLE_BAR}}}}}", + } + d["."]["icvp"] = { + "viewOptionsVersion": 1, + "backgroundType": 2, + "backgroundImageAlias": background_alias(), + "backgroundColorRed": 1.0, + "backgroundColorGreen": 1.0, + "backgroundColorBlue": 1.0, + "gridOffsetX": 0.0, + "gridOffsetY": 0.0, + "gridSpacing": 100.0, + "arrangeBy": "none", + "showIconPreview": True, + "showItemInfo": False, + "labelOnBottom": True, + "textSize": 16.0, + "iconSize": float(ICON_SIZE), + } + d["."]["vSrn"] = ("long", 1) + for name, pos in ICONS.items(): + d[name]["Iloc"] = pos + + +def main(): + if not shutil.which("resvg"): + sys.exit("error: resvg not found (brew install resvg / cargo install resvg --locked)") + write_tiff(os.path.join(HERE, "background.svg"), os.path.join(HERE, "background.tiff")) + write_ds_store(os.path.join(HERE, "dmg-layout.DS_Store")) + + +if __name__ == "__main__": + main() diff --git a/packaging/macos/package.sh b/packaging/macos/package.sh index e9d036be..85902b91 100755 --- a/packaging/macos/package.sh +++ b/packaging/macos/package.sh @@ -131,10 +131,15 @@ STAGE="$WORK/dmg" mkdir -p "$STAGE" ditto "$APP" "$STAGE/FilmCraft.app" ln -s /Applications "$STAGE/Applications" +# Finder window layout: background, icon size and positions (packaging/macos/dmg/README.md). +mkdir -p "$STAGE/.background" +cp "$HERE/dmg/background.tiff" "$STAGE/.background/background.tiff" +cp "$HERE/dmg/dmg-layout.DS_Store" "$STAGE/.DS_Store" rm -f "$DMG" "$WORK/raw.dmg" # makehybrid + convert builds the image without attaching a device, unlike `create -srcfolder`, # which is flaky on CI runners ("Resource busy") and hangs in sandboxed sessions. -hdiutil makehybrid -hfs -hfs-volume-name "FilmCraft $VERSION" -hfs-openfolder "$STAGE" -o "$WORK/raw.dmg" "$STAGE" +# The volume name has no version: .DS_Store finds the background through an alias that includes it. +hdiutil makehybrid -hfs -hfs-volume-name "FilmCraft" -hfs-openfolder "$STAGE" -o "$WORK/raw.dmg" "$STAGE" hdiutil convert "$WORK/raw.dmg" -format UDZO -imagekey zlib-level=9 -o "$DMG" rm -f "$WORK/raw.dmg" sign "$DMG" diff --git a/packaging/windows/package.ps1 b/packaging/windows/package.ps1 index 04b31efb..faa4b863 100644 --- a/packaging/windows/package.ps1 +++ b/packaging/windows/package.ps1 @@ -5,7 +5,7 @@ .DESCRIPTION Produces, in $env:DIST (default: dist/release): filmcraft--windows-.msi per-machine installer (WiX v5) - filmcraft--windows--portable.zip filmcraft.exe + filmcraft-cli.exe + filmcraft--windows--portable.zip filmcraft.exe + filmcraft-cli.exe + portable.txt The binaries link the C runtime statically (+crt-static), so neither the MSI nor the portable zip needs the Visual C++ redistributable. Signing is delegated to sign.ps1 (skipped with a @@ -106,6 +106,11 @@ $Portable = Join-Path $TargetDir "windows-package\filmcraft-$Version-windows-$Ar Remove-Item -Recurse -Force $Portable -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $Portable | Out-Null Copy-Item (Join-Path $Stage '*.exe') $Portable +# The marker that makes FilmCraft keep its data in .\data next to the exe instead of %APPDATA% (#139). +Set-Content -Encoding ascii -Path (Join-Path $Portable 'portable.txt') -Value @( + 'FilmCraft portable: preferences, presets, logs and crash recovery are kept in the data folder', + 'next to filmcraft.exe. Delete this file to use %APPDATA%\FilmCraft instead.' +) foreach ($f in 'README.md', 'LICENSE', 'LICENSE-MIT', 'LICENSE-APACHE') { $p = Join-Path $Root $f if (Test-Path $p) { Copy-Item $p $Portable }