shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into fix-365 

# Conflicts:
#	apps/filmcraft/src/main.rs
This commit is contained in:
Brandon Thomas 2026-10-09 09:44:20 -04:00
parent 8d6461705e d63df6882b
commit aff136ff67
148 changed files with 10890 additions and 710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/packaging-lint.yml vendored
View File

@ -35,6 +35,8 @@ jobs:
"$RUNNER_TEMP/actionlint" -color
- name: shellcheck
run: shellcheck packaging/*.sh packaging/*/*.sh
- name: Linux tarball installer
run: python3 packaging/linux/test_install.py
- name: PowerShell
shell: pwsh
run: |

2
AGENTS.md
View File

@ -21,7 +21,7 @@ panic path, and fix a crash before building on top of it. The cross-app standard
code alone (a literal that always parses): a single-item `#[allow(clippy::expect_used)]` with the reason.
3. **No `unsafe`** (`unsafe_code = "forbid"` for the workspace). The single exception is
`crates/platform`, which needs `unsafe` to call the operating system's media APIs (hardware
video decoding through VideoToolbox and Media Foundation / Direct3D 11; later encoding and Linux) and does nothing else. It
video decoding through VideoToolbox, Media Foundation / Direct3D 11 and VA-API; hardware encoding) and does nothing else. It
uses `unsafe_code = "deny"` with `#[allow(unsafe_code)]` only on its FFI modules, a
`// SAFETY:` comment on every `unsafe` block, a safe `Result`-returning public API, and the
pure-Rust decoder as the tested fallback ([ADR 0001](docs/adr/0001-platform-ffi.md)).

2
ATTRIBUTION.md
View File

@ -26,6 +26,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri
| `docs/images/filmcraft-effects.png` | FilmCraft contributors | Original work: FilmCraft screenshot crop; NASA Earth Views (US Government work, public domain) | MIT OR Apache-2.0; footage public domain |
| `docs/images/filmcraft-timeline.png` | FilmCraft contributors | Original work: FilmCraft UI crop; thumbnails from Night of the Living Dead (1968) and Carnival of Souls (1962), CC0 Chopin waveform | MIT OR Apache-2.0; film frames public domain |
| `assets/app-icon/filmcraft.svg` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; canonical vector | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) |
| `packaging/macos/dmg/background.svg` | @XusBadia | Original work: FilmCraft macOS DMG window background, derived from the FilmCraft app icon (`assets/app-icon/filmcraft.svg`); rendered to `background.tiff` by `packaging/macos/dmg/generate.py` | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`) |
| `packaging/macos/dmg/background.tiff` | @XusBadia | Rendered from `packaging/macos/dmg/background.svg` by `packaging/macos/dmg/generate.py` | MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE`) |
| `assets/app-icon/filmcraft-small.svg` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; lighter vector | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) |
| `assets/app-icon/filmcraft-1024.png` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; 1024 px render | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) |
| `assets/app-icon/filmcraft-macos-512.png` | Brandon Thomas (FilmCraft owner) | Original work: FilmCraft app icon (owl with ear tufts), drawn by the owner in ArtCraft and vectorised; macOS runtime Dock icon (512 px) | MIT OR Apache-2.0 (`assets/app-icon/LICENSE.txt`) |

6
Cargo.lock generated
View File

@ -1843,6 +1843,7 @@ dependencies = [
name = "filmcraft-edit"
version = "0.4.0"
dependencies = [
"filmcraft-geom",
"filmcraft-media",
"filmcraft-project",
"filmcraft-time",
@ -2097,16 +2098,20 @@ dependencies = [
name = "filmcraft-platform"
version = "0.4.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-codecs",
"filmcraft-color",
"filmcraft-export",
"filmcraft-frame",
"filmcraft-h264",
"filmcraft-hevc",
"filmcraft-isobmff",
"filmcraft-media",
"filmcraft-project",
"filmcraft-render",
"filmcraft-testkit",
"filmcraft-time",
"libloading",
"log",
"objc2-core-foundation",
"objc2-core-media",
@ -2202,6 +2207,7 @@ dependencies = [
"skrifa",
"unicode-bidi",
"unicode-linebreak",
"unicode-segmentation",
]
[[package]]

12
README.md
View File

@ -137,7 +137,7 @@ FilmCraft is a non-linear editor for people who know Premiere: the same panels,
- **Premiere 26's full Video Effects bin (93 effects in 16 folders), the Legacy bin and the obsolete effects old projects use, plus around 30 transitions.** Blurs (Bokeh, Focus, Compound), keys (Ultra Key, Track Matte), distortions (Corner Pin, Turbulent Displace, Warp Stabilizer), Lights & Glows, Immersive Video (VR) effects on equirectangular footage, Posterize Time, Echo and more. Cross dissolve, dip to black or white, film dissolve, wipes, irises, pushes, slides, zooms, page peel, cube spin and more.
- **Motion and opacity on every clip:** position, scale, rotation, anchor point and anti-flicker, plus 26 blend modes.
- **Keyframes like Premiere's:** linear, Bezier, auto and continuous Bezier, hold, ease in and ease out. Effect Controls shows a keyframe lane for every parameter, and each animated parameter opens into **value and velocity graphs** with draggable influence handles.
- **Keyframes like Premiere's:** linear, Bezier, auto and continuous Bezier, hold, ease in and ease out. Effect Controls shows a keyframe lane for every parameter under a time ruler with the playhead's handle, and each animated parameter opens into **value and velocity graphs** with draggable influence handles. Effect Controls and the Properties panel share the keyframe navigator (◀ ◆ ▶): add or remove the keyframe at the playhead, step to the previous or next one.
- **A GPU compositor** built on wgpu (Metal, Vulkan, DirectX 12, WebGPU). It samples YUV straight from the decoder with footprint supersampling and blends in linear light. A CPU path renders the same frames, and the two are tested against each other.
<p align="center">
@ -185,7 +185,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus
| **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata |
| **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF |
Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: HEVC and AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox) and Windows (Media Foundation), with Linux to follow; H.264 export can use NVIDIA's encoder on Windows (opt-in) ([#30](https://github.com/storytold/filmcraft/issues/30)).
Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: HEVC and AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows (opt-in) ([#30](https://github.com/storytold/filmcraft/issues/30)).
<br>
@ -287,7 +287,7 @@ We track two numbers ([ROADMAP.md](ROADMAP.md#honest-assessment-2026-10-05)):
The biggest gaps today:
- **Speed on big footage.** Hardware decoding works on macOS and Windows; Linux has no hardware path yet. Blend modes and the most common effects run on the GPU, but Lumetri, keys and export rendering still run on the CPU; H.264 encoding can use the hardware encoder on macOS and on Windows with an NVIDIA GPU (opt-in, Export ▸ Hardware encoding), and H.265 export is hardware-only there ([#30](https://github.com/storytold/filmcraft/issues/30)).
- **Speed on big footage.** Hardware decoding works on macOS and Windows, and for H.264 and HEVC on Linux (VA-API; VP9 and AV1 decode in software there). Blend modes and the most common effects run on the GPU, but Lumetri, keys and export rendering still run on the CPU; H.264 encoding can use the hardware encoder on macOS and on Windows with an NVIDIA GPU (opt-in, Export ▸ Hardware encoding), and H.265 export is hardware-only there ([#30](https://github.com/storytold/filmcraft/issues/30)).
- **No plugins.** No VST3 / Audio Units or OpenFX hosting.
- **Delivery codecs.** H.264 is our only delivery-codec export; no HEVC or AV1 export yet.
- **Real-world media and platforms.** Our decoders are bit-exact on conformance streams, but camera and phone files in the wild are less tested. Windows and Linux get far less testing than macOS.
@ -357,7 +357,11 @@ and web backend defaults are unchanged.
| Flatpak | `filmcraft-<ver>-linux-x86_64.flatpak` | `filmcraft-<ver>-linux-aarch64.flatpak` | Sandboxed; `flatpak install --user <file>` |
| Debian/Ubuntu | `filmcraft-<ver>-linux-x86_64.deb` | `filmcraft-<ver>-linux-aarch64.deb` | |
| Fedora/RHEL/openSUSE | `filmcraft-<ver>-linux-x86_64.rpm` | `filmcraft-<ver>-linux-aarch64.rpm` | |
| Tarball | `filmcraft-<ver>-linux-x86_64.tar.gz` | `filmcraft-<ver>-linux-aarch64.tar.gz` | Unpack anywhere |
| Tarball | `filmcraft-<ver>-linux-x86_64.tar.gz` | `filmcraft-<ver>-linux-aarch64.tar.gz` | Unpack and run `./install.sh` |
The tarball installer places FilmCraft, its command-line tool and desktop integration in
`~/.local`, without administrator permissions. For all users, run
`sudo ./install.sh --prefix /usr/local` instead. Running the installer again updates the installation.
### FreeBSD

10
ROADMAP.md
View File

@ -36,7 +36,7 @@ before choosing work.
| **Correctness in depth** | not tracked | **~60–70%** | A first-time contributor found four real bugs in core paths within hours (#5–#8: AIFF import failed, Slide left linked audio behind, transitions didn't follow ripple trims, export ignored start/end times). User reports: Color Matte always grey with no picker (#29); ALSA audio error on Linux (#23). Settings are "most wired", not all. |
| **Codecs and media** | ~90% | **~75%** | Our decoders are bit-exact on conformance streams. Real camera and phone media (variable frame rate, damaged files, unusual containers) is far less tested. No camera RAW (RED, BRAW, ARRIRAW), no E-AC-3. |
| **Export** | ~92% | **~70%** | Delivery: H.264 only (ProRes, DNxHR and image sequences cover mastering). No HEVC or AV1 export. AAF and OMF have never been validated in Avid Media Composer or Pro Tools. Export renders and encodes on the CPU only. |
| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux has no hardware path. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. |
| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30), and for H.264 and HEVC on Linux (VA-API: 4K H.264 at 11 ms of CPU per frame instead of 417 ms, 4K HEVC at 10 ms instead of 203 ms). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists there only, through the same hardware encoder. Linux decodes H.264 and HEVC in hardware (VP9 and AV1 there are software) and has no hardware encoder. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. |
| **Stability** | not tracked | **improving, unproven** | The never-crash pass (no panics in product code, last-resort guards) landed on 2026-10-04. There's no field record from real users' projects yet. |
| **Plugins and pro ecosystem** | not tracked | **~0–10%** | No audio plugin hosting (VST3 / Audio Units) and no third-party video effects (OpenFX). For many professional editors this alone rules FilmCraft out. Team Projects and Productions are out of scope by design. |
| **AI features** | partial | **~25–35%** | Speech to text exists but is off in default builds (the `whisper` feature). Enhance Speech is a DSP chain, not a model. Auto Reframe is approximate. No Generative Extend, and no media-intelligence search or auto colour. |
@ -48,7 +48,7 @@ before choosing work.
In priority order. Agents should prefer this work over adding more checklist items.
1. **Hardware acceleration** (#30), the most visible gap to users:
- hardware decode on Linux (VA-API);
- hardware decode on Linux (VA-API): H.264 and HEVC are in; VP9 and AV1 remain;
- zero-copy decoded frames into wgpu;
- the remaining effects on the GPU: Lumetri, keys, Vignette, Video Limiter, masks (31 common effects are done);
- GPU export, then hardware encode: H.264 and H.265 on macOS are in (H.264 opt-in; H.265 has no other encoder), H.264 on NVENC (Windows, opt-in); HEVC on Windows and Linux remain.
@ -100,7 +100,7 @@ matched against `filmcraft-cli commands` and the UI command table. Effects: the
| Graphics and captions | 8% | text engine, shapes, per-character styles, our own graphics templates (.fcgt, 8 built-ins, export/install/edit), rolls/crawls, responsive pins + time, captions SRT/VTT/SCC/MCC/STL/TTML/DFXP, transcripts | ~85% |
| Export | 8% | own H.264/AAC, ProRes, DNxHR, MXF OP1a/OP-Atom, image sequences, GIF, WAV, 5.1; preset library + user presets, full Export-mode settings, queue, Quick Export; AAF (Edit Protocol) import/export, OMF 2.0 export; no AAF/OMF validation against Avid/Pro Tools yet | ~92% |
| Preferences and project management | 5% | Settings dialog with 16 categories (most settings wired), project settings, scratch disks, search bins, templates | ~80% |
| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; blend modes on the GPU (#32); 8K and AV1 not yet real-time on a loaded machine; no hardware path on Linux, and on Windows only decode and opt-in NVENC H.264 export (see the honest assessment: ~35–40% against Premiere) | ~62% |
| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; blend modes on the GPU (#32); 8K and AV1 not yet real-time on a loaded machine; H.264 and HEVC decode in hardware on Linux (VA-API), and on Windows only decode and opt-in NVENC H.264 export (see the honest assessment: ~35–40% against Premiere) | ~62% |
| **Weighted total** | | | **~87%** |
## Estimate to parity
@ -154,11 +154,13 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining
## Running now
- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Next: VA-API (Linux) decode, zero-copy upload, Lumetri on the GPU.
- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in). Linux: VA-API H.264 + HEVC decode (HW5). Next: VA-API VP9 / AV1, zero-copy upload, Lumetri on the GPU.
- **Next:** [Where we are lacking](#where-we-are-lacking) items 1–5. GPU export waits for the export-crate work (#19 and the never-crash follow-up) to land.
## Log
- **2026-10-08 (#285):** `filmcraft-cli` no longer panics with "Broken pipe" when its stdout is closed early (`filmcraft-cli commands | head`): later output is dropped, the rest of the work (script lines, `--save`, `--save-as`) still runs, and the exit status still reports failures. Other stdout write errors are reported and make the exit status 1. No change to the checklist or estimates.
- **2026-10-06:** 21 community PRs landed (#19, #28, #38, #39, #41–#60). Highlights: hardware decoding is now actually on in the desktop app (#41: `register()` sat inside a `log::info!` that never ran, so #33's VideoToolbox path only worked in the CLI and benchmarks); decode memory on long timelines cut sharply (frame pool, shared GOP-cache budget, positional file reads); exports are byte-identical across machines (#19); crash fix for odd track names over MCP (#44); ripple trim, ripple delete and speed changes move split-edit clips once (#56–#58); Japanese interface using an installed system font (#38; no fonts bundled); MCP annotations, resources, export progress and cancellation (#28).
- **2026-10-05:** honest assessment added (checklist ~87% vs ready for real work ~50–60%). Hardware acceleration started (#30): blend modes on the GPU (#32); VideoToolbox decode + `platform` crate, the one crate allowed `unsafe` (#33). Community fixes landed: AIFF import, export start/end ranges, transitions following ripple trims, Slide moving linked audio (#5–#8).

109
apps/filmcraft-cli/src/main.rs
View File

@ -4,6 +4,32 @@
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
/// `println!` that doesn't panic when stdout is closed (`filmcraft-cli commands | head`) with
/// "failed printing to stdout: Broken pipe (os error 32)". Once stdout has failed, later output is
/// dropped and the work goes on: see `stdout_failed`.
macro_rules! outln {
($($arg:tt)*) => {{
use std::io::Write as _;
if !$crate::stdout_gone() {
if let Err(e) = writeln!(std::io::stdout(), $($arg)*) {
$crate::stdout_failed(e);
}
}
}};
}
/// `print!` counterpart of `outln!`.
macro_rules! out {
($($arg:tt)*) => {{
use std::io::Write as _;
if !$crate::stdout_gone() {
if let Err(e) = write!(std::io::stdout(), $($arg)*) {
$crate::stdout_failed(e);
}
}
}};
}
mod args;
mod probe;
@ -11,6 +37,28 @@ use args::{Args, parse_value};
use filmcraft_automation::BridgeClient;
use filmcraft_engine::Session;
use serde_json::{Value, json};
use std::sync::atomic::{AtomicBool, Ordering};
/// Set once a write to stdout has failed; later output is dropped.
static STDOUT_GONE: AtomicBool = AtomicBool::new(false);
/// Set when stdout failed for a reason other than a closed reader: the exit status becomes 1.
static STDOUT_ERROR: AtomicBool = AtomicBool::new(false);
fn stdout_gone() -> bool {
STDOUT_GONE.load(Ordering::Relaxed)
}
/// stdout went away. The program doesn't stop there: the rest of `exec`/`run`/`import` (later
/// script lines, `--save`, `--save-as`) still runs and the exit status still tells how it went, so
/// `exec … --save | head -1` keeps the edit. A reader that stopped early (a closed pipe) is not an
/// error, as in ripgrep; any other write error is reported and makes the exit status 1.
fn stdout_failed(e: std::io::Error) {
if STDOUT_GONE.swap(true, Ordering::Relaxed) || e.kind() == std::io::ErrorKind::BrokenPipe {
return;
}
eprintln!("filmcraft-cli: can't write to stdout: {e}");
STDOUT_ERROR.store(true, Ordering::Relaxed);
}
const HELP: &str = "\
filmcraft-cli: drive FilmCraft from the shell (headless engine, or the live app with --bridge)
@ -134,7 +182,7 @@ impl Backend {
fn print(a: &Args, v: &Value) {
let s = if a.flag("--compact") { serde_json::to_string(v) } else { serde_json::to_string_pretty(v) };
println!("{}", s.unwrap_or_default());
outln!("{}", s.unwrap_or_default());
}
/// Export format from the output file's extension.
@ -162,16 +210,23 @@ fn register_hardware_decoders() -> filmcraft_platform::Availability {
#[tokio::main]
async fn main() {
cli().await;
if STDOUT_ERROR.load(Ordering::Relaxed) {
std::process::exit(1);
}
}
async fn cli() {
if matches!(std::env::args().nth(1).as_deref(), Some("--version" | "-V")) {
println!("filmcraft-cli {}", env!("CARGO_PKG_VERSION"));
outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION"));
return;
}
register_hardware_decoders();
let a = Args::parse(std::env::args().skip(1));
let Some(cmd) = a.pos(0) else { usage("missing subcommand") };
match cmd {
"help" | "--help" | "-h" => print!("{HELP}"),
"version" => println!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")),
"help" | "--help" | "-h" => out!("{HELP}"),
"version" => outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")),
"probe" => {
let path = a.pos(1).unwrap_or_else(|| usage("probe <media> [--image-sequence]"));
match probe::probe(path, a.flag("--image-sequence")) {
@ -189,11 +244,11 @@ async fn main() {
src.video_frame(filmcraft_media::FrameRequest::full(rate.tick_of(f))).unwrap_or_else(|e| fail(e));
}
let dt = t0.elapsed().as_secs_f64();
println!("{n} frames in {dt:.2}s → {:.1} fps (sequential, via Mp4Source)", n as f64 / dt);
outln!("{n} frames in {dt:.2}s → {:.1} fps (sequential, via Mp4Source)", n as f64 / dt);
let t1 = std::time::Instant::now();
let f = src.video_frame(filmcraft_media::FrameRequest::full(rate.tick_of(n / 2))).unwrap_or_else(|e| fail(e));
let (w, h, _) = f.to_linear_f32_decimated(2);
println!("½-res linear conversion {w}x{h}: {:.1} ms", t1.elapsed().as_secs_f64() * 1000.0);
outln!("½-res linear conversion {w}x{h}: {:.1} ms", t1.elapsed().as_secs_f64() * 1000.0);
}
"commands" | "describe" => {
let mut b = Backend::open(&a);
@ -219,7 +274,7 @@ async fn main() {
} else {
for c in &list {
let s = |k: &str| c[k].as_str().unwrap_or("").to_string();
println!("{:<32} {:<34} {:<14} {}", s("id"), s("label"), s("shortcut"), s("params"));
outln!("{:<32} {:<34} {:<14} {}", s("id"), s("label"), s("shortcut"), s("params"));
}
}
}
@ -246,9 +301,9 @@ async fn main() {
let v: Value = serde_json::from_str(line).unwrap_or_else(|e| usage(format!("line {}: {e}", n + 1)));
let id = v["id"].as_str().unwrap_or_else(|| usage(format!("line {}: missing \"id\"", n + 1)));
match b.exec(id, v.get("params").cloned().unwrap_or(json!({}))).await {
Ok(r) => println!("{}", json!({"line": n + 1, "id": id, "ok": true, "result": r})),
Ok(r) => outln!("{}", json!({"line": n + 1, "id": id, "ok": true, "result": r})),
Err(e) => {
println!("{}", json!({"line": n + 1, "id": id, "ok": false, "error": e}));
outln!("{}", json!({"line": n + 1, "id": id, "ok": false, "error": e}));
failed += 1;
if !a.flag("--keep-going") {
std::process::exit(1);
@ -344,7 +399,7 @@ async fn main() {
match r {
Ok(v) => {
print(&a, &v);
eprintln!("exported {out} in {:.1}s", t0.elapsed().as_secs_f64());
eprintln!("exported {} in {:.1}s", written_paths(&v, out).join(", "), t0.elapsed().as_secs_f64());
}
Err(e) => fail(format!("export: {e}")),
}
@ -381,6 +436,24 @@ async fn main() {
}
}
/// The files an export wrote, from its result: `path` of a direct export, or each queued item's
/// `path` (the format may have changed the extension: `out.mp4` exported as APV is `out.mp4.mov`).
/// `requested` when the result names none.
fn written_paths(result: &Value, requested: &str) -> Vec<String> {
let paths: Vec<String> = match result.get("path").and_then(Value::as_str) {
Some(p) => vec![p.to_string()],
None => result
.get("items")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|i| i.get("path").and_then(Value::as_str))
.map(str::to_string)
.collect(),
};
if paths.is_empty() { vec![requested.to_string()] } else { paths }
}
#[cfg(test)]
mod format_tests {
/// The CLI (and MCP / headless runs, which share its entry point) registers the hardware
@ -388,7 +461,21 @@ mod format_tests {
#[test]
fn startup_registers_the_hardware_decoders() {
let hardware = super::register_hardware_decoders();
assert_eq!(filmcraft_platform::registered(), cfg!(any(target_os = "macos", target_os = "windows")), "{hardware:?}");
// always on macOS and Windows; on Linux when a VA-API driver is there
let expected = cfg!(any(target_os = "macos", target_os = "windows"))
|| (cfg!(target_os = "linux") && matches!(hardware, filmcraft_platform::Availability::Available(_)));
assert_eq!(filmcraft_platform::registered(), expected, "{hardware:?}");
}
#[test]
fn export_message_names_the_written_files() {
use serde_json::json;
let direct = json!({"job": 1, "path": "out-apv.mp4.mov", "result": {"path": "out-apv.mp4.mov"}});
assert_eq!(super::written_paths(&direct, "out-apv.mp4"), ["out-apv.mp4.mov"]);
let queued = json!({"added": [1, 2], "items": [{"path": "a.mp4.mov"}, {"path": "b.wav"}]});
assert_eq!(super::written_paths(&queued, "a.mp4"), ["a.mp4.mov", "b.wav"]);
assert_eq!(super::written_paths(&json!({}), "x.mov"), ["x.mov"]);
assert_eq!(super::written_paths(&json!({"items": []}), "x.mov"), ["x.mov"]);
}
#[test]

80
apps/filmcraft-cli/tests/cli.rs
View File

@ -179,9 +179,89 @@ fn export_with_presets() {
let v = json_out(&cli(&["--demo", "--data-dir", data_s, "export", q.to_str().unwrap(), "--preset", "Tiny WAV", "--queue", "--range", "entire"]));
assert_eq!(v["items"][0]["status"], "done", "{v}");
assert!(q.exists());
// a format that changes the extension: the completion message names the file written (#341)
let apv = dir.join("apv.mp4");
let o =
cli(&["--demo", "--data-dir", data_s, "export", apv.to_str().unwrap(), "--preset", "APV 422-10", "--start", "0", "--end", "0.1", "--scale", "0.125"]);
let v = json_out(&o);
let written = v["path"].as_str().unwrap().to_string();
assert!(written.ends_with("apv.mp4.mov") && std::path::Path::new(&written).exists(), "{v}");
let msg = String::from_utf8_lossy(&o.stderr);
assert!(msg.contains(&format!("exported {written} in ")), "{msg}");
let qa = dir.join("queued-apv.mp4");
let o = cli(&[
"--demo",
"--data-dir",
data_s,
"export",
qa.to_str().unwrap(),
"--preset",
"APV 422-10",
"--queue",
"--start",
"0",
"--end",
"0.1",
"--scale",
"0.125",
]);
let v = json_out(&o);
let written = v["items"][0]["path"].as_str().unwrap().to_string();
assert!(written.ends_with("queued-apv.mp4.mov"), "{v}");
assert!(String::from_utf8_lossy(&o.stderr).contains(&format!("exported {written} in ")), "{}", String::from_utf8_lossy(&o.stderr));
// unknown preset: exit status 1 and a message naming it
let bad = cli(&["--demo", "--data-dir", data_s, "export", dir.join("x.mp4").to_str().unwrap(), "--preset", "No Such Preset"]);
assert_eq!(bad.status.code(), Some(1));
assert!(String::from_utf8_lossy(&bad.stderr).contains("No Such Preset"));
let _ = std::fs::remove_dir_all(&dir);
}
/// `filmcraft-cli commands | head -c 10` panicked with "failed printing to stdout: Broken pipe
/// (os error 32)" and exit status 101 (#285).
#[test]
fn closed_stdout_ends_quietly() {
use std::io::Read;
for args in [&["commands"][..], &["help"], &["--demo", "commands", "--json"]] {
let mut child = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(args).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
let mut first = [0u8; 10];
child.stdout.as_mut().unwrap().read_exact(&mut first).unwrap();
drop(child.stdout.take());
let out = child.wait_with_output().unwrap();
let err = String::from_utf8_lossy(&out.stderr);
assert!(out.status.success() && err.is_empty(), "{args:?}: {:?}: {err}", out.status);
}
}
/// `run -` with stdout already closed: `run` reads its script to the end before it prints, so its
/// first write fails for sure. The rest of the script and the save still happen, and the exit status
/// still reports a failing command.
fn run_with_closed_stdout(args: &[&str], script: &[u8]) -> Output {
let mut child =
Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
drop(child.stdout.take());
child.stdin.take().unwrap().write_all(script).unwrap();
child.wait_with_output().unwrap()
}
#[test]
fn closed_stdout_still_runs_the_script_and_saves() {
let dir = std::env::temp_dir().join(format!("fc-cli-pipe-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let proj = dir.join("p.fcproj");
let p = proj.to_str().unwrap();
let script = b"{\"id\":\"file.newBin\",\"params\":{\"name\":\"Piped One\"}}\n{\"id\":\"file.newBin\",\"params\":{\"name\":\"Piped Two\"}}\n";
let out = run_with_closed_stdout(&["--demo", "--save-as", p, "run", "-"], script);
assert!(out.status.success(), "{:?}: {}", out.status, String::from_utf8_lossy(&out.stderr));
let tree = json_out(&cli(&["--project", p, "inspect", "project"])).to_string();
assert!(tree.contains("Piped One") && tree.contains("Piped Two"), "{tree}");
// a failing command is still a failure, whether or not anyone reads stdout
let out = run_with_closed_stdout(&["--demo", "run", "-"], b"{\"id\":\"no.such.command\"}\n");
assert_eq!(out.status.code(), Some(1));
let out = run_with_closed_stdout(
&["--demo", "--keep-going", "run", "-"],
b"{\"id\":\"no.such.command\"}\n{\"id\":\"file.newBin\",\"params\":{\"name\":\"x\"}}\n",
);
assert_eq!(out.status.code(), Some(1));
let _ = std::fs::remove_dir_all(&dir);
}

18
apps/filmcraft-web/src/fs.rs
View File

@ -70,9 +70,8 @@ fn file_name(path: &str) -> &str {
path.rsplit('/').next().unwrap_or(path)
}
/// Register a browser `File` (or `Blob` with a name) and return its virtual path. A file with
/// the same name and size as an existing entry replaces it (the same file picked again keeps
/// its path, so projects referencing it come back online); otherwise a numbered name is used.
/// Register a browser `File` (or named `Blob`) at an unused virtual path; collisions get numbered names.
/// Name + size do not identify content: replacing an occupied path changes every project item referencing it.
pub fn register_blob(name: &str, blob: web_sys::Blob) -> String {
let size = blob.size() as u64;
let clean: String = name.chars().map(|c| if c == '/' || c == '\\' { '_' } else { c }).collect();
@ -85,18 +84,11 @@ pub fn register_blob(name: &str, blob: web_sys::Blob) -> String {
let mut n = 1;
let path = loop {
let p = if n == 1 { format!("/files/{clean}") } else { format!("/files/{stem} ({n}){ext}") };
match fs.entries.get(&p) {
None => break p,
Some(Entry::Blob { size: s, .. }) if *s == size => break p,
Some(Entry::Mem(b)) if b.len() as u64 == size => break p,
_ => n += 1,
if !fs.entries.contains_key(&p) {
break p;
}
n += 1;
};
if let Some(Entry::Blob { id, .. }) = fs.entries.get(&path) {
let id = *id;
fs.blobs.remove(&id);
fs.chunks.retain(|k, _| k.0 != id);
}
let id = fs.next_id;
fs.next_id += 1;
fs.blobs.insert(id, blob);

8
apps/filmcraft-web/src/import.rs
View File

@ -81,12 +81,12 @@ async fn prewarm(path: &str, size: u64) {
}
/// `file.import` one path, retried while its bytes are loading.
async fn import_path(path: String) -> Result<Value, String> {
async fn import_path(path: String, bin: u64) -> Result<Value, String> {
for _ in 0..2000 {
let p = path.clone();
let (r, pending) = on_ui(move |w, _| {
let _ = filmcraft_media::pending::take();
let r = w.app.session.execute("file.import", json!({"paths": [p]})).map_err(|e| e.to_string());
let r = w.app.session.execute("file.import", json!({"paths": [p], "bin": bin})).map_err(|e| e.to_string());
(r, filmcraft_media::pending::take())
})
.await;
@ -105,6 +105,8 @@ async fn import_path(path: String) -> Result<Value, String> {
/// Import browser files (no copies): returns `{items, errors, paths}`.
pub async fn import_files(files: Vec<web_sys::File>) -> Value {
// Capture once per batch: navigation while prewarming must not redirect pending imports.
let bin = on_ui(|w, _| w.app.import_bin().0).await;
let mut items = Vec::new();
let mut errors = Vec::new();
let mut paths = Vec::new();
@ -113,7 +115,7 @@ pub async fn import_files(files: Vec<web_sys::File>) -> Value {
let size = f.size() as u64;
let path = fs::register_blob(&name, f.clone().into());
prewarm(&path, size).await;
match import_path(path.clone()).await {
match import_path(path.clone(), bin).await {
Ok(v) => {
items.extend(v.get("items").and_then(Value::as_array).cloned().unwrap_or_default());
errors.extend(v.get("errors").and_then(Value::as_array).cloned().unwrap_or_default());

10
apps/filmcraft-web/src/lib.rs
View File

@ -24,6 +24,7 @@ pub mod fs;
pub mod import;
pub mod opfs;
pub mod recovery;
pub mod recovery_policy;
pub mod webcodecs;
use std::cell::RefCell;
@ -58,6 +59,13 @@ pub fn repaint() {
});
}
/// Give a host that builds its own [`WebApp`] the egui context that [`post`] and [`repaint`] wake.
/// [`start`] calls it; an embedding app that runs `WebApp` in its own eframe runner calls it once
/// from its app creator, or posted actions and JS API requests wait for the next input event.
pub fn set_context(ctx: egui::Context) {
CTX.with(|c| *c.borrow_mut() = Some(ctx));
}
/// Environment facts reported by `filmcraft.info()` (backend, isolation, codecs…).
pub fn set_info(key: &str, v: serde_json::Value) {
INFO.with(|i| {
@ -150,7 +158,7 @@ pub async fn start(canvas_id: String) -> Result<(), JsValue> {
canvas,
web_options,
Box::new(move |cc| {
CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone()));
set_context(cc.egui_ctx.clone());
let ctx = cc.egui_ctx.clone();
fs::set_on_data(move || ctx.request_repaint());
fs::set_on_write(|path, data| {

25
apps/filmcraft-web/src/opfs.rs
View File

@ -10,8 +10,13 @@ use wasm_bindgen::{JsCast, JsValue};
use wasm_bindgen_futures::JsFuture;
use web_sys::{FileSystemDirectoryHandle, FileSystemFileHandle};
/// Called with whether a queued write reached OPFS.
pub type Done = Box<dyn FnOnce(bool)>;
thread_local! {
static QUEUE: RefCell<(BTreeMap<String, Vec<u8>>, bool)> = const { RefCell::new((BTreeMap::new(), false)) };
/// Pending writes per path (bytes and the callbacks of every write coalesced into them), and
/// whether the writer task is running.
static QUEUE: RefCell<(BTreeMap<String, (Vec<u8>, Vec<Done>)>, bool)> = const { RefCell::new((BTreeMap::new(), false)) };
}
/// Whether this browser has OPFS (`navigator.storage.getDirectory`).
@ -60,9 +65,17 @@ async fn write_now(rel: &str, data: &[u8]) -> Result<(), JsValue> {
/// Queue a write of `data` to `rel` (relative to the OPFS root).
pub fn write(rel: String, data: Vec<u8>) {
write_then(rel, data, None);
}
/// [`write`], then call `done` with whether it succeeded. A write replaced by a newer one to the
/// same path reports the newer one's result.
pub fn write_then(rel: String, data: Vec<u8>, done: Option<Done>) {
let start = QUEUE.with(|q| {
let mut q = q.borrow_mut();
q.0.insert(rel, data);
let e = q.0.entry(rel).or_default();
e.0 = data;
e.1.extend(done);
!std::mem::replace(&mut q.1, true)
});
if !start {
@ -81,10 +94,14 @@ pub fn write(rel: String, data: Vec<u8>) {
}
}
});
let Some((rel, data)) = next else { break };
if let Err(e) = write_now(&rel, &data).await {
let Some((rel, (data, done))) = next else { break };
let r = write_now(&rel, &data).await;
if let Err(e) = &r {
log::warn!("OPFS write {rel}: {e:?}");
}
for d in done {
d(r.is_ok());
}
}
});
}

68
apps/filmcraft-web/src/recovery.rs
View File

@ -6,24 +6,32 @@
//! - Imported media are copied to `media/<name>` (the browser streams the copy; nothing goes
//! through wasm memory), so the snapshot's `/files/<name>` references resolve after a reload.
//! - At start-up a dirty snapshot is reopened (`?norecover` skips it, `?fresh` also skips the media).
//! A skipped snapshot stays dirty until this session writes its own ([`crate::recovery_policy`]).
use std::cell::{Cell, RefCell};
use std::rc::Rc;
use filmcraft_engine::Session;
use serde_json::{Value, json};
use crate::opfs;
use crate::recovery_policy::{Action, Policy};
const SNAPSHOT: &str = "recovery/snapshot.fcproj";
const META: &str = "recovery/meta.json";
/// Seconds between snapshots of a changing project.
const INTERVAL_S: f64 = 5.0;
/// Media larger than this are not copied to OPFS.
const KEEP_MAX: f64 = 4.0 * 1024.0 * 1024.0 * 1024.0;
/// A finished OPFS write, reported back to the next [`Autosave::tick`].
enum Outcome {
Snapshot { revision: u64, ok: bool },
Clean { ok: bool },
}
#[derive(Default)]
pub struct Autosave {
last_revision: u64,
last_time: f64,
last_dirty: Option<bool>,
policy: Policy,
outcomes: Rc<RefCell<Vec<Outcome>>>,
}
impl Autosave {
@ -33,22 +41,44 @@ impl Autosave {
return;
}
let now = ctx.input(|i| i.time);
let dirty = s.revision != s.saved_revision;
if dirty {
if s.revision != self.last_revision && now - self.last_time >= INTERVAL_S {
opfs::write(SNAPSHOT.into(), filmcraft_format::encode(&s.project, false));
let meta = json!({"dirty": true, "name": s.project.name, "revision": s.revision, "time": js_sys::Date::now()});
opfs::write(META.into(), meta.to_string().into_bytes());
self.last_revision = s.revision;
self.last_time = now;
} else if s.revision != self.last_revision {
// the UI may go idle: wake up when the snapshot is due
ctx.request_repaint_after(std::time::Duration::from_secs_f64((INTERVAL_S - (now - self.last_time)).max(0.05)));
for o in self.outcomes.borrow_mut().drain(..) {
match o {
Outcome::Snapshot { revision, ok } => self.policy.written(revision, ok, now),
Outcome::Clean { ok } => self.policy.cleaned(ok, now),
}
}
match self.policy.tick(now, s.revision, s.saved_revision) {
Action::Nothing => {}
Action::WakeAfter(secs) => ctx.request_repaint_after(std::time::Duration::from_secs_f64(secs)),
Action::WriteSnapshot(revision) => {
// Snapshot and meta both have to land; either failing retries the pair.
let failed = Rc::new(Cell::new(false));
let left = Rc::new(Cell::new(2u8));
let done = |out: Rc<RefCell<Vec<Outcome>>>, failed: Rc<Cell<bool>>, left: Rc<Cell<u8>>, ctx: egui::Context| -> opfs::Done {
Box::new(move |ok| {
failed.set(failed.get() || !ok);
left.set(left.get().saturating_sub(1));
if left.get() == 0 {
out.borrow_mut().push(Outcome::Snapshot { revision, ok: !failed.get() });
ctx.request_repaint();
}
})
};
let meta = json!({"dirty": true, "name": s.project.name, "revision": s.revision, "time": js_sys::Date::now()});
let d1 = done(self.outcomes.clone(), failed.clone(), left.clone(), ctx.clone());
let d2 = done(self.outcomes.clone(), failed, left, ctx.clone());
opfs::write_then(SNAPSHOT.into(), filmcraft_format::encode(&s.project, false), Some(d1));
opfs::write_then(META.into(), meta.to_string().into_bytes(), Some(d2));
}
Action::WriteClean => {
let (out, ctx) = (self.outcomes.clone(), ctx.clone());
let done: opfs::Done = Box::new(move |ok| {
out.borrow_mut().push(Outcome::Clean { ok });
ctx.request_repaint();
});
opfs::write_then(META.into(), json!({"dirty": false}).to_string().into_bytes(), Some(done));
}
} else if self.last_dirty != Some(false) {
opfs::write(META.into(), json!({"dirty": false}).to_string().into_bytes());
}
self.last_dirty = Some(dirty);
}
}

88
apps/filmcraft-web/src/recovery_policy.rs Normal file
View File

@ -0,0 +1,88 @@
//! When the web recovery snapshot is written or marked clean (pure, so it is tested natively in
//! `tests/recovery_policy.rs`; the OPFS side lives in [`crate::recovery`]).
//!
//! - A dirty project is snapshotted at most every [`INTERVAL_S`] seconds, once per revision. A write
//! that fails is retried after the interval, without waiting for another edit (#197).
//! - The meta is marked clean only to retire a snapshot this session wrote. A snapshot left by an
//! earlier session that was not reopened (`?norecover`, `?fresh`) stays dirty (#193).
/// Seconds between snapshots of a changing project.
pub const INTERVAL_S: f64 = 5.0;
/// What the caller should do this frame.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Action {
Nothing,
/// Write the snapshot and a dirty meta for this revision, then report with [`Policy::written`].
WriteSnapshot(u64),
/// Write a clean meta, then report with [`Policy::cleaned`].
WriteClean,
/// Nothing now; wake up after this many seconds.
WakeAfter(f64),
}
#[derive(Debug, Default)]
pub struct Policy {
/// Revision whose snapshot was last written or is being written.
last_revision: Option<u64>,
last_time: Option<f64>,
/// This session wrote a dirty meta that is not yet retired.
owns_dirty_meta: bool,
/// A clean-meta write is in flight.
cleaning: bool,
/// Snapshot writes started so far, and how many had started when the clean write did (a
/// snapshot started after it makes the meta dirty again).
snapshots: u64,
snapshots_at_clean: u64,
}
impl Policy {
/// Called every frame with the session's revisions and the time in seconds.
pub fn tick(&mut self, now: f64, revision: u64, saved_revision: u64) -> Action {
if revision != saved_revision {
if self.last_revision == Some(revision) {
return Action::Nothing;
}
let since = self.last_time.map_or(f64::INFINITY, |t| now - t);
if since >= INTERVAL_S {
self.last_revision = Some(revision);
self.last_time = Some(now);
self.owns_dirty_meta = true;
self.snapshots += 1;
return Action::WriteSnapshot(revision);
}
return Action::WakeAfter((INTERVAL_S - since).max(0.05));
}
if self.owns_dirty_meta && !self.cleaning {
let since = self.last_time.map_or(f64::INFINITY, |t| now - t);
if self.last_revision.is_some() || since >= INTERVAL_S {
self.cleaning = true;
self.snapshots_at_clean = self.snapshots;
return Action::WriteClean;
}
return Action::WakeAfter((INTERVAL_S - since).max(0.05));
}
Action::Nothing
}
/// The snapshot write for `revision` finished.
pub fn written(&mut self, revision: u64, ok: bool, now: f64) {
if !ok && self.last_revision == Some(revision) {
// retry after the interval, even if nothing else changes
self.last_revision = None;
self.last_time = Some(now);
}
}
/// The clean-meta write finished.
pub fn cleaned(&mut self, ok: bool, now: f64) {
self.cleaning = false;
if ok {
self.owns_dirty_meta = self.snapshots != self.snapshots_at_clean;
} else {
// retry after the interval
self.last_revision = None;
self.last_time = Some(now);
}
}
}

67
apps/filmcraft-web/tests/import-bins.mjs Normal file
View File

@ -0,0 +1,67 @@
// Web import destinations: active bin tab, browser drop + navigation during pending reads.
// Synthetic browser drop event exercises the production handler; physical OS drag/picker UI is outside this test.
export async function checkImportBins({ evaluate, media, timeoutMs = 120000 }) {
const report = {gates: {}, cases: []};
await evaluate('filmcraft.execute("file.newProject", {name: "Import bins"})');
const parent = (await evaluate('filmcraft.execute("file.newBin", {name: "Footage"})')).bin;
const child = (await evaluate(`filmcraft.execute("file.newBin", {name: "Selects", parent: ${parent}})`)).bin;
await evaluate(`(async () => {window.__importBinFixture = new Uint8Array(await (await fetch(${JSON.stringify(media)})).arrayBuffer());})()`);
const setBin = bin => evaluate(`filmcraft.request("ui.set", {projectPanel: {bin: ${JSON.stringify(bin)}, activeTab: null}})`);
const importExpression = name => `filmcraft.importFiles([new File([window.__importBinFixture], ${JSON.stringify(name)}, {type: "video/mp4"})])`;
const locate = async name => {
const deadline = Date.now() + timeoutMs;
const pollMs = 100;
for (;;) {
const project = await evaluate('filmcraft.execute("project.inspect")');
const find = bin => {
for (const entry of bin.children) {
if (entry.item !== undefined && entry.name === name) return bin.id;
if (entry.children) {const found = find(entry); if (found !== undefined) return found;}
}
};
const found = find(project.root);
if (found !== undefined) return found;
if (Date.now() >= deadline) throw new Error(`Import never appeared: ${name}`);
await new Promise(resolve => setTimeout(resolve, pollMs));
}
};
const check = async (key, expected, name) => {
const actual = await locate(name);
report.cases.push({key, expected, actual, name}); report.gates[key] = actual === expected;
};
await evaluate(`filmcraft.request("ui.set", {projectPanel: {bin: ${child}, activeTab: 0, tabs: [{bin: ${parent}}]}})`);
await evaluate(importExpression("tab.mp4")); await check("activeTab", parent, "tab.mp4");
await setBin(child);
await evaluate(`(() => {
const transfer = new DataTransfer();
transfer.items.add(new File([window.__importBinFixture], "drop.mp4", {type: "video/mp4"}));
window.dispatchEvent(new DragEvent("drop", {dataTransfer: transfer, bubbles: true, cancelable: true}));
})()`);
await check("browserDrop", child, "drop.mp4");
await setBin(child);
await evaluate(`(() => {
const original = Blob.prototype.arrayBuffer;
let release;
const pending = new Promise(resolve => {release = resolve;});
const gate = window.__importBinGate = {hits: 0, release, original};
Blob.prototype.arrayBuffer = function () {gate.hits++; return pending.then(() => original.call(this));};
window.__pendingBinImport = ${importExpression("pending.mp4")};
})()`);
try {
const deadline = Date.now() + timeoutMs;
const pollMs = 100;
for (;;) {
if (await evaluate("window.__importBinGate.hits > 0")) break;
if (Date.now() >= deadline) throw new Error("Read-delay probe did not intercept any Blob reads");
await new Promise(resolve => setTimeout(resolve, pollMs));
}
report.delayedReadHits = await evaluate("window.__importBinGate.hits");
await setBin(parent);
await evaluate("window.__importBinGate.release(); window.__pendingBinImport");
await check("destinationCapturedBeforeRead", child, "pending.mp4");
} finally {
await evaluate("Blob.prototype.arrayBuffer = window.__importBinGate.original; window.__importBinGate.release(); delete window.__importBinGate; delete window.__pendingBinImport; delete window.__importBinFixture");
}
report.ok = Object.values(report.gates).every(Boolean);
return report;
}

78
apps/filmcraft-web/tests/import-collision.mjs Normal file
View File

@ -0,0 +1,78 @@
// #194 regression: same-name/same-size imports must retain independent sources across OPFS recovery.
// evaluate(expression) awaits a browser expression; navigate(url) waits for FilmCraft readiness. Generated PCM16 mono WAVs only; no binary fixtures.
import { createHash } from "node:crypto";
function tone(frequency, seconds) {
// RIFF/WAVE PCM16 mono: 44-byte header + 2 bytes/sample; fixture matches #194 (48 kHz, amplitude 12000).
const rate = 48000, bytesPerSample = 2, samples = rate * seconds;
const b = Buffer.alloc(44 + samples * bytesPerSample);
b.write("RIFF", 0); b.writeUInt32LE(b.length - 8, 4); b.write("WAVEfmt ", 8);
b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22);
b.writeUInt32LE(rate, 24); b.writeUInt32LE(rate * bytesPerSample, 28);
b.writeUInt16LE(bytesPerSample, 32); b.writeUInt16LE(16, 34); b.write("data", 36); b.writeUInt32LE(samples * bytesPerSample, 40);
for (let i = 0; i < samples; i++) b.writeInt16LE(Math.round(12000 * Math.sin(2 * Math.PI * frequency * i / rate)), 44 + bytesPerSample * i);
return b;
}
export async function checkImportCollision({ evaluate, navigate, url, timeoutMs = 120000 }) {
const fixtures = [tone(440, 2), tone(880, 2)];
const hashes = fixtures.map(b => createHash("sha256").update(b).digest("hex"));
const report = { hashes, fixtureBytes: fixtures.map(b => b.length), gates: {} };
const hash = expression => evaluate(`(async () => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", ${expression})), b => b.toString(16).padStart(2, "0")).join(""))()`);
const readHash = path => hash(`await filmcraft.readFile(${JSON.stringify(path)})`);
const importFile = index => evaluate(`filmcraft.importFiles([new File([Uint8Array.from(atob(${JSON.stringify(fixtures[index].toString("base64"))}), c => c.charCodeAt(0))], "collision.wav", {type: "audio/wav"})])`);
await evaluate('filmcraft.execute("file.newProject", {name: "Import collision"})');
const a = await importFile(0); report.a = a;
if (a.errors?.length || !a.items?.length) throw new Error("baseline import failed: " + JSON.stringify(a));
report.initialHash = await readHash(a.paths[0]);
report.gates.healthyInitial = report.initialHash === hashes[0];
await evaluate(`filmcraft.execute("file.newSequence", {name: "Original A", fromItem: ${a.items[0]}})`);
const b = await importFile(1); report.b = b;
if (b.errors?.length || !b.items?.length) throw new Error("second import failed: " + JSON.stringify(b));
report.afterBHash = await readHash(a.paths[0]);
report.gates.distinctPaths = a.paths[0] !== b.paths[0];
report.gates.originalPreserved = report.afterBHash === hashes[0];
report.beforeProject = await evaluate('filmcraft.execute("project.inspect")');
const revision = report.beforeProject.revision;
const deadline = Date.now() + timeoutMs;
const persistencePollMs = 100;
for (;;) {
const snapshot = await evaluate(`(async () => {
try {
const dir = await (await navigator.storage.getDirectory()).getDirectoryHandle("recovery");
const read = async name => JSON.parse(await (await (await dir.getFileHandle(name)).getFile()).text());
return {meta: await read("meta.json"), project: await read("snapshot.fcproj")};
} catch (e) {return {error: String(e)};}
})()`);
report.recoveryMeta = snapshot.meta;
if (snapshot.meta?.dirty && snapshot.meta.revision === revision && JSON.stringify(snapshot.project).includes("Original A")) break;
if (Date.now() >= deadline) throw new Error("Recovery persistence gate failed: " + JSON.stringify(snapshot));
await new Promise(resolve => setTimeout(resolve, persistencePollMs));
}
// Media writes run separately from the snapshot queue; gate their actual bytes before navigation.
for (;;) {
report.opfs = await evaluate(`(async () => {
const dir = await (await navigator.storage.getDirectory()).getDirectoryHandle("media");
const files = [];
for await (const [name, handle] of dir.entries()) {
const bytes = await (await handle.getFile()).arrayBuffer();
files.push({name, hash: Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), b => b.toString(16).padStart(2, "0")).join("")});
}
return files;
})()`);
// Before the fix, A and B intentionally share the same path; qualify the last write there.
const expected = new Map([[a.paths[0], hashes[0]], [b.paths[0], hashes[1]]]);
if ([...expected].every(([path, digest]) => report.opfs.some(f => f.name === path.slice("/files/".length) && f.hash === digest))) break;
if (Date.now() >= deadline) throw new Error("Imported media did not reach OPFS: " + JSON.stringify(report.opfs));
await new Promise(resolve => setTimeout(resolve, persistencePollMs));
}
const recoveryUrl = new URL(url);
recoveryUrl.searchParams.delete("fresh"); recoveryUrl.searchParams.delete("norecover");
await navigate(recoveryUrl.href);
report.recovered = await evaluate('filmcraft.execute("project.inspect")');
report.afterReload = {a: await readHash(a.paths[0]), b: await readHash(b.paths[0])};
report.gates.recoveredProject = JSON.stringify(report.recovered).includes("Original A");
report.gates.recoveredSources = report.afterReload.a === hashes[0] && report.afterReload.b === hashes[1];
report.ok = Object.values(report.gates).every(Boolean);
return report;
}

77
apps/filmcraft-web/tests/recovery_policy.rs Normal file
View File

@ -0,0 +1,77 @@
//! The web recovery snapshot policy. It is pure, so it runs natively even though the rest of the
//! crate is wasm32-only.
#[path = "../src/recovery_policy.rs"]
mod recovery_policy;
use recovery_policy::{Action, INTERVAL_S, Policy};
/// #197: a snapshot write that fails is retried after the interval without another edit.
#[test]
fn failed_snapshot_write_is_retried_without_another_edit() {
let mut p = Policy::default();
assert_eq!(p.tick(0.0, 3, 0), Action::WriteSnapshot(3));
p.written(3, false, 0.1);
// not immediately (no spinning on a failing disk)…
assert!(matches!(p.tick(1.0, 3, 0), Action::WakeAfter(_)));
// …but once the interval has passed, with the revision unchanged
assert_eq!(p.tick(0.1 + INTERVAL_S, 3, 0), Action::WriteSnapshot(3));
p.written(3, true, 5.2);
assert_eq!(p.tick(20.0, 3, 0), Action::Nothing);
}
/// A failure reported for an older revision doesn't make the newer one be written again.
#[test]
fn stale_failure_is_ignored() {
let mut p = Policy::default();
assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1));
assert_eq!(p.tick(INTERVAL_S, 2, 0), Action::WriteSnapshot(2));
p.written(1, false, INTERVAL_S);
assert_eq!(p.tick(3.0 * INTERVAL_S, 2, 0), Action::Nothing);
}
/// #193: a clean session (started with `?norecover` / `?fresh`, so the snapshot left by an
/// earlier session was not reopened) never marks that snapshot clean.
#[test]
fn clean_start_leaves_an_unconsumed_snapshot_dirty() {
let mut p = Policy::default();
for t in 0..100 {
assert_eq!(p.tick(t as f64, 0, 0), Action::Nothing);
}
}
/// The snapshot this session wrote is retired once the project is saved.
#[test]
fn saving_marks_own_snapshot_clean() {
let mut p = Policy::default();
assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1));
p.written(1, true, 0.1);
assert_eq!(p.tick(1.0, 1, 1), Action::WriteClean);
assert_eq!(p.tick(1.1, 1, 1), Action::Nothing);
p.cleaned(true, 1.2);
assert_eq!(p.tick(30.0, 1, 1), Action::Nothing);
}
/// A failed clean write is retried after the interval.
#[test]
fn failed_clean_write_is_retried() {
let mut p = Policy::default();
assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1));
p.written(1, true, 0.1);
assert_eq!(p.tick(1.0, 1, 1), Action::WriteClean);
p.cleaned(false, 1.1);
assert!(matches!(p.tick(2.0, 1, 1), Action::WakeAfter(_)));
assert_eq!(p.tick(1.1 + INTERVAL_S, 1, 1), Action::WriteClean);
}
/// An edit made while the clean write is in flight keeps the meta dirty after it lands.
#[test]
fn edit_during_clean_write_keeps_meta_owned() {
let mut p = Policy::default();
assert_eq!(p.tick(0.0, 1, 0), Action::WriteSnapshot(1));
assert_eq!(p.tick(10.0, 1, 1), Action::WriteClean);
assert_eq!(p.tick(20.0, 2, 1), Action::WriteSnapshot(2));
p.cleaned(true, 20.1);
// saved again: the newer snapshot must be retired too
assert_eq!(p.tick(30.0, 2, 2), Action::WriteClean);
}

15
apps/filmcraft-web/tests/smoke.mjs
View File

@ -13,6 +13,8 @@ import { spawn } from "node:child_process";
import { mkdirSync, writeFileSync, readdirSync, statSync, mkdtempSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { checkImportBins } from "./import-bins.mjs";
import { checkImportCollision } from "./import-collision.mjs";
const arg = (k, d) => {
const i = process.argv.indexOf(`--${k}`);
@ -162,6 +164,19 @@ try {
await sleep(500);
await send("Emulation.setDeviceMetricsOverride", { width: 1600, height: 1000, deviceScaleFactor: 1, mobile: false });
report.steps.modes = { screenshot: await shot("05-modes") };
report.steps.importBins = await checkImportBins({evaluate: js, media});
if (!report.steps.importBins.ok) throw new Error("import bin regression: " + JSON.stringify(report.steps.importBins.gates));
report.steps.importCollision = await checkImportCollision({
evaluate: js,
url,
navigate: async (nextUrl) => {
await send("Page.navigate", { url: nextUrl });
await until("!!(window.filmcraftLoad && (window.filmcraftLoad.readyMs || window.filmcraftLoad.error))", 120000);
const load = await js("window.filmcraftLoad");
if (load.error) throw new Error(load.error);
},
});
if (!report.steps.importCollision.ok) throw new Error("import collision regression: " + JSON.stringify(report.steps.importCollision.gates));
// still alive? (a panicked app never answers)
await Promise.race([js("filmcraft.inspect().then(() => true)"), sleep(5000).then(() => { throw new Error("app stopped answering"); })]);
const fatal = logs.filter((l) => /panicked at|RuntimeError|EXCEPTION/.test(l));

562
apps/filmcraft/src/logging.rs Normal file
View File

@ -0,0 +1,562 @@
//! The desktop app's logger: `log::` records go to standard error and to a log file.
//!
//! Without a logger every `log::warn!`/`log::error!` in the workspace (GPU compositor fallbacks,
//! audio output problems, a panicked frame job or audio mixer, container warnings) vanished. A
//! launch from a desktop menu has no terminal, so the file is what a bug report can attach:
//! `<data dir>/Logs/filmcraft.log`, next to the crash logs (`--data-dir`, `FILMCRAFT_DATA_DIR` or
//! `filmcraft_engine::autosave::default_data_dir`; Help ▸ Reveal Log Files opens the folder). Each
//! start moves the previous log to `filmcraft.1.log` (and that one to `.2`), so the log of a run
//! that crashed survives the next launch.
//!
//! Levels: `info` for FilmCraft's own crates, `warn` for everything else (wgpu and naga are
//! chatty). `RUST_LOG` replaces that with env_logger-style directives: `debug`,
//! `warn,filmcraft_codecs=trace`, `wgpu_core=info`. A directive ending in `*` matches every target
//! that starts with it (`filmcraft*=debug`).
//!
//! Records logged before the data directory is known are kept (up to [`MAX_PENDING`]) and written
//! once the file is attached. Writing never panics: a file that can't be created or written leaves
//! standard error as the only sink.
//!
//! Realtime audio: the cpal callbacks (`audio.rs`, `audio_in.rs`, the play-ahead reader) do not
//! log; they report through flags and a one-time `eprintln!`. A record takes the sink lock only for
//! one `write_all` of an already formatted line (standard error is written outside the lock), so a
//! logging thread such as the audio mixer never waits on more than one other line being written.
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::{Mutex, OnceLock, PoisonError};
use std::time::SystemTime;
use log::LevelFilter;
/// The current log file's name inside the log directory.
pub const LOG_FILE: &str = "filmcraft.log";
/// The log directory inside the data directory (shared with the crash logs and Reveal Log Files).
pub const LOG_DIR: &str = "Logs";
/// How many previous logs are kept (`filmcraft.1.log` … `filmcraft.<KEEP>.log`).
pub const KEEP: usize = 2;
/// The log file stops growing past this size (a runaway warning can't fill the disk).
pub const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024;
/// Records kept in memory until the log file is attached.
pub const MAX_PENDING: usize = 512;
/// The built-in filter when `RUST_LOG` is unset or empty.
pub const DEFAULT_FILTER: &str = "warn,filmcraft*=info";
/// Per-target level filter parsed from env_logger-style directives.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Filter {
default: LevelFilter,
directives: Vec<(String, LevelFilter)>,
}
impl Filter {
/// Parse `spec`; unknown levels and empty parts are skipped. Without a bare level, targets no
/// directive names log errors only.
pub fn parse(spec: &str) -> Filter {
let mut f = Filter { default: LevelFilter::Error, directives: Vec::new() };
for part in spec.split(',').map(str::trim).filter(|p| !p.is_empty()) {
match part.split_once('=') {
Some((name, level)) => {
let name = name.trim();
if let (false, Some(level)) = (name.is_empty(), parse_level(level)) {
f.directives.push((name.to_owned(), level));
}
}
None => match parse_level(part) {
Some(level) => f.default = level,
// A bare target name: everything from it (env_logger does the same).
None => f.directives.push((part.to_owned(), LevelFilter::Trace)),
},
}
}
f
}
/// The level that applies to `target` (the most specific matching directive wins).
pub fn level_for(&self, target: &str) -> LevelFilter {
self.directives.iter().filter(|(name, _)| matches(name, target)).max_by_key(|(name, _)| name.len()).map_or(self.default, |(_, level)| *level)
}
/// The most verbose level any target can reach (for `log::set_max_level`).
pub fn max(&self) -> LevelFilter {
self.directives.iter().map(|(_, level)| *level).fold(self.default, Ord::max)
}
}
fn parse_level(s: &str) -> Option<LevelFilter> {
s.trim().parse().ok()
}
/// `name` matches `target` itself and its submodules (`a` matches `a` and `a::b`, not `ab`);
/// `name*` matches every target starting with `name`.
fn matches(name: &str, target: &str) -> bool {
match name.strip_suffix('*') {
Some(prefix) => target.starts_with(prefix),
None => target.strip_prefix(name).is_some_and(|rest| rest.is_empty() || rest.starts_with("::")),
}
}
/// `2026-10-08T07:59:17.728Z` (UTC, milliseconds). A clock before 1970 reads as the epoch.
pub fn timestamp(t: SystemTime) -> String {
let since = t.duration_since(std::time::UNIX_EPOCH).unwrap_or_default();
let secs = since.as_secs();
let (days, rem) = (secs / 86_400, secs % 86_400);
let (y, m, d) = civil_from_days(days);
format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.{:03}Z", rem / 3600, rem % 3600 / 60, rem % 60, since.subsec_millis())
}
/// Days since 1970-01-01 to a proleptic Gregorian (year, month, day) (Howard Hinnant's
/// `civil_from_days`, unsigned because `timestamp` clamps at the epoch; saturating so an absurd
/// clock can't overflow).
fn civil_from_days(days: u64) -> (u64, u64, u64) {
let z = days.saturating_add(719_468);
let era = z / 146_097;
let doe = z % 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = era.saturating_mul(400).saturating_add(yoe).saturating_add(u64::from(m <= 2));
(y, m, d)
}
fn numbered(dir: &Path, n: usize) -> PathBuf {
if n == 0 { dir.join(LOG_FILE) } else { dir.join(format!("filmcraft.{n}.log")) }
}
/// Shift the previous logs up one (`.log` → `.1.log` → … → `.<KEEP>.log`, the oldest dropped)
/// and create a fresh, empty log file in `dir` (created if missing).
pub fn rotate(dir: &Path) -> std::io::Result<(PathBuf, File)> {
std::fs::create_dir_all(dir)?;
for n in (1..=KEEP).rev() {
match std::fs::rename(numbered(dir, n - 1), numbered(dir, n)) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e),
_ => {}
}
}
let path = numbered(dir, 0);
let file = File::create(&path)?;
Ok((path, file))
}
/// One formatted log line: `<timestamp> <LEVEL> [<thread>] <target>: <message>\n`.
pub fn format_line(ts: &str, level: log::Level, thread: &str, target: &str, message: &str) -> String {
let message = message.strip_suffix('\n').unwrap_or(message);
format!("{ts} {level:<5} [{thread}] {target}: {message}\n")
}
/// Where formatted lines go besides standard error: kept in memory until a file is attached,
/// then appended to it until it reaches its size cap.
pub struct Sink {
file: Option<File>,
pending: Vec<String>,
dropped: usize,
written: u64,
max_bytes: u64,
capped: bool,
}
impl Sink {
pub fn new(max_bytes: u64) -> Sink {
Sink { file: None, pending: Vec::new(), dropped: 0, written: 0, max_bytes, capped: false }
}
/// Append `line` to the file, or keep it until [`Sink::attach`].
pub fn write(&mut self, line: &str) {
if self.file.is_none() {
if self.capped {
return;
}
if self.pending.len() < MAX_PENDING {
self.pending.push(line.to_owned());
} else {
self.dropped = self.dropped.saturating_add(1);
}
return;
}
self.append(line);
}
/// Start writing to `file`, first the lines kept so far.
pub fn attach(&mut self, file: File) {
self.file = Some(file);
for line in std::mem::take(&mut self.pending) {
self.append(&line);
}
if self.dropped > 0 {
let note = format!("{} earlier log lines were dropped before the log file was opened\n", self.dropped);
self.dropped = 0;
self.append(&note);
}
}
/// No file will come: drop the kept lines and keep none from now on.
pub fn close(&mut self) {
self.file = None;
self.pending = Vec::new();
self.capped = true;
}
fn append(&mut self, line: &str) {
if self.capped {
return;
}
let Some(file) = self.file.as_mut() else { return };
let len = u64::try_from(line.len()).unwrap_or(u64::MAX);
let text = if self.written.saturating_add(len) > self.max_bytes {
self.capped = true;
format!("log file reached {} bytes; later records go to standard error only\n", self.max_bytes)
} else {
self.written = self.written.saturating_add(len);
line.to_owned()
};
// A full disk or a vanished file must not take the app down: drop the file sink (and
// stop buffering, since no file will come).
if file.write_all(text.as_bytes()).is_err() {
self.file = None;
self.capped = true;
}
}
}
/// The installed logger: a [`Filter`], standard error and a [`Sink`].
pub struct AppLogger {
filter: Filter,
stderr: bool,
sink: Mutex<Sink>,
}
impl AppLogger {
pub fn new(filter: Filter, stderr: bool) -> AppLogger {
AppLogger { filter, stderr, sink: Mutex::new(Sink::new(MAX_FILE_BYTES)) }
}
/// Rotate the logs in `dir` and send records to the fresh file; returns its path.
/// An unusable directory leaves standard error as the only sink ([`AppLogger::stderr_only`]).
pub fn attach_dir(&self, dir: &Path) -> Result<PathBuf, String> {
match rotate(dir) {
Ok((path, file)) => {
self.sink.lock().unwrap_or_else(PoisonError::into_inner).attach(file);
Ok(path)
}
Err(e) => {
self.stderr_only();
Err(format!("{}: {e}", dir.display()))
}
}
}
/// No log file this run (no data directory, or it can't be used): stop keeping records for one.
pub fn stderr_only(&self) {
self.sink.lock().unwrap_or_else(PoisonError::into_inner).close();
}
}
impl log::Log for AppLogger {
fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
metadata.level() <= self.filter.level_for(metadata.target())
}
fn log(&self, record: &log::Record<'_>) {
if !self.enabled(record.metadata()) {
return;
}
// Format before taking the lock, so the lock covers one write only.
let thread = std::thread::current();
let line = format_line(&timestamp(SystemTime::now()), record.level(), thread.name().unwrap_or("?"), record.target(), &record.args().to_string());
if self.stderr {
let _ = std::io::stderr().write_all(line.as_bytes());
}
self.sink.lock().unwrap_or_else(PoisonError::into_inner).write(&line);
}
fn flush(&self) {}
}
/// Install the logger (filter from `RUST_LOG`, else [`DEFAULT_FILTER`]); `None` when another
/// logger was installed first. Call [`AppLogger::attach_dir`] once the data directory is known.
pub fn install() -> Option<&'static AppLogger> {
static LOGGER: OnceLock<AppLogger> = OnceLock::new();
let spec = std::env::var("RUST_LOG").ok().filter(|s| !s.trim().is_empty());
let filter = Filter::parse(spec.as_deref().unwrap_or(DEFAULT_FILTER));
let max = filter.max();
let logger = LOGGER.get_or_init(|| AppLogger::new(filter, true));
log::set_logger(logger).ok()?;
log::set_max_level(max);
Some(logger)
}
#[cfg(test)]
mod tests {
use super::*;
use log::Log;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, UNIX_EPOCH};
fn temp_dir(tag: &str) -> PathBuf {
static N: AtomicUsize = AtomicUsize::new(0);
let d = std::env::temp_dir().join(format!("filmcraft-logging-{tag}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed)));
let _ = std::fs::remove_dir_all(&d);
d
}
fn read(path: &Path) -> String {
std::fs::read_to_string(path).expect("read log")
}
fn record(logger: &AppLogger, level: log::Level, target: &'static str, msg: &'static str) {
logger.log(&log::Record::builder().level(level).target(target).args(format_args!("{msg}")).build());
}
#[test]
fn the_default_filter_shows_filmcraft_info_and_other_crates_warnings() {
let f = Filter::parse(DEFAULT_FILTER);
assert_eq!(f.level_for("filmcraft"), LevelFilter::Info);
assert_eq!(f.level_for("filmcraft::audio"), LevelFilter::Info);
assert_eq!(f.level_for("filmcraft_ui_egui::play_ahead"), LevelFilter::Info);
assert_eq!(f.level_for("filmcraft_codecs::mxf"), LevelFilter::Info);
assert_eq!(f.level_for("wgpu_core::device"), LevelFilter::Warn);
assert_eq!(f.level_for("naga"), LevelFilter::Warn);
assert_eq!(f.max(), LevelFilter::Info);
}
#[test]
fn directives_follow_env_logger_and_the_most_specific_one_wins() {
let f = Filter::parse("info,wgpu_core=error,filmcraft_codecs=trace,filmcraft_codecs::mxf=off");
assert_eq!(f.level_for("eframe"), LevelFilter::Info);
assert_eq!(f.level_for("wgpu_core::instance"), LevelFilter::Error);
assert_eq!(f.level_for("filmcraft_codecs::ogg"), LevelFilter::Trace);
assert_eq!(f.level_for("filmcraft_codecs::mxf"), LevelFilter::Off);
assert_eq!(f.max(), LevelFilter::Trace);
// A module name is matched at `::` boundaries, not as a bare prefix.
assert_eq!(f.level_for("wgpu_core_extra"), LevelFilter::Info);
// A trailing `*` is a prefix.
assert_eq!(Filter::parse("warn,filmcraft_ui*=debug").level_for("filmcraft_ui_egui::frames"), LevelFilter::Debug);
assert_eq!(Filter::parse("warn,filmcraft_ui*=debug").level_for("filmcraft_engine"), LevelFilter::Warn);
// A bare target name sets that target to the most verbose level, as env_logger does.
assert_eq!(Filter::parse("naga").level_for("naga::front"), LevelFilter::Trace);
assert_eq!(Filter::parse("naga").level_for("eframe"), LevelFilter::Error);
// Levels are case-insensitive; the last bare level wins.
assert_eq!(Filter::parse("debug,WARN").level_for("x"), LevelFilter::Warn);
}
#[test]
fn hostile_specs_never_panic_and_fall_back_sensibly() {
let long = "a".repeat(100_000);
let specs = [
"",
",,,",
"=",
"==",
"=debug",
"filmcraft=",
"nonsense=loud",
"🦀=info",
"*",
"*=debug",
"a=b=c",
"warn,,filmcraft=DEBUG",
" \t ",
"\u{0}",
"é*=trace",
&long,
];
for spec in specs {
let f = Filter::parse(spec);
for target in ["filmcraft", "", "🦀", "é", "a::b"] {
let _ = f.level_for(target);
}
let _ = f.max();
}
assert_eq!(Filter::parse("warn,,filmcraft=DEBUG").level_for("filmcraft"), LevelFilter::Debug);
assert_eq!(Filter::parse("nonsense=loud").level_for("nonsense"), LevelFilter::Error);
assert_eq!(Filter::parse("=debug").max(), LevelFilter::Error, "a directive without a name is skipped");
assert_eq!(Filter::parse("*=debug").level_for("anything"), LevelFilter::Debug, "a lone `*` matches every target");
assert_eq!(Filter::parse("é*=trace").level_for("éa"), LevelFilter::Trace);
}
#[test]
fn timestamps_are_utc_with_milliseconds() {
assert_eq!(timestamp(UNIX_EPOCH), "1970-01-01T00:00:00.000Z");
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_millis(1_791_446_357_728)), "2026-10-08T07:59:17.728Z");
// Leap day, and the day after it.
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_164_800)), "2024-02-29T00:00:00.000Z");
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_251_199)), "2024-02-29T23:59:59.000Z");
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(1_709_251_200)), "2024-03-01T00:00:00.000Z");
// 2000 is a leap year (divisible by 400), 2100 is not.
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(951_782_400)), "2000-02-29T00:00:00.000Z");
assert_eq!(timestamp(UNIX_EPOCH + Duration::from_secs(4_107_542_400)), "2100-03-01T00:00:00.000Z");
// Before the epoch (a clock set wrong) clamps instead of panicking.
assert_eq!(timestamp(UNIX_EPOCH - Duration::from_secs(5)), "1970-01-01T00:00:00.000Z");
}
#[test]
fn rotation_keeps_the_previous_logs_and_starts_an_empty_file() {
let dir = temp_dir("rotate");
for run in 1..=4 {
let (path, _file) = rotate(&dir).expect("rotate");
assert_eq!(path, dir.join(LOG_FILE));
assert_eq!(read(&path), "");
std::fs::write(&path, format!("run {run}")).expect("write");
}
assert_eq!(read(&dir.join(LOG_FILE)), "run 4");
assert_eq!(read(&dir.join("filmcraft.1.log")), "run 3");
assert_eq!(read(&dir.join("filmcraft.2.log")), "run 2");
assert!(!dir.join("filmcraft.3.log").exists(), "only {KEEP} old logs are kept");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn rotation_into_an_unusable_directory_is_an_error_not_a_panic() {
let dir = temp_dir("blocked");
std::fs::create_dir_all(dir.parent().expect("parent")).expect("tmp");
std::fs::write(&dir, "a file where the directory should be").expect("block");
assert!(rotate(&dir).is_err());
assert!(rotate(&dir.join("below")).is_err());
let _ = std::fs::remove_file(&dir);
}
#[test]
fn a_line_carries_time_level_thread_target_and_message() {
assert_eq!(
format_line("2026-10-08T07:59:17.728Z", log::Level::Warn, "main", "filmcraft_ui_egui", "audio output unavailable: no output device"),
"2026-10-08T07:59:17.728Z WARN [main] filmcraft_ui_egui: audio output unavailable: no output device\n"
);
// A multi-line message keeps its lines; the record still ends in exactly one newline.
assert!(format_line("t", log::Level::Error, "w", "x", "a\nb\n").ends_with("x: a\nb\n"));
}
#[test]
fn lines_logged_before_the_file_exists_are_written_when_it_is_attached() {
let dir = temp_dir("pending");
let (path, file) = rotate(&dir).expect("rotate");
let mut sink = Sink::new(MAX_FILE_BYTES);
sink.write("early 1\n");
sink.write("early 2\n");
sink.attach(file);
sink.write("late\n");
assert_eq!(read(&path), "early 1\nearly 2\nlate\n");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_pending_buffer_is_bounded_and_says_how_much_it_dropped() {
let dir = temp_dir("bounded");
let (path, file) = rotate(&dir).expect("rotate");
let mut sink = Sink::new(MAX_FILE_BYTES);
for i in 0..MAX_PENDING + 5 {
sink.write(&format!("line {i}\n"));
}
sink.attach(file);
let text = read(&path);
assert_eq!(text.lines().filter(|l| l.starts_with("line ")).count(), MAX_PENDING);
assert_eq!(text.matches("5 earlier log lines were dropped").count(), 1, "{text}");
assert!(text.contains("line 0\n") && !text.contains(&format!("line {MAX_PENDING}\n")));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn exactly_max_pending_lines_drop_nothing() {
let dir = temp_dir("exact");
let (path, file) = rotate(&dir).expect("rotate");
let mut sink = Sink::new(MAX_FILE_BYTES);
for i in 0..MAX_PENDING {
sink.write(&format!("line {i}\n"));
}
sink.attach(file);
let text = read(&path);
assert_eq!(text.lines().count(), MAX_PENDING);
assert!(!text.contains("dropped"), "{text}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_file_stops_at_its_size_cap_with_one_note() {
let dir = temp_dir("cap");
let (path, file) = rotate(&dir).expect("rotate");
let mut sink = Sink::new(100);
sink.attach(file);
for i in 0..50 {
sink.write(&format!("line {i:02} with some padding\n"));
}
let text = read(&path);
assert_eq!(text.lines().filter(|l| l.starts_with("line ")).count(), 3, "{text}");
assert_eq!(text.matches("log file reached 100 bytes").count(), 1, "{text}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_logger_filters_records_and_writes_them_to_the_attached_file() {
let dir = temp_dir("logger");
let logger = AppLogger::new(Filter::parse(DEFAULT_FILTER), false);
record(&logger, log::Level::Info, "filmcraft", "FilmCraft starting");
record(&logger, log::Level::Info, "wgpu_core::device", "too chatty");
let path = logger.attach_dir(&dir).expect("attach");
assert_eq!(path, dir.join(LOG_FILE));
record(&logger, log::Level::Warn, "wgpu_hal::vulkan", "a real warning");
record(&logger, log::Level::Debug, "filmcraft", "below info");
let text = read(&path);
let lines: Vec<&str> = text.lines().collect();
assert_eq!(lines.len(), 2, "{text}");
assert!(lines.first().is_some_and(|l| l.contains(" INFO [") && l.ends_with("filmcraft: FilmCraft starting")), "{text}");
assert!(lines.get(1).is_some_and(|l| l.ends_with("wgpu_hal::vulkan: a real warning")), "{text}");
assert!(logger.enabled(&log::Metadata::builder().level(log::Level::Info).target("filmcraft_engine::autosave").build()));
assert!(!logger.enabled(&log::Metadata::builder().level(log::Level::Info).target("naga").build()));
// A second launch rotates: the first log becomes `.1`.
let second = AppLogger::new(Filter::parse(DEFAULT_FILTER), false);
assert_eq!(second.attach_dir(&dir).expect("attach again"), dir.join(LOG_FILE));
assert_eq!(read(&dir.join("filmcraft.1.log")), text);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn records_from_many_threads_are_whole_lines() {
let dir = temp_dir("threads");
let logger = AppLogger::new(Filter::parse("info"), false);
let path = logger.attach_dir(&dir).expect("attach");
std::thread::scope(|s| {
for _ in 0..4 {
s.spawn(|| {
for _ in 0..100 {
record(&logger, log::Level::Info, "filmcraft", "from a worker");
}
});
}
});
let text = read(&path);
assert_eq!(text.lines().count(), 400);
assert!(text.lines().all(|l| l.ends_with("filmcraft: from a worker")), "{text}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn attaching_to_an_unusable_directory_reports_an_error_and_keeps_logging() {
let dir = temp_dir("unusable");
std::fs::write(&dir, "not a directory").expect("block");
let logger = AppLogger::new(Filter::parse("info"), false);
let e = logger.attach_dir(&dir).expect_err("a file is not a log directory");
assert!(e.contains(&dir.display().to_string()), "{e}");
record(&logger, log::Level::Error, "x", "still fine");
let sink = logger.sink.lock().expect("sink");
assert!(sink.file.is_none() && sink.pending.is_empty(), "nothing is kept for a file that will not come");
drop(sink);
let _ = std::fs::remove_file(&dir);
}
#[test]
fn a_poisoned_lock_does_not_stop_logging() {
let dir = temp_dir("poison");
let logger = AppLogger::new(Filter::parse("info"), false);
let path = logger.attach_dir(&dir).expect("attach");
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _guard = logger.sink.lock();
panic!("poison the sink");
}));
assert!(logger.sink.is_poisoned());
record(&logger, log::Level::Info, "filmcraft", "after the panic");
assert!(read(&path).contains("after the panic"));
let _ = std::fs::remove_dir_all(&dir);
}
}

41
apps/filmcraft/src/main.rs
View File

@ -16,6 +16,9 @@
//! with unsaved changes the app asks to recover them; `--recover` recovers the newest without
//! asking, `--no-recover` starts without asking (the changes stay available via File ▸ Recover
//! Unsaved Changes…).
//!
//! `log` records go to standard error and `<data dir>/Logs/filmcraft.log` (`RUST_LOG` sets the
//! levels); see [`logging`].
#![cfg_attr(all(target_os = "windows", not(debug_assertions)), windows_subsystem = "windows")]
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
@ -26,6 +29,7 @@ mod audio_in;
mod control_server;
#[cfg(any(target_os = "windows", test))]
mod graphics;
mod logging;
#[cfg(target_os = "macos")]
mod native_menu;
mod window_raise;
@ -68,6 +72,8 @@ fn tell(text: &str, error: bool) {
}
fn main() -> eframe::Result {
// First, so the panic hook and every start-up record are logged (`logging`).
let logger = logging::install();
// lossy, so a value that is not Unicode is reported like any other bad port instead of ignored
let env_port = std::env::var_os("FILMCRAFT_CONTROL_PORT").map(|p| p.to_string_lossy().into_owned());
let Launch { control_port, files, demo, startup_flag, recover, data_dir } = match args::parse(env_port, std::env::args().skip(1)) {
@ -85,10 +91,24 @@ fn main() -> eframe::Result {
std::process::exit(2);
}
};
let log_dir = data_dir.clone().or_else(default_data_dir).map(|d| d.join(logging::LOG_DIR));
log::info!("FilmCraft {} ({} {})", env!("CARGO_PKG_VERSION"), std::env::consts::OS, std::env::consts::ARCH);
// The log file lives next to the crash logs; opened after the arguments, so `--help`,
// `--version` and usage errors leave no file behind. Records logged until now go in first.
if let Some(logger) = logger {
match log_dir.as_deref().map(|d| logger.attach_dir(d)) {
Some(Ok(path)) => log::info!("log file {}", path.display()),
Some(Err(e)) => log::warn!("no log file: {e}"),
None => {
logger.stderr_only();
log::warn!("no log file: no data directory (set FILMCRAFT_DATA_DIR or --data-dir)");
}
}
}
app_nap::disable();
// Panics anywhere go to <data dir>/Logs/crash-<day>.log with a backtrace; the UI pass and
// frame workers catch them and keep running (see filmcraft_ui_egui::crash).
filmcraft_ui_egui::crash::install(data_dir.clone().or_else(default_data_dir).map(|d| d.join("Logs")));
filmcraft_ui_egui::crash::install(log_dir);
// OS hardware video decoders (VideoToolbox on macOS) in front of our own; Settings ▸ Playback ▸
// Hardware decoding switches them off. Unsupported streams and failures use our decoders.
register_hardware_decoders();
@ -109,7 +129,7 @@ fn main() -> eframe::Result {
event_loop_builder: agent_event_loop(control_port.is_some()),
..Default::default()
};
// Before eframe creates the wgpu instance: default Windows to DirectX 12 only (see graphics.rs).
// Before eframe creates the wgpu instance: leave OpenGL out on Windows (see graphics.rs).
#[cfg(target_os = "windows")]
let options = {
let mut options = options;
@ -125,7 +145,7 @@ fn main() -> eframe::Result {
let mut cfg = AutosaveConfig::new(dir);
cfg.local_offset = local_offset;
if let Err(e) = session.start_autosave(cfg) {
eprintln!("filmcraft: auto-save and crash recovery unavailable: {e}");
log::warn!("auto-save and crash recovery unavailable: {e}");
}
}
// voice-over recording reads the microphone through cpal
@ -133,7 +153,7 @@ fn main() -> eframe::Result {
let project = files.iter().find(|f| f.ends_with(".fcproj")).cloned();
if let Some(p) = project {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
eprintln!("filmcraft: {e}");
log::error!("{e}");
}
} else if !startup_flag && session.prefs.general.at_startup == "openMostRecent" {
// Settings ▸ General ▸ At Startup ▸ Open Most Recent
@ -141,7 +161,7 @@ fn main() -> eframe::Result {
match recent {
Some(p) => {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
eprintln!("filmcraft: {e}");
log::error!("{e}");
}
}
None => {
@ -159,8 +179,8 @@ fn main() -> eframe::Result {
if recover == Some(true) && !session.recovery_candidates().is_empty() {
let id = session.recovery_candidates()[0].id.clone();
match session.execute("file.recover", json!({"id": id})) {
Ok(r) => eprintln!("filmcraft: recovered {r}"),
Err(e) => eprintln!("filmcraft: recovery failed: {e}"),
Ok(r) => log::info!("recovered {r}"),
Err(e) => log::error!("recovery failed: {e}"),
}
}
let mut app = FilmcraftApp::new(session);
@ -216,7 +236,7 @@ fn main() -> eframe::Result {
// instead of exiting silently, and keep the reason in the crash log.
if let Err(e) = &started {
let msg = format!("FilmCraft could not start its window: {e}\n\nUpdating the graphics driver usually fixes this.");
eprintln!("filmcraft: {msg}");
log::error!("{msg}");
filmcraft_ui_egui::crash::record(&msg);
let _ = rfd::MessageDialog::new().set_title("FilmCraft").set_description(&msg).set_level(rfd::MessageLevel::Error).show();
}
@ -286,6 +306,9 @@ mod tests {
fn startup_registers_the_hardware_decoders_without_a_logger() {
assert!(!log::log_enabled!(log::Level::Info));
let hardware = super::register_hardware_decoders();
assert_eq!(filmcraft_platform::registered(), cfg!(any(target_os = "macos", target_os = "windows")), "{hardware:?}");
// always on macOS and Windows; on Linux when a VA-API driver is there
let expected = cfg!(any(target_os = "macos", target_os = "windows"))
|| (cfg!(target_os = "linux") && matches!(hardware, filmcraft_platform::Availability::Available(_)));
assert_eq!(filmcraft_platform::registered(), expected, "{hardware:?}");
}
}

10
contributors/people.toml
View File

@ -21,3 +21,13 @@
real_name = "Brandon Thomas"
display_name = "Brandon Thomas"
sync_github_name = false
[people.swillunderscore]
display_name = "Swill Software"
[people.IppSec]
sync_github_name = true
[people.ItzBloxy]
display_name = "Bloxy"

65
crates/automation/src/long_job.rs
View File

@ -107,10 +107,11 @@ impl FilmcraftMcp {
}
/// Delete what an interrupted export wrote, and nothing else: the output file itself, the numbered
/// stills of an image sequence (`<stem><digits>.<same extension>`) and the caption sidecar
/// stills of an image sequence (`<stem><3+ digits>.<same extension>`, only when the output is a
/// PNG/TIFF/BMP sequence) and the caption sidecar
/// (`<stem>.srt` / `<stem>.vtt`), each only if it was written since `since`. Other files that
/// merely share the name's stem (`<stem>.aep`, `<stem>2.psd`, …) are never touched, even when they
/// were saved during the export.
/// merely share the name's stem (`<stem>.aep`, `<stem>2.psd`, `<stem>1.mp4` beside a movie, …) are
/// never touched, even when they were saved during the export.
pub fn remove_partial(path: &str, since: SystemTime) {
let p = Path::new(path);
let (Some(dir), Some(stem), Some(file)) = (p.parent(), p.file_stem().and_then(|s| s.to_str()), p.file_name().and_then(|s| s.to_str())) else {
@ -139,9 +140,19 @@ fn is_export_output(name: &str, file: &str, stem: &str, ext: &str) -> bool {
if let Some(sidecar) = rest.strip_prefix('.') {
return sidecar.eq_ignore_ascii_case("srt") || sidecar.eq_ignore_ascii_case("vtt");
}
// image sequence: one or more digits, then the output's own extension
// image sequence: the frame number (at least three digits, like `image_sequence_path`), then the
// output's own extension; movies and audio are a single file, so numbered neighbours are not theirs
if !is_sequence_extension(ext) {
return false;
}
let digits = rest.len() - rest.trim_start_matches(|c: char| c.is_ascii_digit()).len();
digits > 0 && !ext.is_empty() && rest.get(digits..).is_some_and(|r| r.strip_prefix('.').is_some_and(|x| x == ext))
digits >= 3 && rest.get(digits..).is_some_and(|r| r.strip_prefix('.').is_some_and(|x| x == ext))
}
/// Whether an export to a file with extension `ext` writes an image sequence (one still per frame).
fn is_sequence_extension(ext: &str) -> bool {
use filmcraft_engine::export::Format;
Format::from_name(ext).is_some_and(|f| matches!(f, Format::PngSequence | Format::TiffSequence | Format::BmpSequence))
}
#[cfg(test)]
@ -170,6 +181,50 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn numbered_files_beside_a_movie_are_kept() {
let dir = std::env::temp_dir().join(format!("fc-remove-partial-movie-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let started = SystemTime::now();
// a movie export writes one file: `clip1.mp4` and `clip002.mp4` are other exports
let ours = ["clip.mp4", "clip.srt"];
let theirs = ["clip1.mp4", "clip002.mp4", "clip2.mov"];
for f in ours.iter().chain(theirs.iter()) {
std::fs::write(dir.join(f), b"x").unwrap();
}
remove_partial(&dir.join("clip.mp4").to_string_lossy(), started);
for f in ours {
assert!(!dir.join(f).exists(), "{f} should be removed");
}
for f in theirs {
assert!(dir.join(f).exists(), "{f} must not be removed");
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn sequence_frames_have_at_least_three_digits() {
let dir = std::env::temp_dir().join(format!("fc-remove-partial-digits-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let started = SystemTime::now();
// frames are numbered with at least three digits (`shot000.tiff`); `shot2.tiff` is not one
let ours = ["shot.tiff", "shot000.tiff", "shot1234.tiff"];
let theirs = ["shot2.tiff", "shot12.tiff"];
for f in ours.iter().chain(theirs.iter()) {
std::fs::write(dir.join(f), b"x").unwrap();
}
remove_partial(&dir.join("shot.tiff").to_string_lossy(), started);
for f in ours {
assert!(!dir.join(f).exists(), "{f} should be removed");
}
for f in theirs {
assert!(dir.join(f).exists(), "{f} must not be removed");
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn files_older_than_the_export_are_kept() {
let dir = std::env::temp_dir().join(format!("fc-remove-partial-old-{}", std::process::id()));

152
crates/codecs/src/audio.rs
View File

@ -87,7 +87,7 @@ impl PacketDecoder {
/// (HE-AAC, see [`aac_output_rate`]) the decoded core is upsampled to it.
pub fn aac(asc: &[u8], sample_rate: u32) -> Result<Self> {
if let Ok(dec) = filmcraft_aac::Decoder::new(asc) {
let up = (dec.sample_rate().checked_mul(2) == Some(sample_rate)).then(|| Box::new(Upsample2x::new(dec.channels())));
let up = upsamples_core(dec.sample_rate(), sample_rate).then(|| Box::new(Upsample2x::new(dec.channels())));
return Ok(Self { inner: Inner::Aac { dec: Box::new(dec), asc: asc.to_vec(), up }, channels: 0 });
}
Self::new(CODEC_TYPE_AAC, sample_rate, Some(asc.to_vec()))
@ -244,6 +244,77 @@ pub fn aac_output_rate<'a>(asc: &[u8], first_units: impl IntoIterator<Item = &'a
Some(core)
}
/// Whether [`PacketDecoder::aac`] upsamples a decoded AAC core at `core` Hz to `rate` (HE-AAC).
fn upsamples_core(core: u32, rate: u32) -> bool {
core.checked_mul(2) == Some(rate)
}
/// A codec whose packets all decode to the same number of samples (see [`fixed_packet_samples`]).
#[derive(Clone, Copy, Debug)]
pub enum FixedFrames<'a> {
/// AAC with this AudioSpecificConfig.
Aac(&'a [u8]),
/// MPEG-1/2 audio, layer I, II or III.
MpegAudio,
/// AC-3 (and E-AC-3).
Ac3,
}
/// Samples each packet decodes to at the stream's output `rate`, for codecs where that is the same
/// for every packet: AAC 1024 (2048 when [`PacketDecoder::aac`] upsamples an HE-AAC core); MPEG
/// audio and AC-3 from the frame header of `first`, the stream's first packet. `None` when it is
/// not known: an AAC configuration our decoder does not read, a header that does not parse, or a
/// header whose rate is not `rate`.
pub fn fixed_packet_samples(codec: FixedFrames, first: &[u8], rate: u32) -> Option<i64> {
let mpegts = match codec {
FixedFrames::Aac(asc) => {
let core = filmcraft_aac::Decoder::new(asc).ok()?.sample_rate();
return Some(if upsamples_core(core, rate) { 2048 } else { 1024 });
}
FixedFrames::MpegAudio => filmcraft_mpegts::Codec::MpegAudio,
FixedFrames::Ac3 => filmcraft_mpegts::Codec::Ac3,
};
let f = filmcraft_mpegts::frame_info(&mpegts, first)?;
(f.sample_rate == rate && f.samples > 0).then_some(f.samples as i64)
}
/// Where the container puts one audio packet, for [`contiguous_starts`].
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PacketTime {
/// Its timestamp, in output samples.
pub stamp: i64,
/// Whether `stamp` is the packet's own. A Matroska frame after the first in a laced block
/// without a duration repeats the block's timestamp.
pub stamped: bool,
/// Samples it decodes to, when known.
pub samples: Option<i64>,
}
/// Packet start positions in output samples: each packet starts where the previous one ends,
/// unless its own timestamp is more than `tolerance` samples from there (a real gap or overlap in
/// the recording) or the previous packet's length is unknown; then it starts at its timestamp.
///
/// Container timestamps are coarser than a sample. Matroska stores milliseconds, and a remux to MP4
/// (OBS's) turns that rounding into uneven sample durations: a 1024-sample AAC frame at 48 kHz lasts
/// 21.33 ms, so frames are stamped at 0, 1008, 2064, 3072… Placing each decoded packet at its
/// stamp overwrote the end of one packet and left a hole of silence after another, a click every
/// few packets: crackling throughout playback and export (#236).
pub fn contiguous_starts(packets: impl IntoIterator<Item = PacketTime>, tolerance: i64) -> Vec<i64> {
let tolerance = tolerance.max(0).unsigned_abs();
let mut next: Option<i64> = None;
packets
.into_iter()
.map(|p| {
let start = match next {
Some(n) if !p.stamped || n.abs_diff(p.stamp) <= tolerance => n,
_ => p.stamp,
};
next = p.samples.and_then(|k| start.checked_add(k));
start
})
.collect()
}
/// Taps on each side of the interpolation point in [`Upsample2x`].
const UPSAMPLE_HALF: usize = 12;
@ -739,3 +810,82 @@ mod he_aac_tests {
}
}
}
#[cfg(test)]
mod packet_time_tests {
use super::*;
/// 1024-sample frames at 48 kHz stamped to the nearest millisecond, then back in samples.
fn ms_stamps(count: i64) -> Vec<i64> {
(0..count).map(|k| (k * 1024 * 1000 + 24_000) / 48_000 * 48).collect()
}
fn timed(stamps: &[i64], samples: Option<i64>) -> Vec<PacketTime> {
stamps.iter().map(|&stamp| PacketTime { stamp, stamped: true, samples }).collect()
}
#[test]
fn rounded_stamps_give_back_to_back_packets() {
let stamps = ms_stamps(200);
assert_eq!(&stamps[..4], [0, 1008, 2064, 3072]);
let starts = contiguous_starts(timed(&stamps, Some(1024)), 512);
assert_eq!(starts, (0..200).map(|k| k * 1024).collect::<Vec<_>>());
}
#[test]
fn real_gaps_and_overlaps_resynchronise() {
// a 100 ms gap after packet 3, then a packet stamped 600 samples early
let mut stamps: Vec<i64> = (0..8).map(|k| k * 1024).collect();
for s in &mut stamps[4..] {
*s += 4800;
}
stamps[7] -= 600;
let starts = contiguous_starts(timed(&stamps, Some(1024)), 512);
assert_eq!(starts, [0, 1024, 2048, 3072, 8896, 9920, 10944, 11368]);
// within the tolerance: runs on
stamps[7] += 200;
assert_eq!(contiguous_starts(timed(&stamps, Some(1024)), 512)[7], 11968);
}
#[test]
fn unstamped_packets_follow_on_and_unknown_lengths_resynchronise() {
// a laced block of three frames that repeat its timestamp
let packets = [0, 1024, 1024, 1024, 4096].map(|stamp| PacketTime { stamp, stamped: true, samples: Some(1024) });
let mut laced = packets;
laced[2].stamped = false;
laced[3].stamped = false;
assert_eq!(contiguous_starts(laced, 512), [0, 1024, 2048, 3072, 4096]);
// a packet whose length is unknown: the next one starts at its own stamp
let mut unknown = timed(&[0, 1024, 2100, 3124], Some(1024));
unknown[1].samples = None;
assert_eq!(contiguous_starts(unknown, 512), [0, 1024, 2100, 3124]);
}
#[test]
fn hostile_values_stay_finite() {
let packets = [
PacketTime { stamp: i64::MAX, stamped: true, samples: Some(i64::MAX) },
PacketTime { stamp: i64::MIN, stamped: false, samples: Some(-5) },
PacketTime { stamp: 0, stamped: true, samples: Some(i64::MIN) },
PacketTime { stamp: 7, stamped: true, samples: None },
];
assert_eq!(contiguous_starts(packets, i64::MIN).len(), 4);
assert_eq!(contiguous_starts(packets, i64::MAX).len(), 4);
assert!(contiguous_starts([], 512).is_empty());
}
#[test]
fn fixed_frame_lengths() {
// AAC-LC 48 kHz stereo; HE-AAC (24 kHz core) played at 48 kHz is upsampled
assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[0x11, 0x90]), &[], 48_000), Some(1024));
assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[0x13, 0x10]), &[], 48_000), Some(2048));
assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[]), &[], 48_000), None);
// MPEG-1 layer III 44.1 kHz, 128 kbps; MPEG-2 layer III 22.05 kHz
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xFB, 0x90, 0x44], 44_100), Some(1152));
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xF3, 0x90, 0x44], 22_050), Some(576));
// the header's rate must be the stream's
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xFB, 0x90, 0x44], 48_000), None);
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF], 44_100), None);
assert_eq!(fixed_packet_samples(FixedFrames::Ac3, &[0x0B, 0x77], 48_000), None);
}
}

178
crates/codecs/src/audio_timing_tests.rs Normal file
View File

@ -0,0 +1,178 @@
//! Audio packets placed by container timestamps (#236).
//!
//! An AAC access unit decodes to 1024 samples, 21.33 ms at 48 kHz. Matroska stores timestamps in
//! whole milliseconds, and OBS's remux to MP4 carries that rounding into sample durations of 1008,
//! 1008, 1056… Placing each decoded packet at its own timestamp overwrote the end of one packet and
//! left a hole of silence after another: playback and export crackled throughout. The streams here
//! are AAC from our own encoder muxed with such timestamps; read back block by block, as playback
//! reads them, they must be exactly the access units decoded back to back.
use std::io::Cursor;
use std::sync::Arc;
use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
use filmcraft_matroska::{LacingMode, MkvWriter, MuxOptions, TrackKind, TrackSpec};
use filmcraft_media::MediaSource;
use crate::audio::PacketDecoder;
use crate::{MkvSource, Mp4Source};
/// A second of a stereo tone (440 Hz left, 660 Hz right) encoded with our AAC encoder: the
/// AudioSpecificConfig and the access units.
fn aac(rate: u32) -> (Vec<u8>, Vec<Vec<u8>>) {
let tone = |hz: f32| (0..rate as usize).map(|i| 0.5 * (std::f32::consts::TAU * hz * i as f32 / rate as f32).sin()).collect::<Vec<f32>>();
let (l, r) = (tone(440.0), tone(660.0));
let mut enc = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(rate, 2, 160_000)).unwrap();
let mut units = enc.encode(&[&l, &r]);
units.extend(enc.flush());
(enc.audio_specific_config(), units)
}
/// The access units decoded back to back.
fn continuous(asc: &[u8], rate: u32, units: &[Vec<u8>]) -> Vec<Vec<f32>> {
let mut dec = PacketDecoder::aac(asc, rate).unwrap();
let mut out = vec![Vec::new(), Vec::new()];
for u in units {
for (o, c) in out.iter_mut().zip(dec.decode(u, 0).unwrap()) {
o.extend(c);
}
}
out
}
/// Access unit `k`'s start to the nearest millisecond, as muxers stamp it.
fn stamp_ms(rate: u32, k: usize) -> i64 {
(k as i64 * 1024 * 2000 + rate as i64) / (2 * rate as i64)
}
/// Matroska (1 ms timestamps), with the units after `gap` delayed by `gap_ms` (a real gap).
fn mkv(asc: &[u8], rate: u32, units: &[Vec<u8>], lacing: LacingMode, gap: usize, gap_ms: i64) -> Arc<[u8]> {
let mut spec = TrackSpec::new(TrackKind::Audio, "A_AAC");
spec.codec_private = asc.to_vec();
spec.audio = Some((rate as f64, 2, None));
let opts = MuxOptions { lacing, ..MuxOptions::default() };
assert_eq!(opts.timestamp_scale, 1_000_000, "1 ms timestamps");
let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![spec], opts).unwrap();
for (k, u) in units.iter().enumerate() {
let ms = stamp_ms(rate, k) + if k >= gap { gap_ms } else { 0 };
w.write_frame(0, ms * 1_000_000, true, u, None).unwrap();
}
w.finish().unwrap().into_inner().into()
}
/// MP4 as OBS remuxes Matroska: each millisecond timestamp rescaled to the sample rate, the
/// differences written as sample durations.
fn mp4_remuxed(asc: &[u8], rate: u32, units: &[Vec<u8>]) -> Arc<[u8]> {
let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap();
let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc.to_vec(), 2, rate), rate)).unwrap();
let at = |k: usize| (stamp_ms(rate, k) * rate as i64 + 500) / 1000;
for (k, u) in units.iter().enumerate() {
let duration = (at(k + 1) - at(k)) as u32;
w.write_sample(t, WriteSample { data: u, duration, composition_offset: 0, is_sync: true }).unwrap();
}
w.finish().unwrap().into_inner().into()
}
/// `frames` samples per channel read in 512-sample blocks at `rate`, as playback reads them.
fn played(src: &dyn MediaSource, rate: u32, frames: usize) -> Vec<Vec<f32>> {
let mut out = vec![Vec::new(), Vec::new()];
let mut pos = 0;
while pos < frames {
let n = 512.min(frames - pos);
let b = src.audio(pos as i64, n, rate).unwrap();
for (o, c) in out.iter_mut().zip(&b.channels) {
o.extend_from_slice(c);
}
pos += n;
}
out
}
/// Samples of `got` that differ from `want[k - shift]` (silence where that is out of range), and
/// the first few.
fn mismatches(got: &[Vec<f32>], want: &[Vec<f32>], shift: usize) -> (usize, Vec<(usize, usize)>) {
let mut bad = Vec::new();
for (c, (g, w)) in got.iter().zip(want).enumerate() {
for (k, x) in g.iter().enumerate() {
let expect = k.checked_sub(shift).and_then(|j| w.get(j)).copied().unwrap_or(0.0);
if x.to_bits() != expect.to_bits() {
bad.push((c, k));
}
}
}
(bad.len(), bad.into_iter().take(4).collect())
}
/// Runs of two or more exact zeros inside the signal: holes left between packets.
fn holes(got: &[Vec<f32>]) -> usize {
got.iter().map(|c| c.windows(2).filter(|w| w[0] == 0.0 && w[1] == 0.0).count()).sum()
}
#[test]
fn matroska_millisecond_stamps_play_back_to_back() {
for rate in [48_000, 44_100] {
let (asc, units) = aac(rate);
let want = continuous(&asc, rate, &units);
let frames = units.len().saturating_sub(1) * 1024;
for lacing in [LacingMode::None, LacingMode::Xiph] {
let bytes = mkv(&asc, rate, &units, lacing, usize::MAX, 0);
let src = MkvSource::open("tone.mkv", bytes.clone()).unwrap();
let got = played(&src, rate, frames);
let (n, first) = mismatches(&got, &want, 0);
assert_eq!(n, 0, "{rate} Hz, {lacing:?}: {n} samples differ from the continuous decode, first (channel, sample) {first:?}");
// random access (a fresh source, so a cold decoder primed with the packet before) gives
// the same samples
let src = MkvSource::open("tone.mkv", bytes).unwrap();
for start in [30_011usize, 7_000] {
let got = src.audio(start as i64, 4_800, rate).unwrap().channels;
let at: Vec<Vec<f32>> = want.iter().map(|c| c[start..start + 4_800].to_vec()).collect();
assert_eq!(mismatches(&got, &at, 0).0, 0, "{rate} Hz, {lacing:?}: seek to {start}");
}
}
}
}
#[test]
fn obs_remuxed_mp4_plays_back_to_back() {
for rate in [48_000, 44_100] {
let (asc, units) = aac(rate);
let want = continuous(&asc, rate, &units);
let bytes = mp4_remuxed(&asc, rate, &units);
let src = Mp4Source::open("tone.mp4", bytes).unwrap();
let got = played(&src, rate, units.len().saturating_sub(1) * 1024);
let (n, first) = mismatches(&got, &want, 0);
assert_eq!(n, 0, "{rate} Hz: {n} samples differ from the continuous decode, first (channel, sample) {first:?}");
}
}
#[test]
fn a_real_gap_stays_where_the_timestamps_put_it() {
let rate = 48_000;
let (asc, units) = aac(rate);
let want = continuous(&asc, rate, &units);
// 100 ms of the recording missing before unit 20: it plays as silence, and the units after it
// keep their timestamps (to the millisecond)
let src = MkvSource::open("gap.mkv", mkv(&asc, rate, &units, LacingMode::None, 20, 100)).unwrap();
let shift = ((stamp_ms(rate, 20) + 100) * 48 - 20 * 1024) as usize;
assert!((4_700..4_900).contains(&shift), "{shift}");
let got = played(&src, rate, units.len().saturating_sub(1) * 1024 + shift);
let before: Vec<Vec<f32>> = got.iter().map(|c| c[..20 * 1024].to_vec()).collect();
assert_eq!(mismatches(&before, &want, 0).0, 0);
assert!(got.iter().all(|c| c[20 * 1024..20 * 1024 + shift].iter().all(|x| *x == 0.0)), "the gap is silent");
let after: Vec<Vec<f32>> = got.iter().map(|c| c[20 * 1024 + shift..].to_vec()).collect();
let rest: Vec<Vec<f32>> = want.iter().map(|c| c[20 * 1024..].to_vec()).collect();
let (n, first) = mismatches(&after, &rest, 0);
assert_eq!(n, 0, "{n} samples after the gap differ, first {first:?}");
}
#[test]
fn playing_at_another_rate_leaves_no_holes() {
// a 44.1 kHz recording in a 48 kHz sequence (export's default rate)
let (asc, units) = aac(44_100);
let src = MkvSource::open("tone.mkv", mkv(&asc, 44_100, &units, LacingMode::None, usize::MAX, 0)).unwrap();
let frames = (units.len().saturating_sub(2) * 1024) * 48_000 / 44_100;
let got = played(&src, 48_000, frames);
// skip the encoder's priming (the first frame decodes to near-silence)
let tail: Vec<Vec<f32>> = got.iter().map(|c| c[2048..].to_vec()).collect();
assert_eq!(holes(&tail), 0);
}

228
crates/codecs/src/gop.rs
View File

@ -140,6 +140,10 @@ struct State {
bytes: usize,
/// When a frame was last asked of this cache.
last_used: Instant,
/// Only background requests (thumbnails, [`filmcraft_media::cancel::with_background`]) have
/// used the decoder since it was made: it is given up as soon as another background request
/// is done, whatever the caches' recency.
background: bool,
}
impl State {
@ -212,10 +216,27 @@ impl Drop for DecodingGuard {
/// new one when the playhead reaches them, which costs what a seek costs.
const MAX_LIVE_DECODERS: usize = 4;
/// A cache asked for a frame this recently is in use (a layer of the frame being composited,
/// the next clip being prefetched): it keeps its decoder whatever the count.
/// A cache asked for a frame this recently may be needed again soon (the next clip being
/// prefetched, a clip the user scrubs back to): it keeps its decoder up to [`HARD_DECODERS`], and
/// beyond [`MAX_LIVE_DECODERS`] only until it has been idle this long. Its frames are only kept
/// within [`FRAME_BUDGET`] once it has been idle for [`RECENT`]: going back to a clip left (or
/// paused) more than that long ago may cost a re-decode from its keyframe when other clips have
/// filled the budget meanwhile. Frames go least recently used cache first, earliest frames first.
const IDLE: Duration = Duration::from_secs(3);
/// A cache asked for a frame this recently is in use (a layer of the frame being composited): it
/// keeps its decoder and frames whatever the count. Caches idle for less than [`IDLE`] but longer
/// than this are a burst of navigation (thumbnails of every clip, the playhead jumping from clip
/// to clip) that the playhead has already left: they give up decoders beyond [`HARD_DECODERS`]
/// and frames beyond [`FRAME_BUDGET`]. Without this, everything touched within [`IDLE`] counted
/// as in use, so a burst held every clip's decoder (42 live hardware sessions, 6.5 GB of VRAM)
/// and hundreds of MB of frames per clip.
const RECENT: Duration = Duration::from_millis(500);
/// Most decoders caches idle for [`RECENT`] to [`IDLE`] keep together (a composite of a few
/// layers plus the clips just left behind).
const HARD_DECODERS: usize = 8;
/// Decoded frames all caches may hold together before idle ones give theirs up. Each cache also
/// has its own budget, which alone let every clip on a timeline keep hundreds of MB of frames
/// nobody was looking at.
@ -228,6 +249,10 @@ struct Pool {
decoders: AtomicUsize,
max_decoders: usize,
idle: Duration,
/// Caches idle at least this long are no longer in use (see [`RECENT`]).
recent: Duration,
/// Decoders the caches idle for [`Self::recent`] but not [`Self::idle`] may keep.
hard_decoders: usize,
/// Bytes of the frames the caches hold.
bytes: AtomicUsize,
budget: usize,
@ -258,13 +283,29 @@ fn try_state(c: &Shared) -> Option<MutexGuard<'_, State>> {
impl Pool {
fn new(max_decoders: usize, idle: Duration, budget: usize) -> Self {
Self { caches: Mutex::new(Vec::new()), decoders: AtomicUsize::new(0), max_decoders, idle, bytes: AtomicUsize::new(0), budget }
Self {
caches: Mutex::new(Vec::new()),
decoders: AtomicUsize::new(0),
max_decoders,
idle,
recent: idle,
hard_decoders: max_decoders,
bytes: AtomicUsize::new(0),
budget,
}
}
/// Caches idle for `recent` (< the idle time) are still trimmed, down to `hard_decoders`.
fn tiered(mut self, recent: Duration, hard_decoders: usize) -> Self {
self.recent = recent.min(self.idle);
self.hard_decoders = hard_decoders.max(self.max_decoders);
self
}
/// The pool of every cache in the process.
fn global() -> Arc<Pool> {
static POOL: std::sync::OnceLock<Arc<Pool>> = std::sync::OnceLock::new();
POOL.get_or_init(|| Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET))).clone()
POOL.get_or_init(|| Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS))).clone()
}
fn register(&self, c: &Arc<Shared>) {
@ -273,15 +314,15 @@ impl Pool {
g.push(Arc::downgrade(c));
}
/// The idle caches other than `me` that `keep` selects, least recently used first. Other
/// The caches other than `me` idle for at least `idle` that `keep` selects, least recently used first. Other
/// caches are only ever try-locked, so two caches trimming each other cannot deadlock.
fn idle_others(&self, me: &Shared, keep: impl Fn(&State) -> bool) -> Vec<Arc<Shared>> {
fn idle_others(&self, me: &Shared, idle: Duration, keep: impl Fn(&State) -> bool) -> Vec<Arc<Shared>> {
let all: Vec<Arc<Shared>> = self.caches.lock().unwrap_or_else(PoisonError::into_inner).iter().filter_map(Weak::upgrade).collect();
let mut idle: Vec<(Instant, Arc<Shared>)> = all
.into_iter()
.filter(|c| !std::ptr::eq(Arc::as_ptr(c), me))
.filter_map(|c| {
let used = try_state(&c).filter(|st| keep(st) && st.last_used.elapsed() >= self.idle).map(|st| st.last_used)?;
let used = try_state(&c).filter(|st| keep(st) && st.last_used.elapsed() >= idle).map(|st| st.last_used)?;
Some((used, c))
})
.collect();
@ -295,29 +336,53 @@ impl Pool {
#[must_use = "drop the released decoders with no cache lock held"]
fn trim_decoders(&self, me: &Shared) -> Vec<Box<dyn VideoDecoder>> {
let mut released = Vec::new();
if self.decoders.load(Ordering::Relaxed) <= self.max_decoders {
return released;
}
for c in self.idle_others(me, |st| st.decoder.is_some()) {
if self.decoders.load(Ordering::Relaxed) <= self.max_decoders {
break;
// first the caches idle for long, down to the cap; then those a burst has left behind
for (idle, cap) in [(self.idle, self.max_decoders), (self.recent, self.hard_decoders)] {
if self.decoders.load(Ordering::Relaxed) <= cap {
continue;
}
if let Some(mut st) = try_state(&c) {
for c in self.idle_others(me, idle, |st| st.decoder.is_some()) {
if self.decoders.load(Ordering::Relaxed) <= cap {
break;
}
if let Some(mut st) = try_state(&c) {
released.extend(st.release_decoder(self));
}
}
}
released
}
/// Take the decoders of the caches other than `me` that only background requests have used
/// (see [`State::background`]) and that are not decoding now. A burst of thumbnails (an import
/// of many clips) touches every clip within moments, so the recency rules treat them all as
/// in use; each software 4K decoder holds ~0.5 GB. The cache `me` just served keeps its
/// decoder (hovering a clip asks for more of its frames). Drop the result with no cache lock
/// held.
#[must_use = "drop the released decoders with no cache lock held"]
fn release_background(&self, me: &Shared) -> Vec<Box<dyn VideoDecoder>> {
let all: Vec<Arc<Shared>> = self.caches.lock().unwrap_or_else(PoisonError::into_inner).iter().filter_map(Weak::upgrade).collect();
let mut released = Vec::new();
for c in all.iter().filter(|c| !std::ptr::eq(Arc::as_ptr(c), me)) {
if let Some(mut st) = try_state(c)
&& st.background
&& st.decoder.is_some()
{
released.extend(st.release_decoder(self));
}
}
released
}
/// Evict the frames of idle caches, least recently used first, down to the budget. Caches in
/// use keep theirs (each within its own budget): a frame evicted before it is shown costs a
/// Evict the frames of caches idle for [`RECENT`], least recently used first, down to the
/// budget. Caches in use keep theirs (each within its own budget): a frame evicted before it is shown costs a
/// re-decode from the keyframe.
fn trim_frames(&self, me: &Shared) {
let over = || self.bytes.load(Ordering::Relaxed) > self.budget;
if !over() {
return;
}
for c in self.idle_others(me, |st| !st.frames.is_empty()) {
for c in self.idle_others(me, self.recent, |st| !st.frames.is_empty()) {
if !over() {
break;
}
@ -361,7 +426,7 @@ const CONTINUE_THROUGH: usize = 48;
impl GopCache {
pub fn new(explicit_color: Option<ColorInfo>) -> Self {
// unit tests count decoder restarts: each of their caches gets a pool of its own
let pool = if cfg!(test) { Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET)) } else { Pool::global() };
let pool = if cfg!(test) { Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS)) } else { Pool::global() };
Self::in_pool(explicit_color, pool)
}
@ -378,6 +443,7 @@ impl GopCache {
drafts: Default::default(),
bytes: 0,
last_used: Instant::now(),
background: false,
}),
pool: pool.clone(),
});
@ -405,6 +471,7 @@ impl GopCache {
fn store(&self, st: &mut State, pts: i64, f: VideoFrame, draft: bool) {
let f = self.finish(f);
st.last_used = Instant::now();
if draft {
st.drafts.insert(pts);
DRAFT.fetch_add(1, Ordering::Relaxed);
@ -467,6 +534,16 @@ impl GopCache {
/// ([`VideoDecoder::set_draft`]); its draft frames are cached for draft requests only, and an
/// exact request for one re-decodes it.
pub fn frame_late(&self, s: &dyn VideoSamples, target: i64, late_before: Option<i64>) -> crate::Result<Arc<VideoFrame>> {
let r = self.frame_late_locked(s, target, late_before);
if filmcraft_media::cancel::background() {
// Outside the cache's lock (see `release_background`). A thumbnail leaves the decoders
// of earlier thumbnails no reason to stay: each costs hundreds of MB at 4K.
drop(self.shared.pool.release_background(&self.shared));
}
r
}
fn frame_late_locked(&self, s: &dyn VideoSamples, target: i64, late_before: Option<i64>) -> crate::Result<Arc<VideoFrame>> {
let n = s.count();
let i = s.sample_at(target.max(0)).or_else(|| (n > 0).then(|| n - 1)).ok_or_else(|| CodecError::Decode("empty track".into()))?;
let want_pts = s.pts(i);
@ -482,11 +559,23 @@ impl GopCache {
let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner());
let _decoding = DecodingGuard::enter(me);
st.last_used = Instant::now();
if let Some(f) = st.cached(want_pts, draft) {
let background = filmcraft_media::cancel::background();
if let Some(f) = st.cached(want_pts, draft).cloned() {
HITS.fetch_add(1, Ordering::Relaxed);
return Ok(f.clone());
// a monitor asking for a frame of a thumbnail's clip makes its decoder one in use
st.background &= background;
return Ok(f);
}
MISSES.fetch_add(1, Ordering::Relaxed);
if !background {
st.background = false;
} else if st.decoder.is_none() {
st.background = true;
}
if !background && st.decoder.as_ref().is_some_and(|d| d.thread_limited()) {
// made for a thumbnail (few threads): a monitor gets a full one
released.extend(st.release_decoder(&self.shared.pool));
}
if st.decoder.is_none() {
let d = s.make_decoder()?;
st.intra = d.intra_only();
@ -683,6 +772,8 @@ mod tests {
/// (pts, decoded as a draft picture)
held: VecDeque<(i64, bool)>,
draft: bool,
/// made for background work (few threads, like the H.264 decoder's)
limited: bool,
resets: Arc<AtomicUsize>,
decodes: Arc<AtomicUsize>,
}
@ -717,6 +808,9 @@ mod tests {
fn name(&self) -> &str {
"test"
}
fn thread_limited(&self) -> bool {
self.limited
}
fn intra_only(&self) -> bool {
self.intra
}
@ -749,6 +843,7 @@ mod tests {
intra: self.intra,
held: VecDeque::new(),
draft: false,
limited: filmcraft_media::cancel::background(),
resets: self.resets.clone(),
decodes: self.decodes.clone(),
}))
@ -944,6 +1039,99 @@ mod tests {
assert!(s.iter().all(|s| s.resets.load(Ordering::Relaxed) == 1), "no decoder restarted");
}
/// A burst of navigation (every clip of a timeline asked once within moments, as when
/// thumbnails are made or the playhead jumps clip to clip) must not keep every clip's decoder
/// and frames: only the caches of the frame being composited are in use.
#[test]
fn a_burst_of_single_requests_does_not_keep_every_decoder_and_frame() {
// the real decoder caps and idle time; RECENT scaled down from 500 ms to the test's pace
// (3 ms per clip). With the old rule (recent = idle) all 24 decoders stay live.
let pool = Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(Duration::from_millis(2), HARD_DECODERS));
let n = 24;
let s: Vec<Samples> = (0..n).map(|_| samples(300, 250, 3, false)).collect();
let c: Vec<GopCache> = (0..n).map(|_| GopCache::in_pool(None, pool.clone())).collect();
for k in 0..n {
assert_eq!(index_of(&c[k].frame(&s[k], 10_000).expect("frame")), 10);
std::thread::sleep(Duration::from_millis(3));
}
assert!(
pool.decoders.load(Ordering::Relaxed) <= HARD_DECODERS + 1,
"{} live decoders after a burst over {n} clips",
pool.decoders.load(Ordering::Relaxed)
);
// the clip asked last, and a clip in use together with it, keep theirs and carry on
let resets = s[n - 1].resets.load(Ordering::Relaxed);
assert_eq!(index_of(&c[n - 1].frame(&s[n - 1], 11_000).expect("frame")), 11);
assert_eq!(s[n - 1].resets.load(Ordering::Relaxed), resets);
}
/// The thumbnails an import queues: every clip asked once, quickly (the real caps and recency,
/// no sleeping: all of them are "recent"). Each 4K software decoder costs ~0.5 GB, and 42
/// clips used to hold 6+ GB at once.
#[test]
fn background_requests_leave_no_decoders_behind() {
use filmcraft_media::cancel::with_background;
let pool = Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS));
let n = 24;
let s: Vec<Samples> = (0..n).map(|_| samples(300, 250, 3, false)).collect();
let c: Vec<GopCache> = (0..n).map(|_| GopCache::in_pool(None, pool.clone())).collect();
// a clip in use by a monitor
assert_eq!(index_of(&c[0].frame(&s[0], 10_000).expect("frame")), 10);
for k in 1..n {
assert_eq!(index_of(&with_background(true, || c[k].frame(&s[k], 10_000)).expect("frame")), 10);
assert!(pool.decoders.load(Ordering::Relaxed) <= 2, "{} live decoders after thumbnail {k}", pool.decoders.load(Ordering::Relaxed));
}
// the monitor's clip and the last thumbnail keep theirs and carry on without a seek
assert_eq!(pool.decoders.load(Ordering::Relaxed), 2);
assert_eq!(index_of(&c[0].frame(&s[0], 11_000).expect("frame")), 11);
assert_eq!(s[0].resets.load(Ordering::Relaxed), 1);
assert_eq!(index_of(&with_background(true, || c[n - 1].frame(&s[n - 1], 12_000)).expect("frame")), 12);
assert_eq!(s[n - 1].resets.load(Ordering::Relaxed), 1);
// a monitor asking for a frame of a thumbnail's clip makes its decoder one in use
assert_eq!(index_of(&c[n - 1].frame(&s[n - 1], 13_000).expect("frame")), 13);
for k in 1..4 {
with_background(true, || c[k].frame(&s[k], 20_000)).expect("frame");
}
assert_eq!(index_of(&c[n - 1].frame(&s[n - 1], 14_000).expect("frame")), 14);
assert!(c[n - 1].shared.state.lock().unwrap().decoder.is_some(), "the thumbnails left its decoder");
// a cache that lost its decoder still decodes (from the sync sample)
assert_eq!(index_of(&with_background(true, || c[1].frame(&s[1], 30_000)).expect("frame")), 30);
}
#[test]
fn a_monitor_replaces_a_decoder_made_for_a_thumbnail() {
use filmcraft_media::cancel::with_background;
let pool = Arc::new(Pool::new(MAX_LIVE_DECODERS, IDLE, FRAME_BUDGET).tiered(RECENT, HARD_DECODERS));
let (s, c) = (samples(300, 250, 3, false), GopCache::in_pool(None, pool.clone()));
assert_eq!(index_of(&with_background(true, || c.frame(&s, 10_000)).expect("frame")), 10);
// another thumbnail of the same clip carries on with the thumbnail decoder
assert_eq!(index_of(&with_background(true, || c.frame(&s, 12_000)).expect("frame")), 12);
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
// a frame a monitor waits for (not cached: far ahead) is decoded by a full-thread decoder
assert_eq!(index_of(&c.frame(&s, 100_000).expect("frame")), 100);
assert!(!c.shared.state.lock().unwrap().decoder.as_ref().is_some_and(|d| d.thread_limited()));
assert_eq!(pool.decoders.load(Ordering::Relaxed), 1);
// and keeps it for the next frames
let resets = s.resets.load(Ordering::Relaxed);
assert_eq!(index_of(&c.frame(&s, 101_000).expect("frame")), 101);
assert_eq!(s.resets.load(Ordering::Relaxed), resets);
}
#[test]
fn a_burst_of_single_requests_does_not_keep_every_clips_frames() {
// 1×1 RGBA pictures (4 bytes): a shared budget of ten of them
let pool = Arc::new(Pool::new(usize::MAX, Duration::from_secs(3600), 40).tiered(Duration::from_millis(2), usize::MAX));
let n = 12;
let s: Vec<Samples> = (0..n).map(|_| samples(300, 250, 0, false)).collect();
let c: Vec<GopCache> = (0..n).map(|_| GopCache::in_pool(None, pool.clone())).collect();
for k in 0..n {
assert_eq!(index_of(&c[k].frame(&s[k], 10_000).expect("frame")), 10);
std::thread::sleep(Duration::from_millis(5));
}
let held = pool.bytes.load(Ordering::Relaxed);
assert!(held <= 40 + 11 * 4, "{held} bytes held after a burst over {n} clips");
}
#[test]
fn intra_caches_give_up_their_spare_decoders_too() {
let pool = Arc::new(Pool::new(1, Duration::ZERO, FRAME_BUDGET));

2
crates/codecs/src/lib.rs
View File

@ -173,6 +173,8 @@ pub fn open_bytes(name: &str, bytes: Arc<[u8]>) -> std::result::Result<filmcraft
filmcraft_media::open_bytes(name, bytes, &openers())
}
#[cfg(test)]
mod audio_timing_tests;
#[cfg(test)]
mod rounded_pts_tests;
#[cfg(test)]

92
crates/codecs/src/mkv.rs
View File

@ -8,6 +8,9 @@
//! subtracts `CodecDelay` from timestamps, so the pre-skip samples land before zero and are never
//! read; random access decodes `SeekPreRoll` (at least [`crate::audio::OPUS_PRE_ROLL`]) of preceding
//! packets before the target. Packet starts are accumulated from TOC durations (see [`opus_starts`]).
//!
//! AAC, MPEG audio and AC-3 packets each decode to a fixed number of samples, which millisecond
//! timestamps can't express: their packets run on from one another (see [`audio_starts`]).
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
@ -51,35 +54,71 @@ pub struct MkvSource {
audio_preroll: i64,
}
/// Audio packet start positions in source sample frames.
///
/// Block timestamps are quantised to `TimestampScale` (usually 1 ms), so they are only accurate to a
/// tick: a 1024-sample AAC frame at 48 kHz lasts 21.33 ms. Packets that all decode to the same
/// length (AAC, MPEG audio, AC-3) therefore run on from one another, resynchronising to the
/// timestamp only across gaps of more than half a packet (and more than two ticks); see
/// [`crate::audio::contiguous_starts`]. Opus packets: [`opus_starts`]. Other codecs start at their
/// timestamps.
fn audio_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, rate: i64) -> Vec<i64> {
use crate::audio::{FixedFrames, PacketTime, contiguous_starts, fixed_packet_samples};
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
if let Some(h) = opus_head(&t.codec) {
return opus_starts(file, bytes, ti, &h, rate);
}
let (n, d) = tb(t);
let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64;
let first = || file.read_sample(bytes, ti, 0).unwrap_or_default();
let out_rate = u32::try_from(rate).unwrap_or(0);
let frame = match &t.codec {
Codec::Aac { asc } => fixed_packet_samples(FixedFrames::Aac(asc), &[], out_rate),
Codec::Mp3 | Codec::Mp2 => fixed_packet_samples(FixedFrames::MpegAudio, &first(), out_rate),
Codec::Ac3 => fixed_packet_samples(FixedFrames::Ac3, &first(), out_rate),
_ => None,
};
let Some(frame) = frame else {
return t.samples.iter().map(|s| at(s.pts)).collect();
};
let packets = t.samples.iter().map(|s| PacketTime { stamp: at(s.pts), stamped: own_stamp(s), samples: Some(frame) });
contiguous_starts(packets, (frame / 2).max(stamp_tolerance(t, rate)))
}
/// Sample-exact Opus packet start positions (48 kHz frames, pre-skip removed).
///
/// Block timestamps are quantised to `TimestampScale` (usually 1 ms) and the demuxer subtracts a
/// rounded `CodecDelay`, so they are only accurate to a tick. Starts are therefore accumulated from
/// each packet's TOC duration, resynchronising to the timestamp only across real gaps (more than
/// two ticks off). Pre-skip is the exact `CodecDelay` (or the header's pre-skip when it is absent).
/// The demuxer also subtracts a rounded `CodecDelay` from the timestamps. Starts are accumulated
/// from each packet's TOC duration, resynchronising to the timestamp only across real gaps (more
/// than two ticks off). Pre-skip is the exact `CodecDelay` (or the header's pre-skip when it is
/// absent).
fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_opus::OpusHead, rate: i64) -> Vec<i64> {
let t = &file.tracks[ti];
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
let (n, d) = tb(t);
let scale_ns = (n as i128 * 1_000_000_000 / d as i128).max(1);
let delay_ns = t.codec_delay_ns as i128;
// The demuxer's rounding of CodecDelay to ticks (half away from zero), undone here.
let delay_ticks = ((delay_ns + scale_ns / 2) / scale_ns) as i64;
let skip = if t.codec_delay_ns > 0 { (delay_ns * rate as i128 / 1_000_000_000) as i64 } else { head.pre_skip as i64 };
let at = |pts: i64| ((pts + delay_ticks) as i128 * n as i128 * rate as i128 / d as i128) as i64 - skip;
let tolerance = (2 * scale_ns * rate as i128 / 1_000_000_000) as i64 + 1;
let mut starts = Vec::with_capacity(t.samples.len());
let mut next: Option<i64> = None;
for (i, s) in t.samples.iter().enumerate() {
let stamped = at(s.pts);
let start = match next {
Some(p) if (p - stamped).abs() <= tolerance => p,
_ => stamped,
};
starts.push(start);
let dur = file.read_sample(bytes, ti, i).ok().and_then(|p| crate::audio::opus_packet_samples(&p));
next = dur.map(|k| start + k as i64);
}
starts
let at = |pts: i64| ((pts.saturating_add(delay_ticks) as i128 * n as i128 * rate as i128 / d as i128) as i64).saturating_sub(skip);
let packets = t.samples.iter().enumerate().map(|(i, s)| crate::audio::PacketTime {
stamp: at(s.pts),
stamped: own_stamp(s),
samples: file.read_sample(bytes, ti, i).ok().and_then(|p| crate::audio::opus_packet_samples(&p)).map(|k| k as i64),
});
crate::audio::contiguous_starts(packets, stamp_tolerance(t, rate))
}
/// Two timestamp ticks in sample frames at `rate`, plus one: how far a packet's Matroska timestamp
/// can be from its exact start.
fn stamp_tolerance(t: &filmcraft_matroska::Track, rate: i64) -> i64 {
let (n, d) = tb(t);
i64::try_from(2 * n as i128 * rate as i128 / d as i128).unwrap_or(i64::MAX).saturating_add(1)
}
/// Whether a sample's timestamp is its own: frames after the first in a laced block without a
/// duration repeat the block's timestamp.
fn own_stamp(s: &filmcraft_matroska::Sample) -> bool {
s.lace == 0 || s.duration > 0
}
/// The parsed `OpusHead` of an `A_OPUS` track.
@ -311,18 +350,7 @@ impl MkvSource {
start_timecode: None,
file_size: Some(bytes.0.len()),
};
let audio_starts = atrack
.map(|i| {
let t = &file.tracks[i];
let rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64;
let (n, d) = tb(t);
let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64;
match opus_head(&t.codec) {
Some(h) => opus_starts(&file, &bytes, i, &h, rate),
None => t.samples.iter().map(|s| at(s.pts)).collect(),
}
})
.unwrap_or_default();
let audio_starts = atrack.map(|i| audio_starts(&file, &bytes, i, info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64)).unwrap_or_default();
let audio_preroll = atrack
.map(|i| {
let t = &file.tracks[i];

63
crates/codecs/src/mp4.rs
View File

@ -37,11 +37,11 @@ pub struct Mp4Source {
atrack: Option<usize>,
video: GopCache,
audio: Mutex<AudioState>,
/// Cumulative sample start frames for the audio track (for packet lookup).
/// Audio packet start positions in source sample frames (see [`audio_starts`]).
audio_starts: Vec<i64>,
/// Presentation offset of the audio track in its timescale (edit list, or Opus pre-skip).
audio_offset: i64,
/// Decoder pre-roll after a seek, in audio track timescale units (0: prime with one packet).
/// Decoder pre-roll after a seek, in source sample frames (0: prime with one packet).
audio_preroll: i64,
}
@ -49,6 +49,40 @@ pub fn sniff(b: &[u8]) -> bool {
b.len() >= 12 && matches!(&b[4..8], b"ftyp" | b"moov" | b"mdat" | b"wide" | b"free" | b"skip")
}
/// Audio packet start positions in source sample frames at `rate`: the running total of the sample
/// durations, except that packets which all decode to the same length (AAC, MPEG audio, AC-3) run
/// on from one another, resynchronising to the total only across gaps of more than half a packet.
/// A remux from Matroska (OBS's) carries the millisecond rounding of its timestamps into the
/// durations: 1008, 1008, 1056… for 1024-sample AAC frames at 48 kHz (see
/// [`crate::audio::contiguous_starts`]).
fn audio_starts(file: &Mp4File, bytes: &crate::Src, ti: usize, rate: u32) -> Vec<i64> {
use crate::audio::{FixedFrames, PacketTime, contiguous_starts, fixed_packet_samples};
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
let ts = i128::from(t.timescale.max(1));
let mut total = 0i128;
let stamps: Vec<i64> = t
.samples
.iter()
.map(|s| {
let at = total * i128::from(rate) / ts;
total += i128::from(s.duration);
i64::try_from(at).unwrap_or(i64::MAX)
})
.collect();
let first = || file.read_sample(bytes, ti, 0).unwrap_or_default();
let frame = match t.entries.first().map(|e| &e.codec) {
Some(CodecConfig::Aac(a)) => fixed_packet_samples(FixedFrames::Aac(&a.asc), &[], rate),
Some(CodecConfig::Mp3) => fixed_packet_samples(FixedFrames::MpegAudio, &first(), rate),
Some(CodecConfig::Ac3 { .. }) => fixed_packet_samples(FixedFrames::Ac3, &first(), rate),
_ => None,
};
let Some(frame) = frame else { return stamps };
// samples per track timescale unit, rounded up: how coarse the running total is
let unit = i64::try_from((i128::from(rate) + ts - 1) / ts).unwrap_or(i64::MAX);
let packets = stamps.iter().map(|&stamp| PacketTime { stamp, stamped: true, samples: Some(frame) });
contiguous_starts(packets, (frame / 2).max(unit.saturating_mul(2).saturating_add(1)))
}
/// HDR static metadata from the sample entry's `mdcv` / `clli` boxes (0 = unknown).
fn hdr_metadata(md: Option<&filmcraft_isobmff::MasteringDisplay>, cll: Option<(u16, u16)>) -> Option<filmcraft_color::HdrMetadata> {
if md.is_none() && cll.is_none() {
@ -240,20 +274,8 @@ impl Mp4Source {
start_timecode,
file_size: Some(bytes.0.len()),
};
let audio_starts = atrack
.map(|i| {
let t = &file.tracks[i];
let mut acc = 0i64;
t.samples
.iter()
.map(|s| {
let st = acc;
acc += s.duration as i64;
st
})
.collect()
})
.unwrap_or_default();
let audio_rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate);
let audio_starts = atrack.map(|i| audio_starts(&file, &bytes, i, audio_rate)).unwrap_or_default();
let (audio_offset, audio_preroll) = atrack
.map(|i| {
let t = &file.tracks[i];
@ -261,7 +283,8 @@ impl Mp4Source {
match &t.entries[0].codec {
CodecConfig::Opus(o) => {
let off = if t.edit_offset != 0 { t.edit_offset } else { -(o.pre_skip as i64) * ts / 48_000 };
(off, (crate::audio::OPUS_PRE_ROLL as u64 * ts as u64).div_ceil(48_000) as i64)
// (Opus plays at 48 kHz: the pre-roll is in its sample frames)
(off, crate::audio::OPUS_PRE_ROLL as i64)
}
_ => (t.edit_offset, 0),
}
@ -393,14 +416,12 @@ impl MediaSource for Mp4Source {
let offset = i128::from(self.audio_offset) * i128::from(src_rate) / i128::from(track.timescale.max(1));
let s0 = i64::try_from(i128::from(p0) - offset).unwrap_or(if offset > 0 { i64::MIN } else { i64::MAX });
let need = (frames as f64 * ratio).ceil() as i64 + 2;
// Samples-per-unit: the track timescale is usually the sample rate for audio.
let unit = src_rate as f64 / track.timescale.max(1) as f64;
let mut src: Vec<Vec<f32>> = vec![vec![0.0; need.max(0) as usize]; ch];
let mut st = self.audio.lock().unwrap_or_else(|e| e.into_inner());
let first = self.audio_starts.partition_point(|&x| (x as f64 * unit) as i64 <= s0.max(0)).saturating_sub(1);
let first = self.audio_starts.partition_point(|&x| x <= s0.max(0)).saturating_sub(1);
let mut i = first;
while i < track.samples.len() {
let pk_start = (self.audio_starts[i] as f64 * unit) as i64;
let pk_start = self.audio_starts[i];
if pk_start >= s0.saturating_add(need) {
break;
}

41
crates/codecs/src/video.rs
View File

@ -45,8 +45,19 @@ pub trait VideoDecoder: Send {
/// shortcuts that only change pictures nothing references, which come out flagged
/// [`DecodedFrame::draft`]. Decoders without such a mode ignore it.
fn set_draft(&mut self, _on: bool) {}
/// Made with fewer worker threads than a decoder for playback gets, because it was made for
/// background work ([`filmcraft_media::cancel::with_background`]). A request for a monitor
/// replaces such a decoder.
fn thread_limited(&self) -> bool {
false
}
}
/// Worker threads of the H.264 decoders made for background work (thumbnails): a frame-threaded
/// 4K decoder holds `threads + 2` pictures in flight (~0.5 GB with 16 threads), and the frame
/// workers already decode different files in parallel.
const BACKGROUND_H264_THREADS: usize = 2;
/// NAL unit headers of a length-prefixed (avcC / hvcC) sample: the first two bytes of each unit.
fn nal_headers(sample: &[u8], length_size: usize) -> impl Iterator<Item = (u8, u8)> + '_ {
let mut pos = 0usize;
@ -200,19 +211,36 @@ pub struct H264Decoder {
dec: filmcraft_h264::Decoder,
length_size: usize,
draft: bool,
/// Worker threads (a background decoder gets fewer).
threads: usize,
}
impl H264Decoder {
/// The worker threads for a decoder made now.
fn threads_now() -> usize {
if filmcraft_media::cancel::background() { BACKGROUND_H264_THREADS.min(h264_threads()) } else { h264_threads() }
}
fn make_inner(avcc: &[u8], threads: usize) -> Result<filmcraft_h264::Decoder> {
let mut dec = filmcraft_h264::Decoder::with_threads(threads);
if !avcc.is_empty() {
dec.configure_avcc(avcc).map_err(|e| CodecError::Decode(e.to_string()))?;
}
Ok(dec)
}
pub fn new(avcc: Vec<u8>) -> Result<Self> {
recycle_h264_planes();
let dec = filmcraft_h264::Decoder::from_avcc(&avcc).map_err(|e| CodecError::Decode(e.to_string()))?;
let threads = Self::threads_now();
let dec = Self::make_inner(&avcc, threads)?;
let length_size = avcc_length_size(&avcc);
Ok(Self { avcc, dec, length_size, draft: false })
Ok(Self { avcc, dec, length_size, draft: false, threads })
}
/// A decoder for Annex B byte-stream samples (start codes, in-band parameter sets: MXF, TS).
pub fn annexb() -> Self {
recycle_h264_planes();
Self { avcc: Vec::new(), dec: filmcraft_h264::Decoder::new(), length_size: 0, draft: false }
let threads = Self::threads_now();
Self { avcc: Vec::new(), dec: filmcraft_h264::Decoder::with_threads(threads), length_size: 0, draft: false, threads }
}
fn convert(p: filmcraft_h264::Picture) -> DecodedFrame {
use std::sync::Arc;
@ -245,9 +273,7 @@ impl VideoDecoder for H264Decoder {
self.dec.flush().into_iter().map(Self::convert).collect()
}
fn reset(&mut self) {
if self.avcc.is_empty() {
self.dec = filmcraft_h264::Decoder::new();
} else if let Ok(d) = filmcraft_h264::Decoder::from_avcc(&self.avcc) {
if let Ok(d) = Self::make_inner(&self.avcc, self.threads) {
self.dec = d;
}
self.dec.set_draft(self.draft);
@ -255,6 +281,9 @@ impl VideoDecoder for H264Decoder {
fn name(&self) -> &str {
"FilmCraft H.264"
}
fn thread_limited(&self) -> bool {
self.threads < h264_threads()
}
fn set_draft(&mut self, on: bool) {
self.draft = on;
self.dec.set_draft(on);

50
crates/codecs/tests/matroska_audio_oracle.rs Normal file
View File

@ -0,0 +1,50 @@
//! Matroska audio made by ffmpeg, against ffmpeg's decode (#236). ffmpeg stamps blocks to the
//! millisecond, so at 44.1 kHz AAC (1024-sample), MP3 (1152) and AC-3 (1536) packets are each
//! stamped up to half a millisecond off their exact start. Read back block by block, as playback
//! reads it, a tone must stay as smooth as ffmpeg's decode of it: no holes or overlaps between
//! packets.
mod common;
use common::*;
/// Largest |second difference| of `x` from `from` on: a click shows up as a spike.
fn roughness(x: &[f32], from: usize) -> f32 {
x.windows(3).skip(from).map(|w| (w[0] - 2.0 * w[1] + w[2]).abs()).fold(0.0, f32::max)
}
#[test]
fn matroska_audio_plays_without_holes() {
let ff = filmcraft_testkit::require_ffmpeg!();
let cases: [(&str, u32, &[&str]); 5] = [
("tone_aac_48k.mkv", 48_000, &["-c:a", "aac", "-b:a", "160k"]),
("tone_aac_44k.mkv", 44_100, &["-c:a", "aac", "-b:a", "160k"]),
("tone_mp3_44k.mkv", 44_100, &["-c:a", "libmp3lame", "-b:a", "192k"]),
("tone_mp2_44k.mkv", 44_100, &["-c:a", "mp2", "-b:a", "192k"]),
("tone_ac3_44k.mkv", 44_100, &["-c:a", "ac3", "-b:a", "192k"]),
];
for (name, rate, codec) in cases {
let input = format!("sine=frequency=440:sample_rate={rate}:duration=3");
let args: Vec<&str> = ["-f", "lavfi", "-i", input.as_str(), "-af", "volume=4"].iter().copied().chain(codec.iter().copied()).collect();
let Some(path) = fixture(&ff, name, &args) else {
eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
continue;
};
let src = filmcraft_codecs::open_bytes(name, bytes(&path)).unwrap();
assert_eq!(src.info().audio.as_ref().map(|a| a.sample_rate), Some(rate), "{name}");
let frames = rate as usize * 29 / 10;
let mut ours = Vec::with_capacity(frames);
while ours.len() < frames {
let n = 512.min(frames - ours.len());
ours.extend_from_slice(&src.audio(ours.len() as i64, n, rate).unwrap().channels[0]);
}
let theirs = ffmpeg_audio_f32(&ff, &path, &[]);
// past the codecs' start-up (priming, MP3's first frame)
let skip = rate as usize / 10;
let (r_ours, r_theirs) = (roughness(&ours, skip), roughness(&theirs[..theirs.len().min(frames)], skip));
let peak = ours.iter().skip(skip).fold(0f32, |m, x| m.max(x.abs()));
println!("{name}: peak {peak:.3}, max |second difference| ours {r_ours:.5}, ffmpeg {r_theirs:.5}");
assert!(peak > 0.4, "{name}: the tone is missing (peak {peak})");
assert!(r_ours < 2.0 * r_theirs + 0.005, "{name}: clicks between packets (|Δ²| {r_ours} vs ffmpeg's {r_theirs})");
}
}

1
crates/edit/Cargo.toml
View File

@ -16,5 +16,6 @@ thiserror = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
filmcraft-geom = { workspace = true }
filmcraft-media = { workspace = true }
serde_json = { workspace = true }

212
crates/edit/src/frame_size.rs Normal file
View File

@ -0,0 +1,212 @@
//! Sequence Settings ▸ "Scale motion effects proportionally when changing frame size".
//!
//! Motion's Position is in sequence pixels and its Scale is relative to the source, so a new frame
//! size leaves every clip where it was in pixels: smaller and off-centre in a larger frame, cropped
//! in a smaller one. With the option on, the whole picture scales instead, by one factor about the
//! frame centre. On an aspect change (16:9 → 9:16) that factor is the one that fits the old frame
//! inside the new one (`min(W1/W0, H1/H0)`), so nothing is cropped and the old composition sits
//! centred, letterboxed or pillarboxed, rather than filling the new frame.
use filmcraft_project::{ParamValue, Sequence, TrackKind};
/// Motion Scale's upper bound (the parameter's range, in %).
const MAX_SCALE: f64 = 10_000.0;
/// The uniform factor `k` and the old and new frame centres for `from` → `to`, or `None` for an
/// empty frame on either side.
fn fit(from: (u32, u32), to: (u32, u32)) -> Option<(f64, (f64, f64), (f64, f64))> {
if from.0 == 0 || from.1 == 0 || to.0 == 0 || to.1 == 0 {
return None;
}
let (w0, h0, w1, h1) = (f64::from(from.0), f64::from(from.1), f64::from(to.0), f64::from(to.1));
let k = (w1 / w0).min(h1 / h0);
(k.is_finite() && k > 0.0).then_some((k, (w0 / 2.0, h0 / 2.0), (w1 / 2.0, h1 / 2.0)))
}
/// Rescale the Motion of every video clip in `seq` for a frame-size change from `from` to `to`:
/// static values and every keyframe. Returns how many clips were rescaled (0 when the size does not
/// change).
///
/// - Position maps through `p' = c1 + k·(p − c0)`. An automatic (NaN) position stays automatic: it
/// follows the frame centre on its own.
/// - Scale and Scale Width are multiplied by `k`, up to the parameter's 10 000 %. A clip with Scale to
/// Frame Size keeps its Scale, because the renderer refits it to the new frame anyway.
/// - Anchor Point (source pixels) and Rotation do not depend on the frame size and stay as they are.
/// - Only the intrinsic Motion effect changes, as the option's name says: the points of standard
/// effects (a Mirror centre, the Transform effect's Position) keep their pixel values.
pub fn scale_motion(seq: &mut Sequence, from: (u32, u32), to: (u32, u32)) -> usize {
if from == to {
return 0;
}
let Some((k, c0, c1)) = fit(from, to) else { return 0 };
let mut n = 0;
for track in seq.all_tracks_mut().filter(|t| t.kind == TrackKind::Video) {
for item in &mut track.items {
let refits = item.scale_to_frame;
let Some(motion) = item.effect_mut("motion") else { continue };
if let Some(p) = motion.param_mut("position") {
p.map_values(|v| match v {
ParamValue::Vec2(mut v) => {
v.x = c1.0 + k * (v.x - c0.0);
v.y = c1.1 + k * (v.y - c0.1);
ParamValue::Vec2(v)
}
other => other,
});
}
if !refits {
for id in ["scale", "scale_width"] {
if let Some(p) = motion.param_mut(id) {
p.map_values(|v| match v {
ParamValue::Float(s) => ParamValue::Float((s * k).clamp(0.0, MAX_SCALE)),
other => other,
});
}
}
}
n += 1;
}
}
n
}
#[cfg(test)]
mod tests {
use super::*;
use filmcraft_project::{ClipId, ItemId, Keyframe, Label, Project, SequenceSettings, TrackItem, find_effect};
use filmcraft_time::Tick;
fn seq_with(items: Vec<TrackItem>) -> Sequence {
let mut p = Project::new("t");
let s = p.new_sequence("s", SequenceSettings::default(), 2, 1, None);
let mut seq = p.sequence(s).unwrap().clone();
seq.video_tracks[0].items = items;
seq
}
fn clip(id: u64, pos: (f64, f64), scale: f64) -> TrackItem {
let mut motion = find_effect("motion").unwrap().instance();
motion.param_mut("position").unwrap().value = ParamValue::Vec2(filmcraft_geom::Vec2::new(pos.0, pos.1));
motion.param_mut("scale").unwrap().value = ParamValue::Float(scale);
TrackItem {
id: ClipId(id),
item: ItemId(1),
name: format!("c{id}"),
label: Label::Iris,
start: Tick(id as i64 * 1_000_000_000_000),
duration: Tick(1_000_000_000_000),
source_in: Tick::ZERO,
speed: 1.0,
reverse: false,
enabled: true,
link: None,
group: None,
effects: vec![motion],
markers: vec![],
gain_db: 0.0,
frame_hold: None,
scale_to_frame: false,
essential: None,
multicam: None,
time_interpolation: Default::default(),
hold_filters: false,
field_options: None,
source_channels: Vec::new(),
graphic: None,
}
}
fn pos(seq: &Sequence, i: usize) -> (f64, f64) {
match seq.video_tracks[0].items[i].effect("motion").unwrap().param("position").unwrap().value {
ParamValue::Vec2(v) => (v.x, v.y),
_ => panic!("position is a point"),
}
}
fn scale(seq: &Sequence, i: usize) -> f64 {
match seq.video_tracks[0].items[i].effect("motion").unwrap().param("scale").unwrap().value {
ParamValue::Float(s) => s,
_ => panic!("scale is a number"),
}
}
fn close(a: (f64, f64), b: (f64, f64)) -> bool {
(a.0 - b.0).abs() < 1e-9 && (a.1 - b.1).abs() < 1e-9
}
#[test]
fn same_aspect_scales_position_and_scale_together() {
let mut seq = seq_with(vec![clip(1, (960.0, 540.0), 100.0), clip(2, (300.0, 90.0), 50.0)]);
assert_eq!(scale_motion(&mut seq, (1920, 1080), (1280, 720)), 2);
assert!(close(pos(&seq, 0), (640.0, 360.0)), "{:?}", pos(&seq, 0));
assert!(close(pos(&seq, 1), (200.0, 60.0)), "{:?}", pos(&seq, 1));
assert!((scale(&seq, 0) - 100.0 * 2.0 / 3.0).abs() < 1e-9);
assert!((scale(&seq, 1) - 50.0 * 2.0 / 3.0).abs() < 1e-9);
}
#[test]
fn aspect_change_fits_the_old_frame_inside_the_new_one() {
// 1920x1080 -> 1080x1920: k = min(1080/1920, 1920/1080) = 0.5625, centred
let mut seq = seq_with(vec![clip(1, (960.0, 540.0), 100.0), clip(2, (0.0, 0.0), 100.0)]);
scale_motion(&mut seq, (1920, 1080), (1080, 1920));
assert!(close(pos(&seq, 0), (540.0, 960.0)), "{:?}", pos(&seq, 0));
// the old top-left corner lands on the new frame's left edge, letterboxed vertically
assert!(close(pos(&seq, 1), (0.0, 960.0 - 0.5625 * 540.0)), "{:?}", pos(&seq, 1));
assert!((scale(&seq, 0) - 56.25).abs() < 1e-9);
}
#[test]
fn keyframes_scale_too() {
let mut c = clip(1, (960.0, 540.0), 100.0);
let m = c.effect_mut("motion").unwrap();
m.param_mut("position").unwrap().keyframes = vec![
Keyframe::new(Tick::ZERO, ParamValue::Vec2(filmcraft_geom::Vec2::new(0.0, 0.0))),
Keyframe::new(Tick(10), ParamValue::Vec2(filmcraft_geom::Vec2::new(1920.0, 1080.0))),
];
m.param_mut("scale").unwrap().keyframes = vec![Keyframe::new(Tick::ZERO, ParamValue::Float(100.0)), Keyframe::new(Tick(10), ParamValue::Float(200.0))];
let mut seq = seq_with(vec![c]);
scale_motion(&mut seq, (1920, 1080), (3840, 2160));
let m = seq.video_tracks[0].items[0].effect("motion").unwrap();
let kp: Vec<_> = m.param("position").unwrap().keyframes.iter().map(|k| k.value.clone()).collect();
assert_eq!(kp, vec![ParamValue::Vec2(filmcraft_geom::Vec2::new(0.0, 0.0)), ParamValue::Vec2(filmcraft_geom::Vec2::new(3840.0, 2160.0))]);
let ks: Vec<_> = m.param("scale").unwrap().keyframes.iter().map(|k| k.value.clone()).collect();
assert_eq!(ks, vec![ParamValue::Float(200.0), ParamValue::Float(400.0)]);
}
#[test]
fn automatic_position_stays_automatic() {
let mut seq = seq_with(vec![clip(1, (f64::NAN, f64::NAN), 100.0)]);
scale_motion(&mut seq, (1920, 1080), (1280, 720));
let (x, y) = pos(&seq, 0);
assert!(x.is_nan() && y.is_nan());
assert!((scale(&seq, 0) - 100.0 * 2.0 / 3.0).abs() < 1e-9);
}
#[test]
fn scale_to_frame_clips_keep_their_scale_but_move() {
let mut c = clip(1, (480.0, 270.0), 100.0);
c.scale_to_frame = true;
let mut seq = seq_with(vec![c]);
scale_motion(&mut seq, (1920, 1080), (3840, 2160));
assert!(close(pos(&seq, 0), (960.0, 540.0)), "{:?}", pos(&seq, 0));
assert_eq!(scale(&seq, 0), 100.0);
}
#[test]
fn scale_is_capped_at_the_parameter_range() {
let mut seq = seq_with(vec![clip(1, (1.0, 1.0), 5000.0)]);
scale_motion(&mut seq, (2, 2), (8, 8));
assert_eq!(scale(&seq, 0), MAX_SCALE);
}
#[test]
fn same_size_or_empty_frame_changes_nothing() {
let mut seq = seq_with(vec![clip(1, (100.0, 100.0), 100.0)]);
let before = seq.clone();
assert_eq!(scale_motion(&mut seq, (1920, 1080), (1920, 1080)), 0);
assert_eq!(scale_motion(&mut seq, (0, 1080), (1280, 720)), 0);
assert_eq!(scale_motion(&mut seq, (1920, 1080), (1280, 0)), 0);
// Debug, not ==: the automatic anchor is NaN, which never equals itself
assert_eq!(format!("{:?}", seq.video_tracks[0].items[0].effects), format!("{:?}", before.video_tracks[0].items[0].effects));
}
}

1
crates/edit/src/lib.rs
View File

@ -18,6 +18,7 @@
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
pub mod captions;
pub mod frame_size;
pub mod multicam;
pub mod through;
pub mod transcript;

18
crates/engine/src/autosave.rs
View File

@ -22,12 +22,13 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel};
use std::thread::JoinHandle;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use std::time::Duration;
use filmcraft_format::autosave as names;
use filmcraft_project::Project;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use web_time::{Instant, SystemTime, UNIX_EPOCH};
use crate::settings;
@ -470,7 +471,17 @@ impl Preferences {
}
}
/// The per-user data directory (`FILMCRAFT_DATA_DIR` overrides):
/// A file with this name next to the executable makes the install portable: its data directory is
/// `<exe dir>/data` instead of the per-user one (the Windows portable zip ships it).
pub const PORTABLE_MARKER: &str = "portable.txt";
/// `<exe_dir>/data` when `exe_dir` holds [`PORTABLE_MARKER`], else `None`.
pub fn portable_data_dir(exe_dir: &Path) -> Option<PathBuf> {
exe_dir.join(PORTABLE_MARKER).is_file().then(|| exe_dir.join("data"))
}
/// The data directory. `FILMCRAFT_DATA_DIR` overrides; a portable install ([`PORTABLE_MARKER`] next
/// to the executable) uses `<exe dir>/data`; otherwise the per-user directory:
/// macOS `~/Library/Application Support/FilmCraft`, Windows `%APPDATA%\FilmCraft`,
/// elsewhere `$XDG_DATA_HOME/filmcraft` or `~/.local/share/filmcraft`.
pub fn default_data_dir() -> Option<PathBuf> {
@ -478,6 +489,9 @@ pub fn default_data_dir() -> Option<PathBuf> {
if let Some(d) = env("FILMCRAFT_DATA_DIR") {
return Some(d);
}
if let Some(d) = std::env::current_exe().ok().and_then(|exe| exe.parent().and_then(portable_data_dir)) {
return Some(d);
}
if cfg!(target_os = "macos") {
return env("HOME").map(|h| h.join("Library/Application Support/FilmCraft"));
}

13
crates/engine/src/autosave_tests.rs
View File

@ -176,6 +176,19 @@ fn auto_save_ring_rotates_and_can_save_the_project() {
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn portable_marker_keeps_data_next_to_the_executable() {
// #139: a portable install must not write to %APPDATA%.
let d = temp_dir("portable");
assert_eq!(autosave::portable_data_dir(&d), None);
std::fs::create_dir_all(d.join(autosave::PORTABLE_MARKER)).unwrap();
assert_eq!(autosave::portable_data_dir(&d), None, "a directory is not the marker");
std::fs::remove_dir_all(d.join(autosave::PORTABLE_MARKER)).unwrap();
std::fs::write(d.join(autosave::PORTABLE_MARKER), "").unwrap();
assert_eq!(autosave::portable_data_dir(&d), Some(d.join("data")));
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn journal_costs_the_ui_thread_only_a_handoff() {
// Per change the UI thread pays an Arc clone and a channel send; encoding happens on the worker.

69
crates/engine/src/commands.rs
View File

@ -131,9 +131,16 @@ pub(crate) fn named_clips(s: &Session, spec: &CommandSpec, p: &Value) -> Option<
let clips: Vec<ClipId> = named_ids(spec, p, "clips", "clip")?.into_iter().map(ClipId).filter(|c| seq.find_item(*c).is_some()).collect();
(!clips.is_empty()).then_some(clips)
}
/// Project items named explicitly in `items` / `item`, likewise.
/// Project items named explicitly in `items` / `item`, likewise. The Project panel's selection
/// holds bins beside items (one id space, see `project.select`), so a bin named here counts too;
/// the project's own top bin does not, since no command acts on it (#244).
pub(crate) fn named_items(s: &Session, spec: &CommandSpec, p: &Value) -> Option<Vec<ItemId>> {
let items: Vec<ItemId> = named_ids(spec, p, "items", "item")?.into_iter().map(ItemId).filter(|i| s.project.item(*i).is_some()).collect();
let root = s.project.root.id;
let exists = |i: &ItemId| {
let b = filmcraft_project::BinId(i.0);
s.project.item(*i).is_some() || (b != root && s.project.root.find_bin(b).is_some())
};
let items: Vec<ItemId> = named_ids(spec, p, "items", "item")?.into_iter().map(ItemId).filter(exists).collect();
(!items.is_empty()).then_some(items)
}
@ -1071,7 +1078,7 @@ fn build() -> Vec<CommandSpec> {
"Media…",
["File", "Export"],
None,
r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass"?,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#,
r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass"?,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#,
has_seq,
crate::export_tools::export_media
),
@ -1513,41 +1520,70 @@ fn build() -> Vec<CommandSpec> {
"Sequence Settings…",
["Sequence"],
None,
r#"{"width":u32?,"height":u32?,"fps":f64?,"name":str?,"sampleRate":u32?,"mix":"Stereo|Mono|5.1|Adaptive"?}"#,
r#"{"width":u32?,"height":u32?,"fps":f64?,"name":str?,"sampleRate":u32?,"mix":"Stereo|Mono|5.1|Adaptive"?,"dropFrame":bool?,"maxRenderQuality":bool?,"workingSpace":"rec709"|"rec2100-pq"|"rec2100-hlg"?,"wideGamut":bool?,"autoToneMap":bool?,"scaleMotion":bool=false}"#,
has_seq,
|s, p| {
const CMD: &str = "sequence.settings";
let id = s.state.active_sequence.ok_or(EngineError::NoSequence)?;
let fps = fps_p(p, "sequence.settings")?;
let fps = fps_p(p, CMD)?;
let mix = match str_p(p, "mix") {
Some(m) => Some(
crate::mixer::channels_from(m).ok_or_else(|| bad("sequence.settings", format!("unknown mix `{m}` (Stereo, Mono, 5.1, Adaptive)")))?,
),
Some(m) => Some(crate::mixer::channels_from(m).ok_or_else(|| bad(CMD, format!("unknown mix `{m}` (Stereo, Mono, 5.1, Adaptive)")))?),
None => None,
};
let working = match str_p(p, "workingSpace") {
Some(w) => Some(filmcraft_color::WorkingSpace::parse(w).ok_or_else(|| bad(CMD, format!("unknown working space `{w}`")))?),
None => None,
};
let drop_frame = bool_p(p, "dropFrame");
let rate = fps.or_else(|| s.active_sequence().map(|q| q.settings.frame_rate)).unwrap_or(FrameRate::FPS_23_976);
if drop_frame == Some(true) && !rate.supports_drop_frame() {
return Err(bad(CMD, format!("drop-frame timecode needs a 29.97, 59.94 or 119.88 fps timebase, not {rate}")));
}
let scale_motion = bool_p(p, "scaleMotion").unwrap_or(false);
let p = p.clone();
s.edit("Sequence Settings", |pr, _| {
let scaled = s.edit("Sequence Settings", |pr, _| {
if let Some(n) = str_p(&p, "name") {
pr.item_mut(id).ok_or(EngineError::NoSequence)?.name = n.to_string();
}
let q = pr.sequence_mut(id).ok_or(EngineError::NoSequence)?;
if let Some(w) = checked_u32_p(&p, "width", "sequence.settings")? {
let old_size = (q.settings.width, q.settings.height);
if let Some(w) = checked_u32_p(&p, "width", CMD)? {
q.settings.width = w;
}
if let Some(h) = checked_u32_p(&p, "height", "sequence.settings")? {
if let Some(h) = checked_u32_p(&p, "height", CMD)? {
q.settings.height = h;
}
if let Some(f) = fps {
q.settings.frame_rate = f;
}
if let Some(sr) = checked_u32_p(&p, "sampleRate", "sequence.settings")? {
if let Some(sr) = checked_u32_p(&p, "sampleRate", CMD)? {
q.settings.sample_rate = sr;
}
if let Some(m) = mix {
q.settings.audio_master = m;
}
q.settings.validate().map_err(|e| bad("sequence.settings", e))
if let Some(df) = drop_frame {
q.settings.drop_frame = df;
}
if let Some(v) = bool_p(&p, "maxRenderQuality") {
q.settings.max_render_quality = v;
}
let c = &mut q.settings.color;
if let Some(w) = working {
c.working = w;
}
if let Some(v) = bool_p(&p, "wideGamut") {
c.wide_gamut = v;
}
if let Some(v) = bool_p(&p, "autoToneMap") {
c.auto_tone_map = v;
}
q.settings.working_space = q.settings.color.working.label().into();
q.settings.validate().map_err(|e| bad(CMD, e))?;
let new_size = (q.settings.width, q.settings.height);
Ok(if scale_motion { edit::frame_size::scale_motion(q, old_size, new_size) } else { 0 })
})?;
Ok(Value::Null)
Ok(json!({"scaledClips": scaled}))
}
),
cmd!("sequence.renderEffectsInToOut", "Render Effects In to Out", ["Sequence"], Some("Enter"), r#"{"wait":bool=false}"#, has_seq, |s, p| {
@ -1609,14 +1645,15 @@ fn build() -> Vec<CommandSpec> {
})?;
Ok(Value::Null)
}),
// With no In / Out mark, like Premiere: the start / end of the sequence (#208).
cmd!("markers.goToIn", "Go to In", ["Markers"], Some("Shift+I"), "{}", has_seq, |s, _| {
if let Some(i) = s.active_sequence().and_then(|q| q.mark_in) {
if let Some(i) = s.active_sequence().map(|q| q.mark_in.unwrap_or(Tick::ZERO)) {
s.set_playhead(i);
}
Ok(Value::Null)
}),
cmd!("markers.goToOut", "Go to Out", ["Markers"], Some("Shift+O"), "{}", has_seq, |s, _| {
if let Some(o) = s.active_sequence().and_then(|q| q.mark_out) {
if let Some(o) = s.active_sequence().map(|q| q.mark_out.unwrap_or_else(|| q.duration())) {
s.set_playhead(o);
}
Ok(Value::Null)

17
crates/engine/src/essential_sound.rs
View File

@ -12,7 +12,7 @@
//! | `essentialSound.setType` / `clearType` | assign / clear the audio type (clearing removes its effects and gain) |
//! | `essentialSound.set` | one setting by dotted key (`repair.noise.amount`), or several (`values`) |
//! | `essentialSound.applyPreset` / `savePreset` / `deletePreset` | built-in and user presets |
//! | `essentialSound.autoMatch` | measure integrated loudness (BS.1770) and set the match gain to the target |
//! | `essentialSound.autoMatch` | measure integrated loudness (BS.1770, mono clips as one channel) and set the match gain to the target |
//! | `essentialSound.generateDucking` | Volume keyframes that duck Music / Ambience under the trigger types |
use serde_json::{Value, json};
@ -262,15 +262,26 @@ fn delete_preset(s: &mut Session, p: &Value) -> Result<Value> {
/// Integrated loudness (LUFS) of a clip's own signal (clip gain + clip effects) over its whole
/// timeline range, or `None` when it is silent / has no audio.
///
/// The clip is measured with the channels it has: a stereo (or wider) source as the two channels
/// it plays, a mono source or a mono pick as one channel (its two identical sides folded to one),
/// the way BS.1770 and ffmpeg's `ebur128` measure a mono programme. Metering the dual-mono pair the
/// clip plays would read 3 LU louder than the file and match it 3 dB too quietly (#296).
pub fn clip_loudness(item: &TrackItem, sr: u32, sources: &dyn SourceProvider) -> Option<f64> {
let a0 = item.start.to_units_floor(sr as i64);
let a1 = item.end().to_units_floor(sr as i64);
let mut meter = LoudnessMeter::new(sr as f64, 2);
let mono = filmcraft_render::audio::clip_is_mono(item, sources)?;
let mut meter = LoudnessMeter::new(sr as f64, if mono { 1 } else { 2 });
let mut pos = a0;
while pos < a1 {
let n = (a1 - pos).min(sr as i64) as usize;
let [l, r] = filmcraft_render::audio::clip_signal(item, pos, n, sr, sources)?;
meter.process(&[&l, &r]);
if mono {
let m: Vec<f32> = l.iter().zip(&r).map(|(a, b)| 0.5 * (a + b)).collect();
meter.process(&[&m]);
} else {
meter.process(&[&l, &r]);
}
pos += n as i64;
}
Some(meter.integrated()).filter(|v| v.is_finite())

35
crates/engine/src/essential_sound_tests.rs
View File

@ -265,6 +265,41 @@ fn auto_match_hits_the_target_loudness() {
}
}
/// #296: a mono file measured ~3 LU too loud. The same speech imported as a 1-channel WAV and as
/// a dual-mono stereo WAV must measure 3.01 LU apart (the stereo pair carries twice the energy),
/// the mono clip must agree with a one-channel BS.1770 meter over the file (what ffmpeg's
/// `ebur128` reports), and a mono pick (Modify ▸ Audio Channels) of the stereo file counts as mono.
#[test]
fn auto_match_measures_a_mono_clip_as_one_channel() {
let sig = speech(12.0, 0.3, |t| (t % 3.0) < 2.2);
let (mut s, c) = session_with(&[(sig.clone(), 0.0)]);
let stereo = c[0];
let mono_wav: Arc<[u8]> = filmcraft_media::wav::write_wav16(&sig, 1, SR).into();
let mono_item = crate::commands::import_bytes(&mut s, "/mono.wav", mono_wav, None).unwrap();
let r = s.execute("timeline.place", json!({"item": mono_item.0, "audioTrack": "A2", "seconds": 0.0})).unwrap();
let mono = ClipId(r["clips"][0].as_u64().unwrap());
for c in [stereo, mono] {
s.execute("essentialSound.setType", json!({"clips": [c.0], "type": "dialogue"})).unwrap();
}
let measured = |s: &mut Session, c: ClipId| {
let r = s.execute("essentialSound.autoMatch", json!({"clips": [c.0], "target": -23.0})).unwrap();
r["clips"][0]["measuredLufs"].as_f64().unwrap()
};
let (two, one) = (measured(&mut s, stereo), measured(&mut s, mono));
let mut file = LoudnessMeter::new(SR as f64, 1);
file.process(&[&sig]);
let want = file.integrated();
println!("stereo (dual mono) {two:.3} LUFS, mono {one:.3} LUFS, mono file {want:.3} LUFS");
assert!((one - want).abs() <= 0.1, "mono clip {one} LUFS, the file reads {want} LUFS");
assert!((two - one - 3.0103).abs() <= 0.1, "dual mono {two} LUFS should read 3.01 LU above mono {one} LUFS");
let st = item(&s, mono).essential.unwrap();
assert!((st.loudness.gain_db - (-23.0 - want)).abs() <= 0.1, "match gain {} dB for {want} LUFS", st.loudness.gain_db);
// one picked channel of the stereo file plays on both sides: measured as mono too
s.execute("clip.audioChannels", json!({"clips": [stereo.0], "channels": [0]})).unwrap();
let picked = measured(&mut s, stereo);
assert!((picked - want).abs() <= 0.1, "mono pick {picked} LUFS, the file reads {want} LUFS");
}
#[test]
fn ducking_generates_volume_keyframes_under_dialogue() {
// dialogue in [2, 4] and [6, 7.5] s; music for 10 s

36
crates/engine/src/explicit_targets_tests.rs
View File

@ -175,3 +175,39 @@ fn naming_targets_lifts_only_the_selection_condition() {
let _ = s.execute("edit.clear", json!({"clips": [a.id.0]}));
assert!(clip(&s, a.id).is_some(), "a clip on a locked track is not removed");
}
/// `project.delete` with explicit ids needs no Project-panel selection, for bins as much as for
/// items: the selection can hold both, so a named bin stands in for it (#244).
#[test]
fn project_delete_takes_explicit_bins_and_items_without_a_selection() {
let mut s = demo();
let before = s.project.items.len();
let empty = s.execute("file.newBin", json!({"name": "Empty"})).unwrap()["bin"].as_u64().unwrap();
let outer = s.execute("file.newBin", json!({"name": "Old footage"})).unwrap()["bin"].as_u64().unwrap();
let ocean = item_named(&s, "Ocean_Sunset.mp4");
s.execute("project.moveToBin", json!({"items": [ocean.0], "bin": outer})).unwrap();
s.state.project_selection.clear();
assert!(!s.is_enabled("project.delete"));
assert_eq!(disabled(s.execute("project.delete", json!({}))), "select an item in the Project panel");
// an empty bin, by its id alone
let r = s.execute("project.delete", json!({"items": [empty]})).unwrap();
assert_eq!(r, json!({"items": 0, "bins": 1}));
assert!(s.project.root.find_bin(filmcraft_project::BinId(empty)).is_none());
// a bin with a clip in it, and an item, each by id alone
let r = s.execute("project.delete", json!({"items": [outer]})).unwrap();
assert_eq!(r, json!({"items": 1, "bins": 1}));
assert!(s.project.item(ocean).is_none());
let dunes = item_named(&s, "Desert_Dunes.mp4");
let r = s.execute("project.delete", json!({"items": [dunes.0]})).unwrap();
assert_eq!(r, json!({"items": 1, "bins": 0}));
assert_eq!(s.project.items.len(), before - 2);
assert!(s.state.project_selection.is_empty(), "the selection is left as it was");
assert!(!s.is_enabled("project.delete"), "menu enablement still follows the selection");
// ids of nothing, or of the project's own top bin, do not stand in for a selection
let root = s.project.root.id.0;
assert_eq!(disabled(s.execute("project.delete", json!({"items": [root]}))), "select an item in the Project panel");
assert_eq!(disabled(s.execute("project.delete", json!({"items": [999_999]}))), "select an item in the Project panel");
s.execute("edit.undo", json!({})).unwrap();
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(s.project.items.len(), before);
}

17
crates/engine/src/export_tests.rs
View File

@ -484,6 +484,23 @@ fn queue_exports_several_sequences_and_ranges() {
assert_eq!(len("parts_2.wav"), 44 + 12_000 * 4, "0.25 s of 16-bit stereo");
}
#[test]
fn queue_folder_paths_expand_home_and_take_either_separator() {
let mut s = demo();
let first = s.state.active_sequence.unwrap();
let second = s.execute("file.newSequence", json!({"name": "Second Cut", "width": 640, "height": 360, "fps": 25})).unwrap()["sequence"].as_u64().unwrap();
let preset = "Waveform Audio 48 kHz 16-bit";
// a folder that does not exist yet, named by a trailing `\`: each sequence keeps its own name
s.execute("export.queue.add", json!({"preset": preset, "path": "not-yet-made\\", "sequences": [first.0, second]})).unwrap();
let paths: Vec<String> = queue(&mut s).iter().map(|i| i["path"].as_str().unwrap().to_string()).collect();
assert!(paths.iter().any(|p| p.ends_with("Second Cut.wav")), "{paths:?}");
if let Some(home) = crate::media_browser::std_home_dir() {
s.execute("export.queue.add", json!({"preset": preset, "path": "~/Exports/", "sequence": first.0})).unwrap();
let last = queue(&mut s).last().unwrap()["path"].as_str().unwrap().to_string();
assert!(last.starts_with(&home), "{last}");
}
}
#[test]
fn quick_export_uses_and_remembers_a_preset() {
let mut s = demo();

29
crates/engine/src/export_tools.rs
View File

@ -230,6 +230,9 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
if let Some(v) = bool_p(p, "sdr") {
settings.sdr = v;
}
if let Some(v) = bool_p(p, "alpha") {
settings.alpha = v;
}
match (crate::commands::checked_u32_p(p, "width", cmd)?, crate::commands::checked_u32_p(p, "height", cmd)?) {
(Some(w), Some(h)) => settings.frame_size = Some((w, h)),
(None, None) => {}
@ -304,7 +307,7 @@ pub fn default_export_dir(s: &Session) -> PathBuf {
// the web build's virtual file table: written files are offered as downloads
return PathBuf::from("/exports");
}
if let Ok(h) = std::env::var("HOME").or_else(|_| std::env::var("USERPROFILE")) {
if let Some(h) = crate::media_browser::std_home_dir() {
let movies = Path::new(&h).join("Movies");
return if movies.is_dir() { movies } else { PathBuf::from(h) };
}
@ -327,11 +330,19 @@ fn with_extension(path: &str, settings: &ExportSettings) -> String {
}
}
fn expand_home(p: &str) -> String {
match (p.strip_prefix("~/"), std::env::var("HOME")) {
(Some(rest), Ok(h)) => format!("{h}/{rest}"),
/// `~/` expanded to the home directory (`USERPROFILE` on Windows). On Windows `/` becomes `\`:
/// after a `\\?\` prefix (what `canonicalize` returns) `/` is not a separator.
pub fn expand_home(p: &str) -> String {
let p = match (p.strip_prefix("~/"), crate::media_browser::std_home_dir()) {
(Some(rest), Some(h)) => format!("{h}/{rest}"),
_ => p.to_string(),
}
};
if cfg!(windows) { p.replace('/', "\\") } else { p }
}
/// `x` names a folder: it ends in a separator or already exists as one.
fn is_folder(x: &str) -> bool {
x.ends_with(['/', '\\']) || Path::new(x).is_dir()
}
/// Output path for `seq`: the `path` param (a file, or a directory ending in `/` or existing), else
@ -339,7 +350,7 @@ fn expand_home(p: &str) -> String {
fn output_path(s: &Session, p: &Value, seq: ItemId, settings: &ExportSettings, suffix: Option<usize>) -> String {
let name = s.project.item(seq).map(|i| i.name.clone()).unwrap_or_else(|| "Sequence".into());
let base = match str_p(p, "path").map(expand_home) {
Some(x) if x.ends_with('/') || x.ends_with('\\') || Path::new(&x).is_dir() => Path::new(&x).join(file_safe(&name)).to_string_lossy().to_string(),
Some(x) if is_folder(&x) => Path::new(&x).join(file_safe(&name)).to_string_lossy().to_string(),
Some(x) => x,
None => default_export_dir(s).join(file_safe(&name)).to_string_lossy().to_string(),
};
@ -633,11 +644,7 @@ fn queue_add(s: &mut Session, p: &Value) -> Result<Value> {
};
st.range = range_param(s, &project, seq, *r, times, cmd)?;
// several ranges of one sequence get numbered names; several sequences their own names
let suffix = if many && (ranges.len() > 1 || str_p(p, "path").is_some_and(|x| !x.ends_with('/') && !Path::new(&expand_home(x)).is_dir())) {
Some(n)
} else {
None
};
let suffix = if many && (ranges.len() > 1 || str_p(p, "path").is_some_and(|x| !is_folder(&expand_home(x)))) { Some(n) } else { None };
st.path = output_path(s, p, seq, &st, suffix);
n += 1;
items.push((seq, st));

7
crates/engine/src/graphics.rs
View File

@ -98,7 +98,12 @@ pub(crate) fn graphic_source(p: &mut filmcraft_project::Project, w: u32, h: u32,
/// topmost clip at the playhead that is free for the duration (a track is added if needed).
fn new_graphic_clip(s: &mut Session, layer: filmcraft_project::EffectInstance, name: &str, p: &Value) -> Result<ClipId> {
let seconds = f64_p(p, "seconds").unwrap_or(5.0).max(0.01);
place_video_clip(s, name, p, "New Graphic", vec![layer], move |pr, (w, h, rate)| {
// Here `seconds` is duration, not the generic time parser's placement alias.
let mut placement = p.clone();
if let Some(params) = placement.as_object_mut() {
params.remove("seconds");
}
place_video_clip(s, name, &placement, "New Graphic", vec![layer], move |pr, (w, h, rate)| {
let src = graphic_source(pr, w, h, rate);
(src, rate.snap_nearest(Tick::from_seconds_f64(seconds)).max(rate.frame_duration()))
})

41
crates/engine/src/graphics_tests.rs
View File

@ -24,6 +24,47 @@ fn text_of(l: &filmcraft_project::graphic::LayerSpec) -> String {
}
}
#[test]
fn graphic_duration_does_not_set_its_timeline_position() {
for command in ["graphics.newText", "graphics.newShape"] {
for playhead_frame in [0, 72] {
for seconds in [1, 2] {
let mut s = Session::default();
s.execute("file.newSequence", json!({"name": "Placement", "fps": 24, "width": 128, "height": 128, "video": 1, "audio": 1})).unwrap();
s.execute("playhead.set", json!({"frame": playhead_frame})).unwrap();
let start = s.playhead();
let r = s.execute(command, json!({"seconds": seconds})).unwrap();
let clip = ClipId(r["clip"].as_u64().unwrap());
let q = s.active_sequence().unwrap();
let (_, it) = q.find_item(clip).unwrap();
assert_eq!(it.start, start, "{command}, duration {seconds}, playhead {playhead_frame}");
assert_eq!(it.duration, Tick::from_seconds_f64(f64::from(seconds)));
s.execute("edit.undo", json!({})).unwrap();
assert!(s.active_sequence().unwrap().find_item(clip).is_none());
s.execute("edit.redo", json!({})).unwrap();
assert_eq!(s.active_sequence().unwrap().find_item(clip).unwrap().1.start, start);
}
}
}
}
#[test]
fn graphic_duration_keeps_explicit_time_frame_and_timecode_placement() {
for command in ["graphics.newText", "graphics.newShape"] {
for mut params in [json!({"time": 0}), json!({"frame": 0}), json!({"timecode": "00:00:00:00"})] {
let mut s = Session::default();
s.execute("file.newSequence", json!({"name": "Placement", "fps": 24, "width": 128, "height": 128, "video": 1, "audio": 1})).unwrap();
s.execute("playhead.set", json!({"frame": 72})).unwrap();
params["seconds"] = json!(2);
let r = s.execute(command, params).unwrap();
let clip = ClipId(r["clip"].as_u64().unwrap());
let (_, it) = s.active_sequence().unwrap().find_item(clip).unwrap();
assert_eq!(it.start, Tick::ZERO, "{command}");
assert_eq!(it.duration, Tick::from_seconds_f64(2.0));
}
}
}
#[test]
fn new_text_makes_a_graphic_clip_above_the_footage() {
let mut s = demo();

2
crates/engine/src/keyboard.rs
View File

@ -559,7 +559,7 @@ fn export_frame(s: &mut Session, p: &Value) -> Result<Value> {
};
let img = s.render_program(1.0).ok_or(EngineError::NoSequence)?;
let (w, h) = (img.w as u32, img.h as u32);
let bytes = filmcraft_export::encode_still(format, img.to_rgba8(), w, h).map_err(|e| EngineError::Other(e.to_string()))?;
let bytes = filmcraft_export::encode_still(format, img.to_rgba8(), w, h, false).map_err(|e| EngineError::Other(e.to_string()))?;
s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
let mut out = json!({"path": path, "width": w, "height": h});
if p.get("import").and_then(Value::as_bool).unwrap_or(false) {

10
crates/engine/src/lib.rs
View File

@ -553,10 +553,10 @@ impl Session {
let Some(p) = self.persistence.as_mut() else { return };
for ev in p.drain_events() {
match ev {
autosave::WorkerEvent::SavedProject { path, revision } => {
if self.path.as_deref() == Some(path.as_str()) && revision > self.saved_revision && revision <= self.revision {
self.saved_revision = revision;
}
autosave::WorkerEvent::SavedProject { path, revision }
if self.path.as_deref() == Some(path.as_str()) && revision > self.saved_revision && revision <= self.revision =>
{
self.saved_revision = revision;
}
autosave::WorkerEvent::Error(m) => {
self.log.push(panels::Level::Error, "autosave", m.clone());
@ -1105,3 +1105,5 @@ mod trim_tests;
mod vfx_tests;
#[cfg(test)]
mod voiceover_tests;
#[cfg(test)]
mod wasm_clock_tests;

2
crates/engine/src/masks.rs
View File

@ -574,7 +574,7 @@ fn track(s: &mut Session, p: &Value) -> Result<Value> {
let start_path = mask.path_at(mt0);
let run = move || {
use std::sync::atomic::Ordering;
let t0 = std::time::Instant::now();
let t0 = web_time::Instant::now();
let gray = |t: Tick| -> Option<(filmcraft_render::track::Prepared, f64)> {
let f = src.video_frame(filmcraft_media::FrameRequest { time: t, scale }).ok()?;
let g = filmcraft_render::track::Gray::from_rgba8(f.width as usize, f.height as usize, &f.to_rgba8());

6
crates/engine/src/media_browser.rs
View File

@ -663,7 +663,9 @@ pub fn std_volumes() -> Vec<Volume> {
v
}
/// The desktop user's home directory.
/// The desktop user's home directory. Windows prefers `USERPROFILE`: shells like Git Bash set
/// `HOME` to a Unix-style path (`/c/Users/…`) that native file APIs cannot open.
pub fn std_home_dir() -> Option<String> {
std::env::var("HOME").ok().or_else(|| std::env::var("USERPROFILE").ok()).filter(|h| !h.is_empty())
let vars = if cfg!(windows) { ["USERPROFILE", "HOME"] } else { ["HOME", "USERPROFILE"] };
vars.iter().find_map(|v| std::env::var(v).ok().filter(|h| !h.is_empty()))
}

10
crates/engine/src/mixer.rs
View File

@ -1129,19 +1129,19 @@ pub fn audio_gain(s: &mut Session, p: &Value) -> Result<Value> {
}
m => return Err(bad("clip.audioGain", format!("unknown mode `{m}` (set, adjust, normalizeMax, normalizeAll)"))),
};
let n = new_gain.len();
let applied = new_gain.clone();
s.edit_sequence("Audio Gain", move |q, _, _| {
let applied = s.edit_sequence("Audio Gain", move |q, _, _| {
let mut applied = Vec::new();
for t in q.audio_tracks.iter_mut() {
for i in t.items.iter_mut() {
if let Some((_, g)) = new_gain.iter().find(|(c, _)| *c == i.id) {
i.gain_db = g.clamp(-96.0, 96.0);
applied.push((i.id, i.gain_db));
}
}
}
Ok(())
Ok(applied)
})?;
Ok(json!({"mode": mode, "clips": n, "gainDb": applied.iter().map(|(c, g)| json!({"clip": c.0, "gainDb": g})).collect::<Vec<_>>()}))
Ok(json!({"mode": mode, "clips": applied.len(), "gainDb": applied.iter().map(|(c, g)| json!({"clip": c.0, "gainDb": g})).collect::<Vec<_>>()}))
}
/// Effects panel ▸ Set Selected as Default Transition (video or audio, from the effect's kind).

39
crates/engine/src/mixer_tests.rs
View File

@ -312,6 +312,45 @@ fn audio_gain_modes_and_peak() {
assert!(s.execute("clip.audioGain", json!({"mode": "loud"})).is_err());
}
#[test]
fn audio_gain_reports_only_audio_items_it_updates() {
for mode in ["set", "adjust"] {
for target in [29, 30] {
for linked in [true, false] {
let mut s = demo();
s.execute("sequence.linkedSelection", json!({"on": linked})).unwrap();
let r = s.execute("clip.audioGain", json!({"clips": [target], "mode": mode, "db": -6})).unwrap();
let applied = linked || target == 30;
assert_eq!(r["clips"], usize::from(applied), "{mode}, target {target}, linked {linked}: {r}");
assert_eq!(r["gainDb"], if applied { json!([{"clip": 30, "gainDb": -6.0}]) } else { json!([]) });
let q = s.active_sequence().unwrap();
assert_eq!(q.find_item(ClipId(29)).unwrap().1.gain_db, 0.0);
assert_eq!(q.find_item(ClipId(30)).unwrap().1.gain_db, if applied { -6.0 } else { 0.0 });
if applied {
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(s.active_sequence().unwrap().find_item(ClipId(30)).unwrap().1.gain_db, 0.0);
s.execute("edit.redo", json!({})).unwrap();
assert_eq!(s.active_sequence().unwrap().find_item(ClipId(30)).unwrap().1.gain_db, -6.0);
}
}
}
}
}
#[test]
fn audio_gain_reports_the_clamped_stored_gain() {
for mode in ["set", "adjust"] {
for db in [-200.0_f64, 200.0] {
let mut s = demo();
s.execute("sequence.linkedSelection", json!({"on": false})).unwrap();
let r = s.execute("clip.audioGain", json!({"clips": [30], "mode": mode, "db": db})).unwrap();
let stored = s.active_sequence().unwrap().find_item(ClipId(30)).unwrap().1.gain_db;
assert_eq!(stored, db.clamp(-96.0, 96.0));
assert_eq!(r["gainDb"], json!([{"clip": 30, "gainDb": stored}]));
}
}
}
#[test]
fn default_audio_transition_is_used_by_apply() {
let mut s = demo();

5
crates/engine/src/presets.rs
View File

@ -550,13 +550,14 @@ fn export(s: &mut Session, p: &Value) -> Result<Value> {
if chosen.is_empty() {
return Err(bad("presets.export", "no presets to export"));
}
filmcraft_format::atomic_write(Path::new(path), &file_bytes(&chosen)).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
// through the host: atomic on the desktop, a download on the web (no filesystem there)
s.services.write_file(path, &file_bytes(&chosen)).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
Ok(json!({"path": path, "count": chosen.len()}))
}
fn import(s: &mut Session, p: &Value) -> Result<Value> {
let path = str_p(p, "path").ok_or_else(|| bad("presets.import", "need `path`"))?;
let bytes = std::fs::read(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
let presets = parse_file(&bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
let names: Vec<String> = presets.iter().map(|x| x.name.clone()).collect();
for pr in presets {

37
crates/engine/src/presets_tests.rs
View File

@ -1,7 +1,10 @@
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use serde_json::json;
use crate::Session;
use crate::presets::{KeyframeMode, retime};
use crate::{Services, Session};
use filmcraft_project::{ClipId, ParamValue, TrackItem};
use filmcraft_time::{TICKS_PER_SECOND, Tick};
@ -203,3 +206,35 @@ fn mask_geometry_scales_to_the_target_frame_size() {
assert!((c.x - 480.0).abs() < 1e-6 && (c.y - 270.0).abs() < 1e-6, "{c:?}");
assert_eq!(m.feather.value, ParamValue::Float(80.0));
}
/// A host without a filesystem (the web): files exist only through `Services`.
#[derive(Default)]
struct MemFs(Mutex<BTreeMap<String, Vec<u8>>>);
impl Services for MemFs {
fn read_file(&self, path: &str) -> std::io::Result<Vec<u8>> {
self.0.lock().unwrap().get(path).cloned().ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, path.to_string()))
}
fn write_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> {
self.0.lock().unwrap().insert(path.into(), data.to_vec());
Ok(())
}
}
/// #379: export and import go through the host's services, not `std::fs` (which traps on the web).
#[test]
fn export_and_import_go_through_host_services() {
let fs = Arc::new(MemFs::default());
let mut s = Session::new(fs.clone());
let path = "/exports/owned-preset.prfpset";
let r = s.execute("presets.export", json!({"path": path, "names": ["Fade In"]})).unwrap();
assert_eq!(r["count"], 1);
let bytes = fs.read_file(path).unwrap();
assert!(String::from_utf8_lossy(&bytes).contains("filmcraft.effect-presets"));
// a second session on the same host imports it from there
let mut s2 = Session::new(fs);
let r = s2.execute("presets.import", json!({"path": path})).unwrap();
assert_eq!(r["imported"], json!(["Fade In"]));
assert_eq!(s2.presets.user.len(), 1);
assert!(s2.execute("presets.import", json!({"path": "/exports/missing.prfpset"})).is_err());
}

4
crates/engine/src/previews.rs
View File

@ -480,7 +480,7 @@ pub fn render(s: &mut Session, mode: RenderMode, p: &Value) -> Result<Value> {
let nseg = todo.len();
let pool = s.media.clone();
let run = move || {
let t0 = std::time::Instant::now();
let t0 = web_time::Instant::now();
let mut bytes = 0u64;
let mut done_frames = 0u64;
let mut outcome: std::result::Result<(), String> = Ok(());
@ -581,7 +581,7 @@ pub fn render_audio(s: &mut Session, p: &Value) -> Result<Value> {
let (prog, res) = (job.progress.clone(), job.result.clone());
let nseg = todo.len();
let run = move || {
let t0 = std::time::Instant::now();
let t0 = web_time::Instant::now();
let mut bytes = 0u64;
let mut outcome: std::result::Result<(), String> = Ok(());
let Some(q) = project.sequence(seq) else { return };

2
crates/engine/src/project_manager.rs
View File

@ -208,7 +208,7 @@ pub fn run(s: &mut Session, p: &Value) -> Result<Value> {
let prev_dst = crate::previews::dir_for_project(&project_path);
let services = s.services.clone();
let run = move || {
let t0 = std::time::Instant::now();
let t0 = web_time::Instant::now();
let r = execute(proj, work, sources, previews.map(|d| (d, prev_dst)), &project_path, pr, &prog, &*services);
let secs = t0.elapsed().as_secs_f64();
let r = r.map(|bytes| filmcraft_export::Report {

2
crates/engine/src/proxies.rs
View File

@ -200,7 +200,7 @@ pub fn start_job(s: &mut Session, label: String, work: Vec<(ItemId, String, &'st
let outputs: Arc<Mutex<Vec<(ItemId, String)>>> = Arc::default();
let (prog, res, outs) = (job.progress.clone(), job.result.clone(), outputs.clone());
let run = move || {
let t0 = std::time::Instant::now();
let t0 = web_time::Instant::now();
let mut bytes = 0;
let mut err = None;
let n = tasks.len();

21
crates/engine/src/sequence_tools_tests.rs
View File

@ -483,3 +483,24 @@ fn selection_follows_playhead() {
s.execute("playhead.set", json!({"frame": 10})).unwrap();
assert_eq!(s.state.selection, sel, "off again");
}
/// #208: with no In / Out mark, Go to In / Go to Out move to the start / end of the sequence
/// (they did nothing); with marks they go to them.
#[test]
fn go_to_in_out_without_marks_go_to_the_sequence_ends() {
let mut s = demo();
let end = s.active_sequence().unwrap().duration();
let mid = filmcraft_time::Tick(end.0 / 2);
s.execute("playhead.set", json!({"time": mid.0})).unwrap();
s.execute("markers.goToIn", json!({})).unwrap();
assert_eq!(s.playhead(), filmcraft_time::Tick::ZERO, "the start");
s.execute("markers.goToOut", json!({})).unwrap();
assert_eq!(s.playhead(), end, "the end");
// with marks, the marks
let at = filmcraft_time::Tick(end.0 / 4);
s.execute("playhead.set", json!({"time": at.0})).unwrap();
s.execute("markers.markIn", json!({})).unwrap();
s.execute("playhead.set", json!({"time": mid.0})).unwrap();
s.execute("markers.goToIn", json!({})).unwrap();
assert_eq!(s.playhead(), s.active_sequence().unwrap().mark_in.unwrap());
}

8
crates/engine/src/settings.rs
View File

@ -955,7 +955,7 @@ static CATEGORIES: &[Category] = &[
b("playback.draftDecode", "Draft decoding at reduced playback resolution (H.264: faster, some frames less filtered)", true),
f("playback.hardwareDecoding", "Hardware decoding", Kind::Choice(HW_DECODE), true),
Row::Note(
"Hardware decoding: Auto uses the system's video decoder (VideoToolbox on macOS) for the H.264 and HEVC streams it supports, and FilmCraft's own decoder for everything else or if the hardware fails. Media that is already open keeps its decoder until it is reopened.",
"Hardware decoding: Auto uses the system's video decoder (VideoToolbox on macOS, Media Foundation on Windows, VA-API for H.264 and HEVC on Linux) for the streams it supports, and FilmCraft's own decoder for everything else or if the hardware fails. Media that is already open keeps its decoder until it is reopened.",
),
],
},
@ -1084,10 +1084,8 @@ pub fn sanitize(v: &mut Value, defaults: &Value) {
let Some(slot) = path.iter().try_fold(&mut *v, |v, k| v.get_mut(*k)) else { continue };
let def = path.iter().try_fold(defaults, |v, k| v.get(*k)).cloned().unwrap_or(Value::Null);
match f.kind {
Kind::Choice(_) | Kind::Color => {
if validate(f.key, slot).is_err() {
*slot = def;
}
Kind::Choice(_) | Kind::Color if validate(f.key, slot).is_err() => {
*slot = def;
}
Kind::Int { min, max, .. } => match slot.as_f64().filter(|x| x.is_finite()) {
Some(x) => *slot = json!(x.round().clamp(min, max) as u64),

3
crates/engine/src/shortcut_presets.rs
View File

@ -11,6 +11,9 @@ pub type Entry = (&'static str, &'static str, &'static str);
/// FilmCraft additions to the per-command defaults: panel-specific shortcuts.
pub const FILMCRAFT_PANEL: &[Entry] = &[
("project.delete", "Backspace", "Project"),
// the forward-delete key (Delete on Windows and Linux keyboards) clears timeline clips too;
// application-wide it is Project ▸ Clear (#243)
("edit.clear", "Delete", "Timeline"),
("file.newBin", "Cmd+B", "Project"),
("edit.undo", "Left", "History"),
("edit.redo", "Right", "History"),

13
crates/engine/src/shortcuts_tests.rs
View File

@ -132,6 +132,19 @@ fn panel_shortcuts_override_application_ones_with_focus() {
assert!(k.as_array().unwrap().iter().any(|b| b["command"] == "edit.copy" && b["keys"] == "Ctrl+C"), "{k}");
}
#[test]
fn delete_clears_timeline_clips_and_project_items() {
// #243: with the Timeline focused, Delete (the forward-delete key, labelled Delete on Windows
// and Linux keyboards) resolved to Project ▸ Clear and left the selected clips in place.
let s = Session::default();
let del = Chord::parse("Delete").unwrap();
for p in [Platform::Mac, Platform::Windows] {
assert_eq!(s.shortcuts.resolve(&del, Some("Timeline"), p).unwrap().command, "edit.clear", "{p:?}");
assert_eq!(s.shortcuts.resolve(&del, Some("Project"), p).unwrap().command, "project.delete", "{p:?}");
assert_eq!(s.shortcuts.resolve(&del, None, p).unwrap().command, "project.delete", "{p:?}");
}
}
#[test]
fn compat_presets_map_other_editors() {
let mut s = Session::default();

112
crates/engine/src/tests.rs
View File

@ -144,6 +144,73 @@ fn sequence_parameters_are_bounded_and_failed_changes_are_atomic() {
assert_eq!(s.active_sequence().unwrap().settings.height, 8192, "a 16384x8192 panorama is accepted");
}
/// The first video clip's Motion Position and Scale (static values).
fn first_motion(s: &Session) -> (u64, (f64, f64), f64) {
let q = s.active_sequence().unwrap();
let it = q.video_tracks.iter().flat_map(|t| &t.items).find(|i| i.effect("motion").is_some()).unwrap();
let m = it.effect("motion").unwrap();
let p = match m.param("position").unwrap().value {
filmcraft_project::ParamValue::Vec2(v) => (v.x, v.y),
_ => panic!("position is a point"),
};
let sc = match m.param("scale").unwrap().value {
filmcraft_project::ParamValue::Float(v) => v,
_ => panic!("scale is a number"),
};
(it.id.0, p, sc)
}
#[test]
fn sequence_settings_scales_motion_with_the_frame_size_in_one_undo_step() {
let mut s = demo();
let before = (*s.project).clone();
let (id, p0, s0) = first_motion(&s);
assert_eq!((s.active_sequence().unwrap().settings.width, s.active_sequence().unwrap().settings.height), (1920, 1080));
let r = s.execute("sequence.settings", json!({"width":1280,"height":720,"scaleMotion":true})).unwrap();
assert!(r["scaledClips"].as_u64().unwrap() >= 1, "{r}");
let (id1, p1, s1) = first_motion(&s);
assert_eq!(id1, id);
assert!((p1.0 - p0.0 * 2.0 / 3.0).abs() < 1e-9 && (p1.1 - p0.1 * 2.0 / 3.0).abs() < 1e-9, "{p0:?} -> {p1:?}");
assert!((s1 - s0 * 2.0 / 3.0).abs() < 1e-9, "{s0} -> {s1}");
let after = (*s.project).clone();
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(*s.project, before, "size and Motion come back in one undo step");
s.execute("edit.redo", json!({})).unwrap();
assert_eq!(*s.project, after);
// without the option the frame size changes alone (the command's default, as before)
let r = s.execute("sequence.settings", json!({"width":3840,"height":2160})).unwrap();
assert_eq!(r["scaledClips"], json!(0));
assert_eq!(first_motion(&s), (id, p1, s1));
assert_eq!(s.active_sequence().unwrap().settings.width, 3840);
}
#[test]
fn sequence_settings_timecode_render_quality_and_colour() {
let mut s = demo();
s.execute("sequence.settings", json!({"fps":29.97,"dropFrame":true,"maxRenderQuality":true})).unwrap();
let st = s.active_sequence().unwrap().settings.clone();
assert_eq!(st.frame_rate, FrameRate::FPS_29_97);
assert!(st.drop_frame && st.max_render_quality);
s.execute("sequence.settings", json!({"dropFrame":false})).unwrap();
assert!(!s.active_sequence().unwrap().settings.drop_frame);
// drop-frame only exists for the NTSC rates; a refused change leaves everything as it was
let before = (*s.project).clone();
let history = s.history.undo.len();
for params in [json!({"fps":25,"dropFrame":true}), json!({"workingSpace":"rec2020-log"}), json!({"fps":25,"dropFrame":true,"width":1280})] {
assert!(s.execute("sequence.settings", params.clone()).is_err(), "{params}");
assert_eq!(*s.project, before, "{params}");
assert_eq!(s.history.undo.len(), history);
}
s.execute("sequence.settings", json!({"workingSpace":"rec2100-pq","wideGamut":true,"autoToneMap":true})).unwrap();
let st = s.active_sequence().unwrap().settings.clone();
assert_eq!(st.color.working, filmcraft_color::WorkingSpace::Rec2100Pq);
assert!(st.color.wide_gamut && st.color.auto_tone_map);
assert_eq!(st.working_space, "Rec. 2100 PQ");
// disabled without a sequence
let mut empty = Session::default();
assert!(empty.execute("sequence.settings", json!({"width":1280})).is_err());
}
#[test]
fn zero_fps_is_refused() {
let mut s = demo();
@ -170,6 +237,51 @@ fn effects_and_keyframes() {
assert!(it.effect("motion").unwrap().params["scale"].is_animated());
}
/// `effects.addKeyframe` is the keyframe diamond of Effect Controls and Properties: it adds a
/// keyframe at the playhead or removes the one there, and the value then follows the keyframes
/// that are left (Premiere's Add/Remove Keyframe), unlike the stopwatch, which drops them all.
#[test]
fn add_keyframe_toggles_the_keyframe_at_the_playhead() {
let mut s = demo();
let c = s.active_sequence().unwrap().video_tracks[0].items[0].id.0;
let scale = |s: &Session| {
let it = s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1;
let p = &it.effect("motion").unwrap().params["scale"];
(p.keyframes.len(), p.f64_at(it.source_time_at(s.playhead())))
};
let key = json!({"clip": c, "effect": "motion", "param": "scale"});
s.execute("playhead.set", json!({"seconds": 0.5})).unwrap();
s.execute("effects.addKeyframe", key.clone()).unwrap();
assert_eq!(scale(&s), (1, 100.0), "the first keyframe turns animation on");
s.execute("playhead.set", json!({"seconds": 2.5})).unwrap();
s.execute("effects.addKeyframe", key.clone()).unwrap();
s.execute("effects.setParam", json!({"clip": c, "effect": "motion", "param": "scale", "value": 50.0})).unwrap();
assert_eq!(scale(&s), (2, 50.0));
// on a keyframe: only that keyframe goes, and Scale is the other keyframe's 100 again
s.execute("effects.addKeyframe", key.clone()).unwrap();
assert_eq!(scale(&s), (1, 100.0));
s.undo();
assert_eq!(scale(&s), (2, 50.0));
s.redo();
assert_eq!(scale(&s), (1, 100.0));
// parameters that name nothing are errors and change nothing
for bad in [
json!({}),
json!({"clip": c}),
json!({"clip": u64::MAX, "param": "scale"}),
json!({"clip": c, "effect": "nope", "param": "scale"}),
json!({"clip": c, "effect": u64::MAX, "param": "scale"}),
json!({"clip": c, "effect": "motion", "param": "nope"}),
json!({"clip": c, "effect": "motion", "param": "scale", "mask": 7}),
] {
assert!(s.execute("effects.addKeyframe", bad.clone()).is_err(), "{bad}");
}
assert_eq!(scale(&s), (1, 100.0));
// the stopwatch ends the animation and keeps the value at the playhead
s.execute("effects.toggleAnimation", key).unwrap();
assert_eq!(scale(&s), (0, 100.0));
}
#[test]
fn transitions_and_markers() {
let mut s = demo();

53
crates/engine/src/wasm_clock_tests.rs Normal file
View File

@ -0,0 +1,53 @@
//! The web build runs every job synchronously (`wait || cfg!(target_arch = "wasm32")`), so a job
//! body executes on wasm32 — where `std::time::Instant::now()` panics with "time not implemented
//! on this platform" and takes the whole app down (#230: Create Proxies and mask tracking both
//! did). `web_time::Instant` is `std::time::Instant` on native targets and `performance.now()`
//! on the web, so there is never a reason to name the std one in this crate.
use std::path::Path;
/// Every non-test source file of the crate, as (path, contents).
fn sources() -> Vec<(String, String)> {
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let mut out = Vec::new();
let mut stack = vec![dir];
while let Some(d) = stack.pop() {
for e in std::fs::read_dir(&d).unwrap().flatten() {
let p = e.path();
if p.is_dir() {
stack.push(p);
continue;
}
let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default();
if !name.ends_with(".rs") || name.ends_with("_tests.rs") || name == "media_test_util.rs" {
continue;
}
out.push((p.display().to_string(), std::fs::read_to_string(&p).unwrap()));
}
}
assert!(out.len() > 20, "found {} source files", out.len());
out
}
#[test]
fn no_std_instant_in_code_the_web_build_runs() {
let mut hits = Vec::new();
for (path, text) in sources() {
for (i, line) in text.lines().enumerate() {
// `use std::time::{.., Instant, ..}` or a fully qualified `std::time::Instant`
let names_std_instant = line.contains("std::time::Instant") || (line.contains("use std::time::") && line.contains("Instant"));
if names_std_instant && !line.contains("web_time") && !line.trim_start().starts_with("//") {
hits.push(format!("{path}:{}: {}", i + 1, line.trim()));
}
}
}
assert!(hits.is_empty(), "std::time::Instant panics on wasm32; use web_time::Instant:\n{}", hits.join("\n"));
}
/// The clock the jobs use works here as well: a job's "Done in 0.0s" report needs an elapsed time.
#[test]
fn web_time_instant_measures_elapsed_time() {
let t0 = web_time::Instant::now();
let secs = t0.elapsed().as_secs_f64();
assert!((0.0..60.0).contains(&secs), "{secs}");
}

21
crates/export/src/lib.rs
View File

@ -253,6 +253,10 @@ pub struct ExportSettings {
/// sequence's HDR space and signal it (VUI / `colr` / `mdcv` / `clli` / SEI).
#[serde(default)]
pub sdr: bool,
/// Keep the alpha channel in PNG and TIFF sequences (straight alpha, Premiere's "Include
/// Alpha Channel"). Off, every frame is flattened over black (#160). Other formats ignore it.
#[serde(default)]
pub alpha: bool,
/// Output frame size (None = Match Source: the sequence size times `scale`).
pub frame_size: Option<(u32, u32)>,
/// Output frame rate (None = Match Source).
@ -445,6 +449,7 @@ impl Default for ExportSettings {
apv_profile: String::new(),
mxf_video_codec: MxfVideoCodec::default(),
sdr: false,
alpha: false,
frame_size: None,
frame_rate: None,
scaling: Scaling::default(),
@ -1266,11 +1271,12 @@ impl Write for SharedBuf {
/// Encode straight sRGB RGBA8 pixels as a PNG file (thumbnails, previews).
pub fn encode_png(rgba: Vec<u8>, w: u32, h: u32) -> Result<Vec<u8>> {
encode_still(Format::PngSequence, rgba, w, h)
encode_still(Format::PngSequence, rgba, w, h, false)
}
/// Encode one still of an image sequence (also Export Frame).
pub fn encode_still(format: Format, rgba: Vec<u8>, w: u32, h: u32) -> Result<Vec<u8>> {
/// Encode one still of an image sequence (also Export Frame). PNG always carries the alpha
/// channel it is given; `alpha` keeps it in TIFF too, which is otherwise written as RGB.
pub fn encode_still(format: Format, rgba: Vec<u8>, w: u32, h: u32, alpha: bool) -> Result<Vec<u8>> {
let enc = |e: image::ImageError| ExportError::Encode(e.to_string());
let mut out = std::io::Cursor::new(Vec::new());
match format {
@ -1279,9 +1285,12 @@ pub fn encode_still(format: Format, rgba: Vec<u8>, w: u32, h: u32) -> Result<Vec
}
Format::TiffSequence | Format::BmpSequence => {
let img = image::RgbaImage::from_raw(w, h, rgba).ok_or_else(|| ExportError::Encode("frame size".into()))?;
let rgb = image::DynamicImage::ImageRgba8(img).to_rgb8();
let f = if format == Format::TiffSequence { image::ImageFormat::Tiff } else { image::ImageFormat::Bmp };
rgb.write_to(&mut out, f).map_err(enc)?
if alpha && format == Format::TiffSequence {
img.write_to(&mut out, f).map_err(enc)?
} else {
image::DynamicImage::ImageRgba8(img).to_rgb8().write_to(&mut out, f).map_err(enc)?
}
}
_ => return Err(ExportError::Unsupported(format!("{} is not an image sequence", format.label()))),
}
@ -1358,7 +1367,7 @@ pub fn export(project: &Arc<Project>, seq: ItemId, settings: &ExportSettings, so
let written: Vec<Result<u64>> = (f..end)
.into_par_iter()
.map(|fi| {
let data = encode_still(settings.format, pipe.frame(fi, sources).0, w, h)?;
let data = encode_still(settings.format, pipe.frame(fi, sources).0, w, h, settings.alpha)?;
write_output(settings, &pcm::image_sequence_path(&settings.path, (fi - f0) as u64, count), data)
})
.collect();

7
crates/export/src/pipeline.rs
View File

@ -12,7 +12,7 @@ use filmcraft_time::FrameRate;
use rayon::prelude::*;
use crate::settings::{ExportEffects, Placement, Scaling, TextOverlay};
use crate::{ExportError, ExportSettings, Result};
use crate::{ExportError, ExportSettings, Format, Result};
/// Where the rendered picture lands in the output frame.
#[derive(Clone, Copy, Debug, PartialEq)]
@ -61,6 +61,8 @@ pub(crate) struct Pipeline {
geom: Geometry,
opts: RenderOptions,
pub hdr_out: bool,
/// Keep straight alpha in the RGBA8 output instead of flattening over black.
alpha: bool,
out_tf: Option<filmcraft_color::OutputTransform>,
effects: ExportEffects,
overlay: Option<Image>,
@ -106,6 +108,7 @@ impl Pipeline {
geom,
opts,
hdr_out,
alpha: settings.alpha && matches!(settings.format, Format::PngSequence | Format::TiffSequence),
out_tf,
effects,
overlay,
@ -134,7 +137,7 @@ impl Pipeline {
filmcraft_frame::pool::recycle_f32(img.px);
return (Vec::new(), out);
}
let mut rgba = img.over_black_rgba8();
let mut rgba = if self.alpha { img.to_rgba8() } else { img.over_black_rgba8() };
// the float image is not needed any more: its buffer serves the next frame's layers
filmcraft_frame::pool::recycle_f32(img.px);
if lim.enabled {

22
crates/export/src/settings_tests.rs
View File

@ -109,6 +109,28 @@ fn image_sequences_are_numbered_like_premiere() {
}
}
/// `alpha` keeps straight alpha in PNG and TIFF sequences; off (the default) flattens over black (#160).
#[test]
fn image_sequences_keep_alpha_when_asked() {
let (p, seq, m) = matte([0.0, 0.0, 1.0, 0.5], 64, 36, None);
for (fmt, ext) in [(Format::PngSequence, "png"), (Format::TiffSequence, "tif")] {
for alpha in [false, true] {
let dir = Scratch::new(&format!("alpha-{ext}-{alpha}"));
let mut s = ExportSettings { format: fmt, path: dir.path(&format!("a.{ext}")), alpha, ..Default::default() };
s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(2)));
export(&p, seq, &s, &m, &Progress::default()).unwrap();
let img = image::open(dir.0.join(format!("a000.{ext}"))).unwrap();
let px = img.to_rgba8().get_pixel(32, 18).0;
if alpha {
assert!(img.color().has_alpha(), "{fmt:?} must be written with an alpha channel");
assert!((100..=160).contains(&px[3]) && px[2] > 200 && px[0] < 15, "{fmt:?} straight alpha: {px:?}");
} else {
assert!(px[3] == 255 && px[2] > 100 && px[2] < 240 && px[0] < 15, "{fmt:?} flattened over black: {px:?}");
}
}
}
}
#[test]
fn wav_and_aiff_audio_only() {
let (p, seq, m) = matte([0.0, 0.0, 0.0, 1.0], 64, 36, Some(-6.0));

28
crates/gpu/src/lib.rs
View File

@ -505,15 +505,28 @@ impl GpuCompositor {
/// The source description of an uploaded frame.
fn src_info(&self, f: &VideoFrame, key: (usize, u32, u32)) -> Option<SrcInfo> {
let up = self.uploads.get(&key)?;
Some(SrcInfo { kind: up.kind, code_scale: up.code_scale, alpha: up.alpha_scale, chroma: up.chroma, size: (f.width, f.height), color: f.color })
let code_levels = match &f.data {
PixelData::Yuv8 { .. } => 256.0,
PixelData::Yuv16 { bits, .. } => (*bits as f32).exp2(),
_ => 1.0,
};
Some(SrcInfo {
kind: up.kind,
code_scale: up.code_scale,
code_levels,
alpha: up.alpha_scale,
chroma: up.chroma,
size: (f.width, f.height),
color: f.color,
})
}
fn uniforms(src: &SrcInfo, m: &filmcraft_geom::Affine, opacity: f32, blend: Blend, out: (u32, u32)) -> [f32; 28] {
let (kr, kb) = src.color.matrix.kr_kb();
let bits_scale = src.code_scale / 255.0; // code units relative to 8-bit
let bits_scale = src.code_levels / 256.0; // 2^(bits - 8), independent of texture encoding
let (yo, ys, co, cs) = match (src.color.range, src.kind) {
(Range::Limited, 2) => (16.0 * bits_scale, 219.0 * bits_scale, 128.0 * bits_scale, 224.0 * bits_scale),
(Range::Full, 2) => (0.0, src.code_scale - 1.0, src.code_scale / 2.0, src.code_scale - 1.0),
(Range::Full, 2) => (0.0, src.code_levels - 1.0, src.code_levels / 2.0, src.code_levels - 1.0),
_ => (0.0, 1.0, 0.0, 1.0),
};
let transfer = match src.color.transfer {
@ -645,6 +658,7 @@ impl GpuCompositor {
let src = self.src_info(&l.frame, *k).unwrap_or(SrcInfo {
kind: 1,
code_scale: 1.0,
code_levels: 1.0,
alpha: 0.0,
chroma: (1, 1),
size: (l.frame.width, l.frame.height),
@ -678,7 +692,7 @@ impl GpuCompositor {
// what the layer draws: the effect result (linear premultiplied RGBA) or the frame
let (u, tex) = match &job {
Some(j) => {
let s = SrcInfo { kind: 1, code_scale: 1.0, alpha: 0.0, chroma: l.size(), size: l.size(), color: l.frame.color };
let s = SrcInfo { kind: 1, code_scale: 1.0, code_levels: 1.0, alpha: 0.0, chroma: l.size(), size: l.size(), color: l.frame.color };
(Self::uniforms(&s, &l.matrix, l.opacity, l.blend, (w, h)), [j.result.clone(), self.dummy.clone(), self.dummy.clone(), self.dummy.clone()])
}
None => {
@ -884,11 +898,13 @@ impl GpuCompositor {
}
}
/// What a layer's shader samples: the texture kind (0 RGBA8 sRGB, 1 linear premultiplied RGBA
/// float, 2 YUV planes), code scale, chroma plane size, picture size and colour description.
/// Shader source: kind (0 RGBA8 sRGB, 1 linear premultiplied RGBA, 2 YUV), sampling scales, dimensions, color.
struct SrcInfo {
kind: u32,
/// Texture sample -> code units: 255 for R8Unorm, 2^bits for R16Float.
code_scale: f32,
/// YUV quantization levels: 2^bits; R8Unorm's decoding scale is one less.
code_levels: f32,
/// Alpha texture sample → alpha (0: the frame has no alpha plane).
alpha: f32,
chroma: (u32, u32),

77
crates/gpu/src/tests.rs
View File

@ -9,6 +9,83 @@ fn device() -> Option<(wgpu::Device, wgpu::Queue)> {
pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()
}
#[test]
fn yuv_code_range_uniforms() {
let mut mismatches = Vec::new();
for bits in [8, 10, 12, 16] {
let levels = (1u32 << bits) as f32;
for range in [Range::Full, Range::Limited] {
let src = SrcInfo {
kind: 2,
code_scale: if bits == 8 { 255.0 } else { levels },
code_levels: levels,
alpha: 0.0,
chroma: (1, 1),
size: (1, 1),
color: filmcraft_color::ColorInfo { range, ..filmcraft_color::ColorInfo::REC709 },
};
let u = GpuCompositor::uniforms(&src, &Affine::IDENTITY, 1.0, Blend::Normal, (1, 1));
// Full: 0..2^bits-1, neutral 2^(bits-1). Limited: BT.709 8-bit codes * 2^(bits-8).
let expected = match range {
Range::Full => [0.0, levels - 1.0, levels / 2.0, levels - 1.0],
Range::Limited => [16.0, 219.0, 128.0, 224.0].map(|v| v * (1u32 << (bits - 8)) as f32),
};
if u[16..20] != expected {
mismatches.push(format!("{bits}-bit {range:?}: {:?} != {expected:?}", &u[16..20]));
}
}
}
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
}
#[test]
fn yuv_code_range_gpu_matches_cpu() {
let Some((dev, q)) = device() else {
eprintln!("no GPU adapter; skipping");
return;
};
let mut c = GpuCompositor::new(&dev, &q);
let mut prepared_compositor = GpuCompositor::new(&dev, &q);
for bits in [8, 10, 12, 16] {
let scale = 1u32 << (bits - 8);
for range in [Range::Full, Range::Limited] {
// Constant chroma isolates range normalization from CPU/GPU chroma reconstruction.
for (u, v) in [(128, 128), (64, 192), (192, 64)] {
let y = (0..=255).map(|code| (code * scale) as u16).collect::<Vec<_>>();
let planes = [Arc::new(y), Arc::new(vec![(u * scale) as u16; 256]), Arc::new(vec![(v * scale) as u16; 256])];
let data = if bits == 8 {
PixelData::Yuv8 { planes: planes.map(|p| Arc::new(p.iter().map(|v| *v as u8).collect())), chroma: Chroma::C444, alpha: None }
} else {
PixelData::Yuv16 { planes, chroma: Chroma::C444, bits, alpha: None }
};
let frame = Arc::new(VideoFrame {
width: 256,
height: 1,
data,
color: filmcraft_color::ColorInfo { range, ..filmcraft_color::ColorInfo::REC709 },
par: (1, 1),
pts: Default::default(),
});
let cpu = frame.to_rgba8();
let plan = FramePlan::Layers { width: 256, height: 1, layers: vec![PlanLayer::new(frame, Affine::IDENTITY, 1.0, Blend::Normal)] };
c.composite(&plan);
let (w, h, gpu) = c.read_output().expect("readback");
assert_eq!((w, h, gpu.len()), (256, 1, cpu.len()));
let error = cpu.iter().zip(&gpu).map(|(a, b)| a.abs_diff(*b)).max().expect("pixels");
// One output-code tolerance: CPU f32 vs GPU f16 accumulator quantization.
assert!(error <= 1, "{bits}-bit {range:?} U={u} V={v}: max error {error}");
if u == 128 && v == 128 {
assert!(gpu.chunks_exact(4).all(|p| p[0] == p[1] && p[1] == p[2]), "neutral tint: {bits}-bit {range:?}");
}
let prepared = prepare(&plan);
prepared_compositor.composite_prepared(&plan, Some(&prepared));
assert_eq!(prepared_compositor.read_output().expect("prepared readback").2, gpu, "prepared: {bits}-bit {range:?}");
eprintln!("range gate: {bits}-bit {range:?} U={u} V={v}, max error={error}, prepared=inline");
}
}
}
}
fn yuv_frame(w: u32, h: u32) -> Arc<VideoFrame> {
let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
let y: Vec<u8> = (0..w * h).map(|i| (16 + ((i % w) * 219 / w)) as u8).collect();

39
crates/h264/src/decoder.rs
View File

@ -6,11 +6,11 @@
//! jobs of later pictures block per row on the reference data they need, so several pictures decode
//! concurrently (frame-level parallelism).
use crate::dpb::{Dpb, Output, OutputMeta};
use crate::dpb::{Dpb, Output, OutputMeta, RefMark};
use crate::error::{Error, Result, ensure, invalid, unsupported};
use crate::params::{Pps, Sps};
use crate::picture::{Frame, FrameRef, MbKind, MbState, Planes, RefPic};
use crate::slice::{NalHeader, Poc, PocState, SliceHeader, SliceType, nal_type};
use crate::slice::{NalHeader, Poc, PocState, SliceHeader, SliceType, is_new_picture, nal_type};
use crate::slicedec::{PicState, SliceDecoder};
use crate::transform::LevelScale;
use crate::{ColorInfo, Picture};
@ -81,7 +81,7 @@ pub struct Decoder {
spss: Vec<Option<Arc<Sps>>>,
ppss: Vec<Option<Arc<Pps>>>,
nal_length_size: Option<usize>,
dpb: Dpb,
dpb: Dpb<FrameRef>,
poc_state: PocState,
prev_ref_frame_num: u32,
pending: Option<PendingPic>,
@ -89,7 +89,7 @@ pub struct Decoder {
active_sps: Option<Arc<Sps>>,
ls_cache: Vec<(Arc<Pps>, Arc<LevelScale>)>,
/// Pictures leaving the DPB, converted once their decoding job has finished.
out_queue: VecDeque<Output>,
out_queue: VecDeque<Output<FrameRef>>,
shared: Arc<Shared>,
#[cfg(feature = "threads")]
pool: Option<rayon::ThreadPool>,
@ -106,27 +106,6 @@ impl Default for Decoder {
}
}
/// Is `sh` the first slice of a new primary coded picture relative to `prev` (7.4.1.2.4)?
fn is_new_picture(prev: &SliceHeader, sh: &SliceHeader, sps: &Sps) -> bool {
if sh.frame_num != prev.frame_num
|| sh.pps_id != prev.pps_id
|| sh.field_pic != prev.field_pic
|| sh.bottom_field != prev.bottom_field
|| (sh.nal_ref_idc == 0) != (prev.nal_ref_idc == 0)
|| sh.idr != prev.idr
|| (sh.idr && prev.idr && sh.idr_pic_id != prev.idr_pic_id)
{
return true;
}
if sps.pic_order_cnt_type == 0 && (sh.pic_order_cnt_lsb != prev.pic_order_cnt_lsb || sh.delta_pic_order_cnt_bottom != prev.delta_pic_order_cnt_bottom) {
return true;
}
if sps.pic_order_cnt_type == 1 && sh.delta_pic_order_cnt != prev.delta_pic_order_cnt {
return true;
}
false
}
/// The worker threads [`Decoder::new`] uses: the available cores (at most 16) with the `threads`
/// feature on native targets, 1 otherwise.
pub fn default_threads() -> usize {
@ -384,7 +363,9 @@ impl Decoder {
let Some(pending) = self.pending.as_mut() else {
return invalid("slice without a started picture");
};
let refs = self.dpb.build_ref_lists(&sh, pending.poc.frame(), sps.max_frame_num())?;
let refs = self
.dpb
.build_ref_lists(&sh, pending.poc.frame(), sps.max_frame_num(), |e| RefPic { frame: e.frame.clone(), long_term: e.mark == RefMark::Long })?;
if !sh.slice_type.is_intra() {
ensure!(!refs[0].is_empty(), "no reference pictures available for inter slice");
if sh.slice_type.is_b() {
@ -464,7 +445,7 @@ impl Decoder {
}
_ => Planes::gray(mb_w * 16, mb_h * 16),
};
(Arc::new(Frame::from_planes(id, 0, &planes)), meta.clone())
(Arc::new(Frame::from_planes(id, 0, &planes)), 0, meta.clone())
};
let mut upd = |fnum: u32| poc_state.update_gap_frame(fnum, sps);
self.shared.stats.lock().map(|mut s| s.frame_num_gaps += 1).ok();
@ -516,7 +497,7 @@ impl Decoder {
run_job(frame, slices, &shared, draft);
}
fn emit(&mut self, outs: Vec<Output>) {
fn emit(&mut self, outs: Vec<Output<FrameRef>>) {
self.out_queue.extend(outs);
}
}
@ -613,7 +594,7 @@ fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta {
}
}
fn make_picture(o: &Output) -> Picture {
fn make_picture(o: &Output<FrameRef>) -> Picture {
let (cx, cy, cw, ch) = o.meta.crop;
let (cx, cy, cw, ch) = (cx as usize, cy as usize, cw as usize, ch as usize);
let (y, u, v) = o.frame.copy_cropped((cx, cy, cw, ch));

68
crates/h264/src/dpb.rs
View File

@ -1,8 +1,11 @@
//! Decoded picture buffer: reference marking (8.2.5), reference list construction (8.2.4) and output
//! ordering / bumping (C.4).
//!
//! The buffer only does bookkeeping on picture headers; what a picture is (`F`: the software
//! decoder's [`crate::picture::FrameRef`], a hardware decoder's surface) is up to the decoder, so
//! hardware decoders driven by the same headers (VA-API) share this logic and output order.
use crate::error::{Result, ensure};
use crate::picture::{FrameRef, RefPic};
use crate::slice::{Mmco, SliceHeader, SliceType};
use std::sync::Arc;
@ -28,8 +31,8 @@ pub struct OutputMeta {
pub draft: bool,
}
pub struct DpbEntry {
pub frame: FrameRef,
pub struct DpbEntry<F> {
pub frame: F,
pub poc: i32,
pub frame_num: u32,
pub long_term_frame_idx: u32,
@ -39,8 +42,8 @@ pub struct DpbEntry {
pub meta: Arc<OutputMeta>,
}
pub struct Dpb {
pub entries: Vec<DpbEntry>,
pub struct Dpb<F> {
pub entries: Vec<DpbEntry<F>>,
/// None = "no long-term frame indices".
pub max_long_term_frame_idx: Option<u32>,
pub capacity: usize,
@ -48,18 +51,18 @@ pub struct Dpb {
}
/// A picture leaving the DPB for output.
pub struct Output {
pub frame: FrameRef,
pub struct Output<F> {
pub frame: F,
pub poc: i32,
pub meta: Arc<OutputMeta>,
}
impl Dpb {
impl<F: Clone> Dpb<F> {
pub fn new() -> Self {
Dpb { entries: Vec::new(), max_long_term_frame_idx: None, capacity: 16, max_reorder: 16 }
}
fn frame_num_wrap(e: &DpbEntry, cur_frame_num: u32, max_frame_num: u32) -> i32 {
fn frame_num_wrap(e: &DpbEntry<F>, cur_frame_num: u32, max_frame_num: u32) -> i32 {
if e.frame_num > cur_frame_num { e.frame_num as i32 - max_frame_num as i32 } else { e.frame_num as i32 }
}
@ -73,7 +76,7 @@ impl Dpb {
}
/// Output the picture with the smallest POC (C.4.5.3 "bumping"). Returns false if none waits.
fn bump(&mut self, out: &mut Vec<Output>) -> bool {
fn bump(&mut self, out: &mut Vec<Output<F>>) -> bool {
let Some(i) = self.entries.iter().enumerate().filter(|(_, e)| e.needed_for_output).min_by_key(|(_, e)| e.poc).map(|(i, _)| i) else {
return false;
};
@ -85,13 +88,13 @@ impl Dpb {
}
/// Output everything in POC order and clear non-reference entries.
pub fn flush(&mut self, out: &mut Vec<Output>) {
pub fn flush(&mut self, out: &mut Vec<Output<F>>) {
while self.bump(out) {}
self.prune();
}
/// IDR: mark all references unused; output (or drop) prior pictures.
pub fn idr(&mut self, no_output_of_prior_pics: bool, out: &mut Vec<Output>) {
pub fn idr(&mut self, no_output_of_prior_pics: bool, out: &mut Vec<Output<F>>) {
for e in &mut self.entries {
e.mark = RefMark::Unused;
}
@ -122,7 +125,8 @@ impl Dpb {
self.prune();
}
/// Insert "non-existing" frames for a frame_num gap (8.2.5.2).
/// Insert "non-existing" frames for a frame_num gap (8.2.5.2). `make_frame` gives each one's
/// picture and POC.
#[allow(clippy::too_many_arguments)]
pub fn fill_frame_num_gap(
&mut self,
@ -130,17 +134,17 @@ impl Dpb {
frame_num: u32,
max_frame_num: u32,
max_num_ref_frames: usize,
make_frame: &mut dyn FnMut(u32) -> (FrameRef, Arc<OutputMeta>),
make_frame: &mut dyn FnMut(u32) -> (F, i32, Arc<OutputMeta>),
poc_state: &mut dyn FnMut(u32),
) {
let mut unused = (prev_ref_frame_num + 1) % max_frame_num;
let mut guard = 0;
while unused != frame_num && guard < max_frame_num {
self.sliding_window(unused, max_frame_num, max_num_ref_frames);
let (frame, meta) = make_frame(unused);
let (frame, poc, meta) = make_frame(unused);
poc_state(unused);
self.entries.push(DpbEntry {
poc: frame.poc,
poc,
frame,
frame_num: unused,
long_term_frame_idx: 0,
@ -154,26 +158,26 @@ impl Dpb {
}
}
/// Initial reference picture lists (8.2.4.2) + modification (8.2.4.3) for a frame slice.
pub fn build_ref_lists(&self, sh: &SliceHeader, cur_poc: i32, max_frame_num: u32) -> Result<[Vec<RefPic>; 2]> {
let mut lists: [Vec<RefPic>; 2] = [Vec::new(), Vec::new()];
/// Initial reference picture lists (8.2.4.2) + modification (8.2.4.3) for a frame slice, each
/// entry made by `to_ref` from the DPB entry it refers to.
pub fn build_ref_lists<R>(&self, sh: &SliceHeader, cur_poc: i32, max_frame_num: u32, to_ref: impl Fn(&DpbEntry<F>) -> R) -> Result<[Vec<R>; 2]> {
let mut lists: [Vec<R>; 2] = [Vec::new(), Vec::new()];
if sh.slice_type.is_intra() {
return Ok(lists);
}
let cur_fn = sh.frame_num;
let pic_num = |e: &DpbEntry| Self::frame_num_wrap(e, cur_fn, max_frame_num);
let short: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Short).collect();
let mut long: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Long).collect();
let pic_num = |e: &DpbEntry<F>| Self::frame_num_wrap(e, cur_fn, max_frame_num);
let short: Vec<&DpbEntry<F>> = self.entries.iter().filter(|e| e.mark == RefMark::Short).collect();
let mut long: Vec<&DpbEntry<F>> = self.entries.iter().filter(|e| e.mark == RefMark::Long).collect();
long.sort_by_key(|e| e.long_term_frame_idx);
let to_ref = |e: &&DpbEntry| RefPic { frame: e.frame.clone(), long_term: e.mark == RefMark::Long };
let mut entry_lists: [Vec<&DpbEntry>; 2] = [Vec::new(), Vec::new()];
let mut entry_lists: [Vec<&DpbEntry<F>>; 2] = [Vec::new(), Vec::new()];
if sh.slice_type != SliceType::B {
let mut s = short.clone();
s.sort_by_key(|e| std::cmp::Reverse(pic_num(e)));
entry_lists[0] = s.into_iter().chain(long.iter().copied()).collect();
} else {
let mut before: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc < cur_poc).collect();
let mut after: Vec<&DpbEntry> = short.iter().copied().filter(|e| e.poc > cur_poc).collect();
let mut before: Vec<&DpbEntry<F>> = short.iter().copied().filter(|e| e.poc < cur_poc).collect();
let mut after: Vec<&DpbEntry<F>> = short.iter().copied().filter(|e| e.poc > cur_poc).collect();
before.sort_by_key(|e| std::cmp::Reverse(e.poc));
after.sort_by_key(|e| e.poc);
entry_lists[0] = before.iter().chain(after.iter()).chain(long.iter()).copied().collect();
@ -188,7 +192,7 @@ impl Dpb {
let nlists = if sh.slice_type == SliceType::B { 2 } else { 1 };
for l in 0..nlists {
let n = sh.num_ref_idx_active[l] as usize;
let mut list: Vec<Option<&DpbEntry>> = entry_lists[l].iter().copied().map(Some).collect();
let mut list: Vec<Option<&DpbEntry<F>>> = entry_lists[l].iter().copied().map(Some).collect();
list.truncate(n);
list.resize(n, None);
if !sh.ref_pic_list_mod[l].is_empty() {
@ -199,7 +203,7 @@ impl Dpb {
for m in &sh.ref_pic_list_mod[l] {
ensure!(ref_idx < n, "too many ref_pic_list_modification entries");
list.push(None); // temporarily one longer
let (target, is_long): (Option<&DpbEntry>, bool) = match m.idc {
let (target, is_long): (Option<&DpbEntry<F>>, bool) = match m.idc {
0 | 1 => {
let d = m.value as i32 + 1;
let no_wrap = if m.idc == 0 {
@ -249,7 +253,7 @@ impl Dpb {
}
// Fill missing entries (should not happen in conforming streams) with the first valid entry.
let fallback = list.iter().flatten().next().copied().or_else(|| self.entries.last());
let out: Vec<RefPic> = list.iter().filter_map(|e| e.or(fallback)).map(|e| to_ref(&e)).collect();
let out: Vec<R> = list.iter().filter_map(|e| e.or(fallback)).map(&to_ref).collect();
lists[l] = out;
}
Ok(lists)
@ -327,12 +331,12 @@ impl Dpb {
pub fn store_picture(
&mut self,
sh: &SliceHeader,
frame: FrameRef,
frame: F,
poc: i32,
max_frame_num: u32,
max_num_ref_frames: usize,
meta: Arc<OutputMeta>,
out: &mut Vec<Output>,
out: &mut Vec<Output<F>>,
) {
let mut mark = RefMark::Unused;
let mut long_idx = 0;
@ -404,7 +408,7 @@ impl Dpb {
}
}
impl Default for Dpb {
impl<F: Clone> Default for Dpb<F> {
fn default() -> Self {
Self::new()
}

2
crates/h264/src/lib.rs
View File

@ -28,7 +28,7 @@ mod cavlc;
mod cavlc_tables;
mod deblock;
mod decoder;
mod dpb;
pub mod dpb;
mod error;
mod inter;
mod intra;

21
crates/h264/src/slice.rs
View File

@ -387,6 +387,27 @@ pub struct PocState {
prev_frame_num: u32,
}
/// Is `sh` the first slice of a new primary coded picture relative to `prev` (7.4.1.2.4)?
pub fn is_new_picture(prev: &SliceHeader, sh: &SliceHeader, sps: &Sps) -> bool {
if sh.frame_num != prev.frame_num
|| sh.pps_id != prev.pps_id
|| sh.field_pic != prev.field_pic
|| sh.bottom_field != prev.bottom_field
|| (sh.nal_ref_idc == 0) != (prev.nal_ref_idc == 0)
|| sh.idr != prev.idr
|| (sh.idr && prev.idr && sh.idr_pic_id != prev.idr_pic_id)
{
return true;
}
if sps.pic_order_cnt_type == 0 && (sh.pic_order_cnt_lsb != prev.pic_order_cnt_lsb || sh.delta_pic_order_cnt_bottom != prev.delta_pic_order_cnt_bottom) {
return true;
}
if sps.pic_order_cnt_type == 1 && sh.delta_pic_order_cnt != prev.delta_pic_order_cnt {
return true;
}
false
}
/// Output of POC computation.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Poc {

8
crates/hevc/src/decoder.rs
View File

@ -25,7 +25,7 @@ struct PendingPic {
/// Header of the latest independent slice segment (for dependent slice segments).
last_sh: SliceHeader,
first_sh: SliceHeader,
rps: RefPicSet,
rps: RefPicSet<FrameRef>,
poc: i32,
output: bool,
meta: Arc<OutputMeta>,
@ -72,7 +72,7 @@ pub struct Decoder {
spss: Vec<Option<Arc<Sps>>>,
ppss: Vec<Option<Arc<Pps>>>,
nal_length_size: Option<usize>,
dpb: Dpb,
dpb: Dpb<FrameRef>,
poc_state: PocState,
pending: Option<PendingPic>,
next_id: u32,
@ -82,7 +82,7 @@ pub struct Decoder {
/// RASL pictures associated with the last IRAP are skipped (it had NoRaslOutputFlag = 1).
skip_rasl: bool,
layout_cache: Vec<(Arc<Pps>, Arc<Sps>, Arc<Layout>, Option<Arc<Vec<Vec<u8>>>>)>,
out_queue: VecDeque<Output>,
out_queue: VecDeque<Output<FrameRef>>,
shared: Arc<Shared>,
#[cfg(feature = "threads")]
pool: Option<rayon::ThreadPool>,
@ -578,7 +578,7 @@ fn output_meta(sps: &Sps, pts: i64, key: bool, draft: bool) -> OutputMeta {
}
}
fn make_picture(o: &Output) -> Picture {
fn make_picture(o: &Output<FrameRef>) -> Picture {
let (cx, cy, cw, ch) = o.meta.crop;
let (cx, cy, cw, ch) = (cx as usize, cy as usize, cw as usize, ch as usize);
let (y, u, v) = o.frame.copy_cropped((cx, cy, cw, ch));

73
crates/hevc/src/dpb.rs
View File

@ -1,8 +1,11 @@
//! Decoded picture buffer: POC derivation (8.3.1), reference picture set (8.3.2), missing
//! reference generation (8.3.3), reference picture lists (8.3.4) and output / bumping (C.5.2).
//!
//! The buffer only does bookkeeping on picture headers; what a picture is (`F`: the software
//! decoder's [`crate::picture::FrameRef`], a hardware decoder's surface) is up to the decoder, so
//! hardware decoders driven by the same headers (VA-API) share this logic and output order.
use crate::error::{Result, ensure};
use crate::picture::{FrameRef, RefPic};
use crate::slice::SliceHeader;
use std::sync::Arc;
@ -29,8 +32,8 @@ pub enum Marking {
Long,
}
pub struct Entry {
pub frame: FrameRef,
pub struct Entry<F> {
pub frame: F,
pub poc: i32,
pub marking: Marking,
pub needed_for_output: bool,
@ -39,31 +42,49 @@ pub struct Entry {
}
/// A picture leaving the DPB for output.
pub struct Output {
pub frame: FrameRef,
pub struct Output<F> {
pub frame: F,
pub poc: i32,
pub meta: Arc<OutputMeta>,
}
/// Reference picture set of the current picture (8.3.2) as DPB indices / generated frames.
#[derive(Default)]
pub struct RefPicSet {
pub st_curr_before: Vec<RefPic>,
pub st_curr_after: Vec<RefPic>,
pub lt_curr: Vec<RefPic>,
/// A reference picture of the current picture.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Ref<F> {
pub frame: F,
pub poc: i32,
pub long_term: bool,
}
#[derive(Default)]
pub struct Dpb {
pub entries: Vec<Entry>,
/// Reference picture set of the current picture (8.3.2) as DPB indices / generated frames.
pub struct RefPicSet<F> {
pub st_curr_before: Vec<Ref<F>>,
pub st_curr_after: Vec<Ref<F>>,
pub lt_curr: Vec<Ref<F>>,
}
impl<F> Default for RefPicSet<F> {
fn default() -> Self {
Self { st_curr_before: Vec::new(), st_curr_after: Vec::new(), lt_curr: Vec::new() }
}
}
pub struct Dpb<F> {
pub entries: Vec<Entry<F>>,
pub max_dec_pic_buffering: usize,
pub max_num_reorder: usize,
/// SpsMaxLatencyPictures (0 = no limit).
pub max_latency: u32,
}
impl Dpb {
fn bump(&mut self, outs: &mut Vec<Output>) -> bool {
impl<F> Default for Dpb<F> {
fn default() -> Self {
Self { entries: Vec::new(), max_dec_pic_buffering: 0, max_num_reorder: 0, max_latency: 0 }
}
}
impl<F: Clone> Dpb<F> {
fn bump(&mut self, outs: &mut Vec<Output<F>>) -> bool {
let Some(i) = (0..self.entries.len()).filter(|&i| self.entries[i].needed_for_output).min_by_key(|&i| self.entries[i].poc) else { return false };
let e = &mut self.entries[i];
e.needed_for_output = false;
@ -85,7 +106,7 @@ impl Dpb {
}
/// Output everything (end of stream or IRAP without no_output_of_prior_pics).
pub fn flush(&mut self, outs: &mut Vec<Output>) {
pub fn flush(&mut self, outs: &mut Vec<Output<F>>) {
while self.bump(outs) {}
for e in &mut self.entries {
e.marking = Marking::Unused;
@ -100,7 +121,7 @@ impl Dpb {
/// C.5.2.2: removal of pictures before decoding the current (non-IRAP-reset) picture, after the RPS
/// has been applied.
pub fn bump_before_decode(&mut self, outs: &mut Vec<Output>) {
pub fn bump_before_decode(&mut self, outs: &mut Vec<Output<F>>) {
self.remove_unused();
loop {
let full = self.entries.len() >= self.max_dec_pic_buffering.max(1);
@ -116,7 +137,7 @@ impl Dpb {
}
/// C.5.2.3: insert the current picture and apply "additional bumping".
pub fn insert(&mut self, frame: FrameRef, poc: i32, output: bool, meta: Arc<OutputMeta>, outs: &mut Vec<Output>) {
pub fn insert(&mut self, frame: F, poc: i32, output: bool, meta: Arc<OutputMeta>, outs: &mut Vec<Output<F>>) {
for e in &mut self.entries {
if e.needed_for_output {
e.latency += 1;
@ -138,8 +159,8 @@ impl Dpb {
poc: i32,
max_poc_lsb: i32,
irap_no_rasl: bool,
make_missing: &mut dyn FnMut(i32) -> FrameRef,
) -> Result<RefPicSet> {
make_missing: &mut dyn FnMut(i32) -> F,
) -> Result<RefPicSet<F>> {
let mut set = RefPicSet::default();
if sh.nal.is_irap() && irap_no_rasl {
for e in &mut self.entries {
@ -196,13 +217,13 @@ impl Dpb {
self.entries[i].marking = Marking::Long;
}
}
let mut make = |found: Option<usize>, p: i32, lt: bool, entries: &mut Vec<Entry>| -> RefPic {
let mut make = |found: Option<usize>, p: i32, lt: bool, entries: &mut Vec<Entry<F>>| -> Ref<F> {
match found {
Some(i) => RefPic { frame: entries[i].frame.clone(), poc: entries[i].poc, long_term: lt },
Some(i) => Ref { frame: entries[i].frame.clone(), poc: entries[i].poc, long_term: lt },
None => {
// 8.3.3: generate an unavailable reference picture
let frame = make_missing(p);
RefPic { frame, poc: p, long_term: lt }
Ref { frame, poc: p, long_term: lt }
}
}
};
@ -228,8 +249,8 @@ impl Dpb {
}
/// Build RefPicList0/1 (8.3.4).
pub fn build_ref_lists(sh: &SliceHeader, rps: &RefPicSet) -> Result<[Vec<RefPic>; 2]> {
let mut lists: [Vec<RefPic>; 2] = [Vec::new(), Vec::new()];
pub fn build_ref_lists<F: Clone>(sh: &SliceHeader, rps: &RefPicSet<F>) -> Result<[Vec<Ref<F>>; 2]> {
let mut lists: [Vec<Ref<F>>; 2] = [Vec::new(), Vec::new()];
if sh.is_intra() {
return Ok(lists);
}

2
crates/hevc/src/lib.rs
View File

@ -23,7 +23,7 @@
mod cabac;
mod decoder;
mod dpb;
pub mod dpb;
mod error;
mod filter;
mod inter;

7
crates/hevc/src/picture.rs
View File

@ -207,9 +207,4 @@ fn copy_line(line: &[u16], x0: i32, out: &mut [i16]) {
pub type FrameRef = Arc<Frame>;
/// An entry of RefPicList0/1 of the current slice.
#[derive(Clone)]
pub struct RefPic {
pub frame: FrameRef,
pub poc: i32,
pub long_term: bool,
}
pub type RefPic = crate::dpb::Ref<FrameRef>;

14
crates/interchange/src/fcp7.rs
View File

@ -932,6 +932,10 @@ impl Exp<'_, '_> {
let hold = c.frame_hold;
let src_in = hold.unwrap_or(c.source_in);
let src_out = if hold.is_some() { src_in + c.duration } else { c.source_out() };
let (start_f, end_f, in_f) = (frames_round(rate, c.start), frames_round(rate, c.end()), frames_round(rate, src_in));
// At normal speed (and on a freeze frame) the source span is the record span. Rounded on its own
// it can come out a frame shorter or longer than `end - start` (#340), so derive it from them.
let out_f = if hold.is_some() || c.speed.abs() == 1.0 { in_f.saturating_add(end_f.saturating_sub(start_f)) } else { frames_round(rate, src_out) };
self.w.open(tag, &[("id", xid)]);
if generator.is_none() && !is_seq {
let mid = self.master_id(base);
@ -940,12 +944,12 @@ impl Exp<'_, '_> {
self.w.text("name", &c.name);
self.w.text("enabled", bool_str(c.enabled));
let media_dur = self.p.item(base).map(|i| i.duration()).unwrap_or(Tick::ZERO).max(src_out);
self.w.text("duration", frames_round(rate, media_dur));
self.w.text("duration", frames_round(rate, media_dur).max(out_f));
write_rate(&mut self.w, rate);
self.w.text("start", frames_round(rate, c.start));
self.w.text("end", frames_round(rate, c.end()));
self.w.text("in", frames_round(rate, src_in));
self.w.text("out", frames_round(rate, src_out));
self.w.text("start", start_f);
self.w.text("end", end_f);
self.w.text("in", in_f);
self.w.text("out", out_f);
self.w.text("pproTicksIn", src_in.0);
self.w.text("pproTicksOut", src_out.0);
if let Some(g) = &generator {

44
crates/interchange/tests/fcp7.rs
View File

@ -483,3 +483,47 @@ fn a_master_clip_used_by_an_earlier_sequence_still_lands_in_its_bin() {
assert_ne!(bin, p.root.id, "the take was left loose at the top");
assert_eq!(p.root.find_bin(bin).unwrap().name, "Comedy");
}
/// `start`/`end`/`in`/`out` of every sequence clip item in an exported document.
fn clip_spans(xml: &str) -> Vec<(i64, i64, i64, i64)> {
let tag = |block: &str, name: &str| -> i64 {
let open = format!("<{name}>");
let at = block.find(&open).unwrap_or_else(|| panic!("no <{name}> in {block}")) + open.len();
block[at..].split('<').next().unwrap().trim().parse().unwrap()
};
xml.split("<clipitem").skip(1).filter(|b| b.contains("<start>")).map(|b| (tag(b, "start"), tag(b, "end"), tag(b, "in"), tag(b, "out"))).collect()
}
/// #340: a clip placed by seconds in a 23.976 sequence has sub-frame start, duration and source in.
/// Its record span and source span were rounded to frames independently, so `out - in` could come
/// out a frame shorter than `end - start` (Premiere then reads a different source range).
#[test]
fn sub_frame_clips_keep_equal_source_and_record_spans() {
const SECOND: i64 = 254_016_000_000;
let r = FrameRate::FPS_23_976;
let mut p = Project::new("P");
let a = media(&mut p, "/m/a.mov", true, true, r);
let s = sequence(&mut p, "t", r, false);
let c = clip(&mut p, s, TrackKind::Video, 0, a, 0, 24, 0);
// the issue's case first (start 0, source in 20 s, 3 s long), then starts, ins and lengths in tenths of a second
let mut cases = vec![(0, 200, 30)];
for start in [0, 1, 5, 17, 33] {
for src in [0, 3, 7, 200, 413] {
for dur in [1, 4, 30, 71, 125] {
cases.push((start, src, dur));
}
}
}
for (start, src, dur) in cases {
let (_, it) = p.sequence_mut(s).unwrap().find_item_mut(c).unwrap();
it.start = Tick(start * SECOND / 10);
it.source_in = Tick(src * SECOND / 10);
it.duration = Tick(dur * SECOND / 10);
let (bytes, _) = export(&p, s, Format::Fcp7Xml, &ExportOptions::default()).expect("export");
let xml = String::from_utf8(bytes).unwrap();
let spans = clip_spans(&xml);
assert_eq!(spans.len(), 1, "{xml}");
let (start_f, end_f, in_f, out_f) = spans[0];
assert_eq!(out_f - in_f, end_f - start_f, "start {start}, in {src}, duration {dur} (tenths of a second): {spans:?}");
}
}

6
crates/isobmff/src/codec.rs
View File

@ -1505,10 +1505,8 @@ impl SampleEntry {
CodecConfig::Aac(c) => {
b.leaf(b"esds", &esds_payload(0x40, &c.asc, c.max_bitrate, c.avg_bitrate, c.buffer_size));
}
CodecConfig::Mp3 => {
if format == *b"mp4a" {
b.leaf(b"esds", &esds_payload(0x6B, &[], 0, 0, 0));
}
CodecConfig::Mp3 if format == *b"mp4a" => {
b.leaf(b"esds", &esds_payload(0x6B, &[], 0, 0, 0));
}
CodecConfig::Alac { cookie } => {
let m = b.start_full(b"alac", 0, 0);

6
crates/matroska/src/demux.rs
View File

@ -713,10 +713,8 @@ impl Walker<'_> {
match h.id {
SEEK_HEAD => file.seek_head.extend(parse_seekhead(&data)),
INFO => file.info = parse_info(&data),
TRACKS => {
if file.tracks.is_empty() {
file.tracks = parse_tracks(&data, file.info.timestamp_scale);
}
TRACKS if file.tracks.is_empty() => {
file.tracks = parse_tracks(&data, file.info.timestamp_scale);
}
CUES => parse_cues(&data, &mut file.cues),
CHAPTERS => parse_chapters(&data, &mut file.chapters),

44
crates/media/src/cancel.rs
View File

@ -13,6 +13,10 @@
//! [`with_draft`] marks reduced-resolution playback (opt-in): decoders may take spec-safe
//! shortcuts that only change pictures nothing references (H.264: no deblocking of non-reference
//! pictures). Never set for exports, renders or a frame shown while paused.
//!
//! [`with_background`] marks work nobody waits on interactively (thumbnails of the Project panel,
//! the Media Browser and the timeline): sources should not keep expensive state, such as a
//! frame-threaded 4K decoder, for it once it is done.
use std::cell::{Cell, RefCell};
use std::sync::Arc;
@ -24,6 +28,28 @@ thread_local! {
static CURRENT: RefCell<Option<Arc<AtomicBool>>> = const { RefCell::new(None) };
static CATCH_UP: Cell<Option<Tick>> = const { Cell::new(None) };
static DRAFT: Cell<bool> = const { Cell::new(false) };
static BACKGROUND: Cell<bool> = const { Cell::new(false) };
}
/// Run `f` as background work when `on` (restoring the previous hint after): a thumbnail, not a
/// frame a monitor or playback waits for.
pub fn with_background<R>(on: bool, f: impl FnOnce() -> R) -> R {
/// Restores the previous hint however `f` ends (a panic caught further up must not leave the
/// worker thread marked as background).
struct Restore(bool);
impl Drop for Restore {
fn drop(&mut self) {
BACKGROUND.with(|c| c.set(self.0));
}
}
let _restore = Restore(BACKGROUND.with(|c| c.replace(on)));
f()
}
/// Whether the work running on this thread is background work (off unless inside
/// [`with_background`]`(true, …)`).
pub fn background() -> bool {
BACKGROUND.with(Cell::get)
}
/// Run `f` with the draft-decoding hint `on` (restoring the previous one after). Frames decoded
@ -97,6 +123,24 @@ mod tests {
assert!(!draft());
}
#[test]
fn background_is_off_by_default_and_scoped() {
assert!(!background());
with_background(true, || {
assert!(background());
with_background(false, || assert!(!background()));
assert!(background());
});
assert!(!background());
}
#[test]
fn a_panic_inside_background_work_does_not_leave_the_thread_marked() {
let r = std::panic::catch_unwind(|| with_background(true, || panic!("boom")));
assert!(r.is_err());
assert!(!background());
}
#[test]
fn catch_up_is_scoped_to_the_closure() {
assert_eq!(catch_up(), None);

10
crates/platform/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "filmcraft-platform"
description = "OS media integration: hardware video decoding through the operating system's codecs (VideoToolbox on macOS), with transparent fallback to the pure-Rust decoders"
description = "OS media integration: hardware video decoding through the operating system's codecs (VideoToolbox on macOS, Media Foundation on Windows, VA-API on Linux), with transparent fallback to the pure-Rust decoders"
version.workspace = true
edition.workspace = true
license.workspace = true
@ -21,10 +21,13 @@ type_complexity = "allow"
undocumented_unsafe_blocks = "deny"
[dependencies]
filmcraft-bitstream = { workspace = true }
filmcraft-codecs = { workspace = true }
filmcraft-color = { workspace = true }
filmcraft-export = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-h264 = { workspace = true }
filmcraft-hevc = { workspace = true }
filmcraft-isobmff = { workspace = true }
filmcraft-time = { workspace = true }
log = { workspace = true }
@ -49,6 +52,11 @@ windows = { version = "0.62", features = [
"Win32_System_LibraryLoader",
] }
# Linux: libva (VA-API) is loaded at run time with libloading (ISC; already in the lockfile through
# wgpu), so building needs no libva headers and a system without libva simply decodes in software.
[target.'cfg(target_os = "linux")'.dependencies]
libloading = "0.8"
[dev-dependencies]
filmcraft-testkit = { workspace = true }
filmcraft-media = { workspace = true }

60
crates/platform/README.md
View File

@ -8,7 +8,7 @@ It is the one crate of the workspace allowed to contain `unsafe`, under the rule
```rust
// at startup (the desktop app, filmcraft-cli, the bench)
let availability = filmcraft_platform::register(); // Available("VideoToolbox") on macOS, Available("Media Foundation") on Windows
let availability = filmcraft_platform::register(); // Available("VideoToolbox") on macOS, Available("Media Foundation") on Windows, Available("VA-API") on Linux with a driver
```
## What it does
@ -70,6 +70,37 @@ let availability = filmcraft_platform::register(); // Available("VideoToolbox")
It declines two-pass VBR, HDR, MXF, interlaced output, sizes outside NVENC's limits and systems
without an NVIDIA GPU or driver, and the software encoder runs instead. A failure during an export
ends it with an error, since a hardware stream cannot be finished in software.
- **Linux: VA-API, H.264 (`avcC`) and HEVC (`hvcC`)**: H.264 8-bit 4:2:0 progressive, Constrained
Baseline / Main / High; HEVC Main, Main 10 and Main Still Picture, 4:2:0 8- and 10-bit (`vaapi/`), on Intel (iHD, i965), AMD and other Mesa drivers. `libva.so.2` and `libva-drm.so.2`
are loaded at run time (`vaapi/va.rs`; no libva headers needed to build), and each decoder opens
its own display on the first DRM render node (`/dev/dri/renderD128`…) with a working driver.
VA-API decoding is *stateless*: the host parses the stream and keeps the decoded picture buffer,
and the GPU decodes one picture's slice data at a time into a surface. That host side
(`vaapi/h264.rs`, `vaapi/hevc.rs`, safe code) is the software decoders' own: their parameter-set
and slice-header parsers, POC computation and DPBs (`filmcraft_h264::dpb`, `filmcraft_hevc::dpb`,
generic over what a picture is: H.264 reference marking including MMCOs and reference lists with
modifications; HEVC reference picture sets and lists, RASL pictures of a CRA the run starts at
left out; output order). The two decoders therefore make
the same decisions and output the same pictures in the same order, and the parity tests are
bit-exact. Each picture is sent as one picture-parameter buffer, the scaling matrices, and a slice
parameter + slice data buffer per slice (the NAL unit as stored, emulation prevention included).
HEVC pictures are sent the same way per slice segment (scaling lists only when enabled, converted
to raster order). Pictures the DPB outputs are read back right away (`vaGetImage` into an NV12 or
P010 image, then `biplanar.rs`), **the one GPU to CPU copy**. Surfaces: the stream's DPB size plus two.
- *frame_num gaps* (and every start at a non-IDR picture: an open-GOP seek) get "non-existing"
frames like the software decoder's: copies of the latest decoded frame, mid-gray when there is
none (`vaPutImage`). Pictures predicted from them (the leading pictures of an open GOP after a
seek) are concealment in both decoders and can differ: the hardware also reads motion data the
stand-ins do not have. Everything from the seek point on is bit-exact.
- *Declined up front* (our decoder is used): VP9 and AV1 (not through VA-API yet), H.264 profiles
other than Baseline / Main / High, 10-bit H.264, HEVC range extensions and profiles other than
Main / Main 10 / Main Still Picture, 4:2:2 / 4:4:4, field / MBAFF coding, no libva, no render
node or driver, a profile or size the driver does not decode. *Mid-stream errors*
(`HybridDecoder` continues in software): FMO, SP / SI slices, data partitioning, a size or DPB
change, a missing reference picture (HEVC: one the RPS names that was never decoded), more than
15 HEVC reference frames, any driver error.
- An HEVC `flush` ends the stream as in software (references are dropped and the next CRA
leaves out its RASL pictures); the GOP cache always seeks after a flush.
- **Other systems:** `register()` does nothing and returns `Availability::Unavailable`.
- **`HybridDecoder`** (`hybrid.rs`, safe code): the hardware decoder plus the means to build our
software decoder for the same `SampleEntry` (`filmcraft_codecs::software_video_decoder`). On a
@ -114,8 +145,10 @@ built-in encoder, which changes three things from H.264:
- **Choosing the format is the opt-in.** No `hardware_encoding` setting: Export ▸ Format ▸ H.265
(HEVC), or `"format": "hevc"` in `file.exportMedia` (`h265` is accepted too). The format is
listed as available only on a machine with a hardware HEVC encoder: `register()` hands
`hardware_encode::hevc_available` (one small hardware session, created on the first question) to
`filmcraft_export::register_format_probe`.
`hardware_encode::hevc_available` (one small hardware session, about 0.1 s) to
`filmcraft_export::register_format_probe`, and asks it right away on a thread of its own
(`warm_hevc_probe`), so the first draw of the format list does not create the session on the UI
thread.
- **There is no fallback encoder.** What the hardware path does not take is an error, not a
different encoder: two-pass VBR is refused up front (`ExportSettings::validate`), HDR sequences
are exported as SDR (the H.265 path is 8-bit), and odd sizes, non-square pixels or a machine
@ -140,13 +173,14 @@ level and flags are the encoder's own. If it is missing the export stops with th
- **Never undecodable:** the factory declines (returns `None`, so the software decoder is used)
when Settings ▸ Playback ▸ Hardware decoding is Off, for formats it does not take (field-coded
H.264, bit depths other than 8 / 10, 4:4:4 or monochrome, luma / chroma depth mismatch, larger
than 8192×8192; on Windows also 4:2:2 and the profiles listed above) and when the OS cannot
create a hardware session (VideoToolbox) or a GPU-backed decoder (Media Foundation).
than 8192×8192; on Windows also 4:2:2 and the profiles listed above; on Linux everything but
8-bit 4:2:0 progressive H.264 and 4:2:0 HEVC Main / Main 10) and when the OS cannot create a hardware session (VideoToolbox), a
GPU-backed decoder (Media Foundation) or a VA-API configuration and context (Linux).
- **Interchangeable:** colour, pixel aspect, pts, presentation order, `is_random_access` and
`is_disposable` come from the software decoders' own helpers (`filmcraft_codecs::hw`,
`video::vui_color`, `sar_par`).
- **Never crash:** no `unwrap` / `expect` / `panic!` outside tests; the output callback runs under
`catch_unwind`; every `unsafe` block has a `// SAFETY:` comment; the public API is safe.
- **Never crash:** no `unwrap` / `expect` / `panic!` outside tests; the output callback (and
libva's error-message callback) runs under `catch_unwind`; every `unsafe` block has a `// SAFETY:` comment; the public API is safe.
- **Counted:** `perf.stats` `decode.hardware` (frames, software frames, sessions, declined,
fallbacks; `filmcraft_codecs::hw::hw_stats`) and `backend` (the registered backend's name,
`filmcraft_codecs::hw::hw_backend`). `export.hardware` counts the NVENC encoder's frames, sessions and declines.
@ -162,6 +196,9 @@ level and flags are the encoder's own. If it is missing the export stops with th
| `tests/nvenc.rs` (Windows, NVIDIA) | H.264 from NVENC (1280×720, 6 Mbps, 72 frames) decodes with our decoder at worst 46.9 dB luma PSNR; IDR at 0, 24 and 48; dts / pts right |
| `tests/nvenc_export.rs` (Windows, NVIDIA) | Export with hardware encoding against the software encoder through the export pipeline: the two decoded files at worst 54.8 dB luma PSNR; ffmpeg decodes the file without errors; declined cases go to the software encoder; the counters |
| `src/nvenc/abi_tests.rs` (Windows) | FFI structs' sizes, alignments, field offsets, constants and GUIDs against a C compiler's view of NVIDIA's `nvEncodeAPI.h` (12.1) |
| `tests/vaapi.rs` (Linux, VA-API) | HEVC Main (open GOP: CRA + RASL), Main 10, scaling lists, 3 slices with WPP (10-bit), weighted prediction with 4 references, transform skip + AMP + B-pyramid (10-bit, open GOP), Main / Main 10 at 1080p and 2160p (compared picture by picture, little memory); H.264 High (B-pyramid), Constrained Baseline with 3 slices per picture, Main with explicit weighted prediction (P and B) and 4 references, temporal direct, custom scaling matrices (JVT) with the 8x8 transform and 2 slices, an open-GOP stream, and 1080p / 2160p, 640×360 (coded 368: cropping): every picture **bit-exact** with our software decoder, same pts order, count, colour and aspect, also after `reset` + reseek to every later sync sample (after an open-GOP I picture, from the seek point on), a mid-stream `flush` with decoding carrying on, and a full pass after resets; forced mid-stream failures (`VaDecoder::fail_after`) at five points continue with the software decoder's exact output; interlaced and 10-bit H.264, 4:2:2 HEVC and the Off setting are declined; an HEVC mid-GOP flush is not checked (it ends the stream, as in software); seeded mutation of samples (through the hybrid and straight into the hardware decoder) and of `avcC` records never panics or hangs; 40 decoders created and dropped; concurrent decoders; a decoder moving between threads |
| `src/vaapi/tests.rs` (every OS) | The H.264 and HEVC front ends with a recording stand-in for the hardware, on x264 streams (B-pyramid, weighted prediction, 4 slices, open GOP with temporal direct) and x265 streams (open GOP with RASL, B-pyramid, Main 10 with 2 slices and weighted prediction): output order and pts are the software decoder's, also after seeks and a mid-stream flush; every buffer is consistent (current surface free, references decoded and listed once, reference lists of the active length, slice data offsets inside the slice); damaged samples are errors |
| `src/vaapi/abi_tests.rs` | FFI structs' sizes, alignments, field offsets, constants and bit-field positions against a C compiler's view of libva's `va.h` and `va_dec_hevc.h` (VA-API 1.23) |
Fixtures are made with ffmpeg into `target/fixtures/platform/` (generator only, never linked);
tests skip without ffmpeg or without a hardware decoder.
@ -185,9 +222,14 @@ M4 Pro, load 150–190 (`cargo xtask bench --hw off|auto`): CPU per decoded fram
playback with no dropped frames at Full, 1/2 and 1/4. Details in
[docs/performance.md](../../docs/performance.md).
Hardware decoding, Linux (Intel Iris Xe, iHD driver; `cargo xtask bench --hw off|auto`): CPU per
decoded frame H.264 1080p 104 → 2.3 ms, 2160p 417 → 11.4 ms; HEVC 1080p 46 → 2.3 ms, 2160p
203 → 10.1 ms, Main 10 2160p 212 → 18.4 ms.
Details in [docs/performance.md](../../docs/performance.md).
## Not yet
Zero-copy upload of decoded pictures into wgpu textures (`CVPixelBuffer`s on macOS, Direct3D 11
textures on Windows); B-frames and 10-bit / HDR HEVC (Main 10) in hardware encoding; VA-API (Linux)
decoders; field-coded H.264; HEVC, 10-bit and HDR encoding with NVENC, and encoders from other
textures on Windows); B-frames and 10-bit / HDR HEVC (Main 10) in hardware encoding; HEVC, VP9 and
VP9 and AV1 through VA-API (Linux), and VA-API encoding; field-coded H.264; HEVC, 10-bit and HDR encoding with NVENC, and encoders from other
vendors on Windows (through Media Foundation); VP9 / AV1 4:4:4 and 12-bit on Windows.

33
crates/platform/src/hardware_encode.rs
View File

@ -13,7 +13,7 @@
//! configurations are declined, and a declined export fails with "encoder not available" instead of
//! falling back; [`hevc_available`] tells the format list whether this machine can do it at all.
use std::sync::OnceLock;
use std::sync::{Once, OnceLock};
use filmcraft_export::{BitrateMode, EncodedPacket, EncoderFrame, ExportError, ExportSettings, Format, H264Pass, H264Profile, HardwareEncoding, VideoEncoder};
use filmcraft_isobmff::{AvcConfig, HevcConfig, SampleEntry};
@ -66,11 +66,13 @@ pub fn config_for(w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Result
})
}
/// Whether this machine has a hardware HEVC encoder (asked once, by creating a small session): what
/// makes `Format::Hevc` available.
static HEVC_PROBE: OnceLock<bool> = OnceLock::new();
/// Whether this machine has a hardware HEVC encoder (asked once, by creating a small session, which
/// takes about 0.1 s): what makes `Format::Hevc` available. [`warm_hevc_probe`] asks it ahead of the
/// first caller.
pub fn hevc_available() -> bool {
static AVAILABLE: OnceLock<bool> = OnceLock::new();
*AVAILABLE.get_or_init(|| {
*HEVC_PROBE.get_or_init(|| {
let probe = VtConfig {
width: 640,
height: 360,
@ -88,6 +90,27 @@ pub fn hevc_available() -> bool {
})
}
/// Whether [`hevc_available`] has its answer yet (diagnostics and tests).
pub fn hevc_probed() -> bool {
HEVC_PROBE.get().is_some()
}
/// Ask [`hevc_available`] now, on a thread of its own, so that the first draw of the format list does
/// not spend its 0.1 s creating a hardware session on the UI thread. Whoever asks while the thread
/// runs waits for the same answer (no longer than asking first); if the thread cannot start, or the
/// probe panics, the first caller asks, as before. Only the first call starts a thread.
pub fn warm_hevc_probe() {
static STARTED: Once = Once::new();
STARTED.call_once(|| {
let spawned = std::thread::Builder::new().name("filmcraft-hevc-probe".into()).spawn(|| {
let _ = std::panic::catch_unwind(hevc_available);
});
if let Err(why) = spawned {
log::info!("HEVC probe not started ahead of time: {why}");
}
});
}
/// The encoder factory `register` puts in front of the built-in ones: a hardware encoder when the
/// export asks for one (H.264 with `hardware_encoding`; HEVC always) and the OS takes the
/// configuration, `None` (the built-in encoder, if there is one) otherwise.

64
crates/platform/src/lib.rs
View File

@ -3,8 +3,9 @@
//!
//! [`register`] puts the platform's hardware decoder factory in front of FilmCraft's own decoders
//! (`filmcraft_codecs::register_video_decoder`). Today that is VideoToolbox on macOS for H.264
//! (`avcC`) and HEVC (`hvcC`) streams, 8- and 10-bit, 4:2:0 and 4:2:2; on other systems
//! registration does nothing and reports [`Availability::Unavailable`]. It also registers a
//! (`avcC`) and HEVC (`hvcC`) streams, 8- and 10-bit, 4:2:0 and 4:2:2; Media Foundation / DXVA on
//! Windows; VA-API on Linux for H.264 ([`vaapi`]); on other systems (and on Linux without a VA-API
//! driver) registration does nothing and reports [`Availability::Unavailable`]. It also registers a
//! hardware H.264 encoder factory (`filmcraft_export::register_encoder`) that only acts when an
//! export asks for it (`ExportSettings::hardware_encoding` = `Auto`), see [`hardware_encode`].
//!
@ -25,10 +26,11 @@
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
// Used by the Windows decoder only; compiled everywhere so their tests run on every system.
// Used by the Windows (and, for `biplanar`, Linux) decoders; compiled everywhere so their tests run
// on every system.
#[cfg_attr(not(target_os = "windows"), allow(dead_code))]
mod annexb;
#[cfg_attr(not(target_os = "windows"), allow(dead_code))]
#[cfg_attr(not(any(target_os = "windows", target_os = "linux")), allow(dead_code))]
mod biplanar;
#[cfg(target_os = "macos")]
pub mod hardware_encode;
@ -37,6 +39,8 @@ pub mod hybrid;
pub mod media_foundation;
#[cfg(target_os = "windows")]
pub mod nvenc;
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub mod vaapi;
#[cfg(target_os = "macos")]
#[allow(unsafe_code)]
pub mod videotoolbox;
@ -64,6 +68,9 @@ pub fn register() -> Availability {
filmcraft_codecs::register_video_decoder(videotoolbox_factory);
filmcraft_export::register_encoder(hardware_encode::videotoolbox_encoder_factory);
filmcraft_export::register_format_probe(filmcraft_export::Format::Hevc, hardware_encode::hevc_available);
// the probe creates a hardware session (about 0.1 s): answer it now, off the UI thread,
// before the first draw of the format list asks
hardware_encode::warm_hevc_probe();
filmcraft_codecs::hw::set_hw_backend("VideoToolbox");
Availability::Available("VideoToolbox")
}
@ -77,7 +84,23 @@ pub fn register() -> Availability {
filmcraft_codecs::hw::set_hw_backend("Media Foundation");
Availability::Available("Media Foundation")
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
#[cfg(target_os = "linux")]
{
match vaapi::va::probe() {
Ok(driver) => {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| log::info!("hardware decoding through VA-API: {driver}"));
filmcraft_codecs::register_video_decoder(vaapi_factory);
filmcraft_codecs::hw::set_hw_backend("VA-API");
Availability::Available("VA-API")
}
Err(why) => {
log::info!("no VA-API hardware decoding: {why}");
Availability::Unavailable("no VA-API driver (libva and a DRM render node)")
}
}
}
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
{
Availability::Unavailable("no hardware video decoder for this system yet")
}
@ -93,7 +116,11 @@ pub fn registered() -> bool {
{
filmcraft_codecs::video_decoder_registered(media_foundation_factory)
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
#[cfg(target_os = "linux")]
{
filmcraft_codecs::video_decoder_registered(vaapi_factory)
}
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
{
false
}
@ -110,7 +137,11 @@ pub fn hardware_decoder_for(entry: &filmcraft_isobmff::SampleEntry) -> bool {
{
media_foundation::stream_info(entry).is_some_and(|info| media_foundation::MfDecoder::new(info).is_ok())
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
#[cfg(target_os = "linux")]
{
filmcraft_codecs::hw::NalStreamInfo::from_entry(entry).and_then(|r| r.ok()).is_some_and(|info| vaapi::VaDecoder::new(info).is_ok())
}
#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
{
let _ = entry;
false
@ -153,3 +184,22 @@ pub fn media_foundation_factory(entry: &filmcraft_isobmff::SampleEntry) -> Optio
}
}
}
/// The VA-API factory: a [`HybridDecoder`] around [`vaapi::VaDecoder`] for H.264 and HEVC
/// streams this system's VA-API driver decodes, `None` otherwise (other codecs keep the software
/// decoders).
#[cfg(target_os = "linux")]
pub fn vaapi_factory(entry: &filmcraft_isobmff::SampleEntry) -> Option<filmcraft_codecs::Result<Box<dyn filmcraft_codecs::VideoDecoder>>> {
if !filmcraft_codecs::hw::hardware_decoding() {
return None;
}
let info = filmcraft_codecs::hw::NalStreamInfo::from_entry(entry)?.ok()?;
match vaapi::VaDecoder::new(info.clone()) {
Ok(va) => Some(Ok(Box::new(HybridDecoder::new(Box::new(va), entry.clone(), info)))),
Err(why) => {
log::info!("hardware decoding declined for {} video: {why}", entry.codec.name());
filmcraft_codecs::hw::note_hw_declined();
None
}
}
}

365
crates/platform/src/vaapi/abi_tests.rs Normal file
View File

@ -0,0 +1,365 @@
//! The VA-API FFI structures against the C compiler's view of libva's `va.h` and `va_dec_hevc.h` (VA-API 1.23, libva
//! 2.23): sizes, alignments, field offsets, constants and bit-field positions. Generated by running a
//! C program (built with GCC against the libva-dev headers, x86-64 Linux) that prints them; the
//! header itself is not in this repository. To regenerate, compile a program printing `sizeof` /
//! `offsetof` of the structures of `src/vaapi/ffi.rs`, the constants it uses, and the `value` word of
//! each `seq_fields` / `pic_fields` bit-field set alone, against libva's MIT-licensed headers.
#![allow(clippy::unreadable_literal)]
use super::ffi::*;
use std::mem::{align_of, offset_of, size_of};
#[test]
fn layouts_match_the_header() {
assert_eq!((size_of::<VAConfigAttrib>(), align_of::<VAConfigAttrib>()), (8, 4), "VAConfigAttrib");
assert_eq!(offset_of!(VAConfigAttrib, type_), 0, "VAConfigAttrib.type");
assert_eq!(offset_of!(VAConfigAttrib, value), 4, "VAConfigAttrib.value");
assert_eq!((size_of::<VAPictureH264>(), align_of::<VAPictureH264>()), (36, 4), "VAPictureH264");
assert_eq!(offset_of!(VAPictureH264, picture_id), 0, "VAPictureH264.picture_id");
assert_eq!(offset_of!(VAPictureH264, frame_idx), 4, "VAPictureH264.frame_idx");
assert_eq!(offset_of!(VAPictureH264, flags), 8, "VAPictureH264.flags");
assert_eq!(offset_of!(VAPictureH264, TopFieldOrderCnt), 12, "VAPictureH264.TopFieldOrderCnt");
assert_eq!(offset_of!(VAPictureH264, BottomFieldOrderCnt), 16, "VAPictureH264.BottomFieldOrderCnt");
assert_eq!(offset_of!(VAPictureH264, va_reserved), 20, "VAPictureH264.va_reserved");
assert_eq!((size_of::<VAPictureParameterBufferH264>(), align_of::<VAPictureParameterBufferH264>()), (672, 4), "VAPictureParameterBufferH264");
assert_eq!(offset_of!(VAPictureParameterBufferH264, CurrPic), 0, "VAPictureParameterBufferH264.CurrPic");
assert_eq!(offset_of!(VAPictureParameterBufferH264, ReferenceFrames), 36, "VAPictureParameterBufferH264.ReferenceFrames");
assert_eq!(offset_of!(VAPictureParameterBufferH264, picture_width_in_mbs_minus1), 612, "VAPictureParameterBufferH264.picture_width_in_mbs_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferH264, picture_height_in_mbs_minus1), 614, "VAPictureParameterBufferH264.picture_height_in_mbs_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferH264, bit_depth_luma_minus8), 616, "VAPictureParameterBufferH264.bit_depth_luma_minus8");
assert_eq!(offset_of!(VAPictureParameterBufferH264, bit_depth_chroma_minus8), 617, "VAPictureParameterBufferH264.bit_depth_chroma_minus8");
assert_eq!(offset_of!(VAPictureParameterBufferH264, num_ref_frames), 618, "VAPictureParameterBufferH264.num_ref_frames");
assert_eq!(offset_of!(VAPictureParameterBufferH264, seq_fields), 620, "VAPictureParameterBufferH264.seq_fields");
assert_eq!(offset_of!(VAPictureParameterBufferH264, num_slice_groups_minus1), 624, "VAPictureParameterBufferH264.num_slice_groups_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferH264, slice_group_map_type), 625, "VAPictureParameterBufferH264.slice_group_map_type");
assert_eq!(offset_of!(VAPictureParameterBufferH264, slice_group_change_rate_minus1), 626, "VAPictureParameterBufferH264.slice_group_change_rate_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferH264, pic_init_qp_minus26), 628, "VAPictureParameterBufferH264.pic_init_qp_minus26");
assert_eq!(offset_of!(VAPictureParameterBufferH264, pic_init_qs_minus26), 629, "VAPictureParameterBufferH264.pic_init_qs_minus26");
assert_eq!(offset_of!(VAPictureParameterBufferH264, chroma_qp_index_offset), 630, "VAPictureParameterBufferH264.chroma_qp_index_offset");
assert_eq!(offset_of!(VAPictureParameterBufferH264, second_chroma_qp_index_offset), 631, "VAPictureParameterBufferH264.second_chroma_qp_index_offset");
assert_eq!(offset_of!(VAPictureParameterBufferH264, pic_fields), 632, "VAPictureParameterBufferH264.pic_fields");
assert_eq!(offset_of!(VAPictureParameterBufferH264, frame_num), 636, "VAPictureParameterBufferH264.frame_num");
assert_eq!(offset_of!(VAPictureParameterBufferH264, va_reserved), 640, "VAPictureParameterBufferH264.va_reserved");
assert_eq!((size_of::<VAIQMatrixBufferH264>(), align_of::<VAIQMatrixBufferH264>()), (240, 4), "VAIQMatrixBufferH264");
assert_eq!(offset_of!(VAIQMatrixBufferH264, ScalingList4x4), 0, "VAIQMatrixBufferH264.ScalingList4x4");
assert_eq!(offset_of!(VAIQMatrixBufferH264, ScalingList8x8), 96, "VAIQMatrixBufferH264.ScalingList8x8");
assert_eq!(offset_of!(VAIQMatrixBufferH264, va_reserved), 224, "VAIQMatrixBufferH264.va_reserved");
assert_eq!((size_of::<VASliceParameterBufferH264>(), align_of::<VASliceParameterBufferH264>()), (3128, 4), "VASliceParameterBufferH264");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_size), 0, "VASliceParameterBufferH264.slice_data_size");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_offset), 4, "VASliceParameterBufferH264.slice_data_offset");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_flag), 8, "VASliceParameterBufferH264.slice_data_flag");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_data_bit_offset), 12, "VASliceParameterBufferH264.slice_data_bit_offset");
assert_eq!(offset_of!(VASliceParameterBufferH264, first_mb_in_slice), 14, "VASliceParameterBufferH264.first_mb_in_slice");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_type), 16, "VASliceParameterBufferH264.slice_type");
assert_eq!(offset_of!(VASliceParameterBufferH264, direct_spatial_mv_pred_flag), 17, "VASliceParameterBufferH264.direct_spatial_mv_pred_flag");
assert_eq!(offset_of!(VASliceParameterBufferH264, num_ref_idx_l0_active_minus1), 18, "VASliceParameterBufferH264.num_ref_idx_l0_active_minus1");
assert_eq!(offset_of!(VASliceParameterBufferH264, num_ref_idx_l1_active_minus1), 19, "VASliceParameterBufferH264.num_ref_idx_l1_active_minus1");
assert_eq!(offset_of!(VASliceParameterBufferH264, cabac_init_idc), 20, "VASliceParameterBufferH264.cabac_init_idc");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_qp_delta), 21, "VASliceParameterBufferH264.slice_qp_delta");
assert_eq!(offset_of!(VASliceParameterBufferH264, disable_deblocking_filter_idc), 22, "VASliceParameterBufferH264.disable_deblocking_filter_idc");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_alpha_c0_offset_div2), 23, "VASliceParameterBufferH264.slice_alpha_c0_offset_div2");
assert_eq!(offset_of!(VASliceParameterBufferH264, slice_beta_offset_div2), 24, "VASliceParameterBufferH264.slice_beta_offset_div2");
assert_eq!(offset_of!(VASliceParameterBufferH264, RefPicList0), 28, "VASliceParameterBufferH264.RefPicList0");
assert_eq!(offset_of!(VASliceParameterBufferH264, RefPicList1), 1180, "VASliceParameterBufferH264.RefPicList1");
assert_eq!(offset_of!(VASliceParameterBufferH264, luma_log2_weight_denom), 2332, "VASliceParameterBufferH264.luma_log2_weight_denom");
assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_log2_weight_denom), 2333, "VASliceParameterBufferH264.chroma_log2_weight_denom");
assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l0_flag), 2334, "VASliceParameterBufferH264.luma_weight_l0_flag");
assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l0), 2336, "VASliceParameterBufferH264.luma_weight_l0");
assert_eq!(offset_of!(VASliceParameterBufferH264, luma_offset_l0), 2400, "VASliceParameterBufferH264.luma_offset_l0");
assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l0_flag), 2464, "VASliceParameterBufferH264.chroma_weight_l0_flag");
assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l0), 2466, "VASliceParameterBufferH264.chroma_weight_l0");
assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_offset_l0), 2594, "VASliceParameterBufferH264.chroma_offset_l0");
assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l1_flag), 2722, "VASliceParameterBufferH264.luma_weight_l1_flag");
assert_eq!(offset_of!(VASliceParameterBufferH264, luma_weight_l1), 2724, "VASliceParameterBufferH264.luma_weight_l1");
assert_eq!(offset_of!(VASliceParameterBufferH264, luma_offset_l1), 2788, "VASliceParameterBufferH264.luma_offset_l1");
assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l1_flag), 2852, "VASliceParameterBufferH264.chroma_weight_l1_flag");
assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_weight_l1), 2854, "VASliceParameterBufferH264.chroma_weight_l1");
assert_eq!(offset_of!(VASliceParameterBufferH264, chroma_offset_l1), 2982, "VASliceParameterBufferH264.chroma_offset_l1");
assert_eq!(offset_of!(VASliceParameterBufferH264, va_reserved), 3112, "VASliceParameterBufferH264.va_reserved");
assert_eq!((size_of::<VAImageFormat>(), align_of::<VAImageFormat>()), (48, 4), "VAImageFormat");
assert_eq!(offset_of!(VAImageFormat, fourcc), 0, "VAImageFormat.fourcc");
assert_eq!(offset_of!(VAImageFormat, byte_order), 4, "VAImageFormat.byte_order");
assert_eq!(offset_of!(VAImageFormat, bits_per_pixel), 8, "VAImageFormat.bits_per_pixel");
assert_eq!(offset_of!(VAImageFormat, depth), 12, "VAImageFormat.depth");
assert_eq!(offset_of!(VAImageFormat, red_mask), 16, "VAImageFormat.red_mask");
assert_eq!(offset_of!(VAImageFormat, green_mask), 20, "VAImageFormat.green_mask");
assert_eq!(offset_of!(VAImageFormat, blue_mask), 24, "VAImageFormat.blue_mask");
assert_eq!(offset_of!(VAImageFormat, alpha_mask), 28, "VAImageFormat.alpha_mask");
assert_eq!(offset_of!(VAImageFormat, va_reserved), 32, "VAImageFormat.va_reserved");
assert_eq!((size_of::<VAImage>(), align_of::<VAImage>()), (120, 4), "VAImage");
assert_eq!(offset_of!(VAImage, image_id), 0, "VAImage.image_id");
assert_eq!(offset_of!(VAImage, format), 4, "VAImage.format");
assert_eq!(offset_of!(VAImage, buf), 52, "VAImage.buf");
assert_eq!(offset_of!(VAImage, width), 56, "VAImage.width");
assert_eq!(offset_of!(VAImage, height), 58, "VAImage.height");
assert_eq!(offset_of!(VAImage, data_size), 60, "VAImage.data_size");
assert_eq!(offset_of!(VAImage, num_planes), 64, "VAImage.num_planes");
assert_eq!(offset_of!(VAImage, pitches), 68, "VAImage.pitches");
assert_eq!(offset_of!(VAImage, offsets), 80, "VAImage.offsets");
assert_eq!(offset_of!(VAImage, num_palette_entries), 92, "VAImage.num_palette_entries");
assert_eq!(offset_of!(VAImage, entry_bytes), 96, "VAImage.entry_bytes");
assert_eq!(offset_of!(VAImage, component_order), 100, "VAImage.component_order");
assert_eq!(offset_of!(VAImage, va_reserved), 104, "VAImage.va_reserved");
assert_eq!((size_of::<VAPictureHEVC>(), align_of::<VAPictureHEVC>()), (28, 4), "VAPictureHEVC");
assert_eq!(offset_of!(VAPictureHEVC, picture_id), 0, "VAPictureHEVC.picture_id");
assert_eq!(offset_of!(VAPictureHEVC, pic_order_cnt), 4, "VAPictureHEVC.pic_order_cnt");
assert_eq!(offset_of!(VAPictureHEVC, flags), 8, "VAPictureHEVC.flags");
assert_eq!(offset_of!(VAPictureHEVC, va_reserved), 12, "VAPictureHEVC.va_reserved");
assert_eq!((size_of::<VAPictureParameterBufferHEVC>(), align_of::<VAPictureParameterBufferHEVC>()), (604, 4), "VAPictureParameterBufferHEVC");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, CurrPic), 0, "VAPictureParameterBufferHEVC.CurrPic");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, ReferenceFrames), 28, "VAPictureParameterBufferHEVC.ReferenceFrames");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pic_width_in_luma_samples), 448, "VAPictureParameterBufferHEVC.pic_width_in_luma_samples");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pic_height_in_luma_samples), 450, "VAPictureParameterBufferHEVC.pic_height_in_luma_samples");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pic_fields), 452, "VAPictureParameterBufferHEVC.pic_fields");
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, sps_max_dec_pic_buffering_minus1),
456,
"VAPictureParameterBufferHEVC.sps_max_dec_pic_buffering_minus1"
);
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, bit_depth_luma_minus8), 457, "VAPictureParameterBufferHEVC.bit_depth_luma_minus8");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, bit_depth_chroma_minus8), 458, "VAPictureParameterBufferHEVC.bit_depth_chroma_minus8");
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, pcm_sample_bit_depth_luma_minus1),
459,
"VAPictureParameterBufferHEVC.pcm_sample_bit_depth_luma_minus1"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, pcm_sample_bit_depth_chroma_minus1),
460,
"VAPictureParameterBufferHEVC.pcm_sample_bit_depth_chroma_minus1"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_min_luma_coding_block_size_minus3),
461,
"VAPictureParameterBufferHEVC.log2_min_luma_coding_block_size_minus3"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_diff_max_min_luma_coding_block_size),
462,
"VAPictureParameterBufferHEVC.log2_diff_max_min_luma_coding_block_size"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_min_transform_block_size_minus2),
463,
"VAPictureParameterBufferHEVC.log2_min_transform_block_size_minus2"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_diff_max_min_transform_block_size),
464,
"VAPictureParameterBufferHEVC.log2_diff_max_min_transform_block_size"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_min_pcm_luma_coding_block_size_minus3),
465,
"VAPictureParameterBufferHEVC.log2_min_pcm_luma_coding_block_size_minus3"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_diff_max_min_pcm_luma_coding_block_size),
466,
"VAPictureParameterBufferHEVC.log2_diff_max_min_pcm_luma_coding_block_size"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, max_transform_hierarchy_depth_intra),
467,
"VAPictureParameterBufferHEVC.max_transform_hierarchy_depth_intra"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, max_transform_hierarchy_depth_inter),
468,
"VAPictureParameterBufferHEVC.max_transform_hierarchy_depth_inter"
);
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, init_qp_minus26), 469, "VAPictureParameterBufferHEVC.init_qp_minus26");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, diff_cu_qp_delta_depth), 470, "VAPictureParameterBufferHEVC.diff_cu_qp_delta_depth");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_cb_qp_offset), 471, "VAPictureParameterBufferHEVC.pps_cb_qp_offset");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_cr_qp_offset), 472, "VAPictureParameterBufferHEVC.pps_cr_qp_offset");
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_parallel_merge_level_minus2),
473,
"VAPictureParameterBufferHEVC.log2_parallel_merge_level_minus2"
);
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_tile_columns_minus1), 474, "VAPictureParameterBufferHEVC.num_tile_columns_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_tile_rows_minus1), 475, "VAPictureParameterBufferHEVC.num_tile_rows_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, column_width_minus1), 476, "VAPictureParameterBufferHEVC.column_width_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, row_height_minus1), 514, "VAPictureParameterBufferHEVC.row_height_minus1");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, slice_parsing_fields), 556, "VAPictureParameterBufferHEVC.slice_parsing_fields");
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, log2_max_pic_order_cnt_lsb_minus4),
560,
"VAPictureParameterBufferHEVC.log2_max_pic_order_cnt_lsb_minus4"
);
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_short_term_ref_pic_sets), 561, "VAPictureParameterBufferHEVC.num_short_term_ref_pic_sets");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_long_term_ref_pic_sps), 562, "VAPictureParameterBufferHEVC.num_long_term_ref_pic_sps");
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, num_ref_idx_l0_default_active_minus1),
563,
"VAPictureParameterBufferHEVC.num_ref_idx_l0_default_active_minus1"
);
assert_eq!(
offset_of!(VAPictureParameterBufferHEVC, num_ref_idx_l1_default_active_minus1),
564,
"VAPictureParameterBufferHEVC.num_ref_idx_l1_default_active_minus1"
);
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_beta_offset_div2), 565, "VAPictureParameterBufferHEVC.pps_beta_offset_div2");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, pps_tc_offset_div2), 566, "VAPictureParameterBufferHEVC.pps_tc_offset_div2");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, num_extra_slice_header_bits), 567, "VAPictureParameterBufferHEVC.num_extra_slice_header_bits");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, st_rps_bits), 568, "VAPictureParameterBufferHEVC.st_rps_bits");
assert_eq!(offset_of!(VAPictureParameterBufferHEVC, va_reserved), 572, "VAPictureParameterBufferHEVC.va_reserved");
assert_eq!((size_of::<VASliceParameterBufferHEVC>(), align_of::<VASliceParameterBufferHEVC>()), (264, 4), "VASliceParameterBufferHEVC");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_size), 0, "VASliceParameterBufferHEVC.slice_data_size");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_offset), 4, "VASliceParameterBufferHEVC.slice_data_offset");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_flag), 8, "VASliceParameterBufferHEVC.slice_data_flag");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_byte_offset), 12, "VASliceParameterBufferHEVC.slice_data_byte_offset");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_segment_address), 16, "VASliceParameterBufferHEVC.slice_segment_address");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, RefPicList), 20, "VASliceParameterBufferHEVC.RefPicList");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, LongSliceFlags), 52, "VASliceParameterBufferHEVC.LongSliceFlags");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, collocated_ref_idx), 56, "VASliceParameterBufferHEVC.collocated_ref_idx");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, num_ref_idx_l0_active_minus1), 57, "VASliceParameterBufferHEVC.num_ref_idx_l0_active_minus1");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, num_ref_idx_l1_active_minus1), 58, "VASliceParameterBufferHEVC.num_ref_idx_l1_active_minus1");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_qp_delta), 59, "VASliceParameterBufferHEVC.slice_qp_delta");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_cb_qp_offset), 60, "VASliceParameterBufferHEVC.slice_cb_qp_offset");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_cr_qp_offset), 61, "VASliceParameterBufferHEVC.slice_cr_qp_offset");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_beta_offset_div2), 62, "VASliceParameterBufferHEVC.slice_beta_offset_div2");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_tc_offset_div2), 63, "VASliceParameterBufferHEVC.slice_tc_offset_div2");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, luma_log2_weight_denom), 64, "VASliceParameterBufferHEVC.luma_log2_weight_denom");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_chroma_log2_weight_denom), 65, "VASliceParameterBufferHEVC.delta_chroma_log2_weight_denom");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_luma_weight_l0), 66, "VASliceParameterBufferHEVC.delta_luma_weight_l0");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, luma_offset_l0), 81, "VASliceParameterBufferHEVC.luma_offset_l0");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_chroma_weight_l0), 96, "VASliceParameterBufferHEVC.delta_chroma_weight_l0");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, ChromaOffsetL0), 126, "VASliceParameterBufferHEVC.ChromaOffsetL0");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_luma_weight_l1), 156, "VASliceParameterBufferHEVC.delta_luma_weight_l1");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, luma_offset_l1), 171, "VASliceParameterBufferHEVC.luma_offset_l1");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, delta_chroma_weight_l1), 186, "VASliceParameterBufferHEVC.delta_chroma_weight_l1");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, ChromaOffsetL1), 216, "VASliceParameterBufferHEVC.ChromaOffsetL1");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, five_minus_max_num_merge_cand), 246, "VASliceParameterBufferHEVC.five_minus_max_num_merge_cand");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, num_entry_point_offsets), 248, "VASliceParameterBufferHEVC.num_entry_point_offsets");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, entry_offset_to_subset_array), 250, "VASliceParameterBufferHEVC.entry_offset_to_subset_array");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, slice_data_num_emu_prevn_bytes), 252, "VASliceParameterBufferHEVC.slice_data_num_emu_prevn_bytes");
assert_eq!(offset_of!(VASliceParameterBufferHEVC, va_reserved), 256, "VASliceParameterBufferHEVC.va_reserved");
assert_eq!((size_of::<VAIQMatrixBufferHEVC>(), align_of::<VAIQMatrixBufferHEVC>()), (1016, 4), "VAIQMatrixBufferHEVC");
assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList4x4), 0, "VAIQMatrixBufferHEVC.ScalingList4x4");
assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList8x8), 96, "VAIQMatrixBufferHEVC.ScalingList8x8");
assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList16x16), 480, "VAIQMatrixBufferHEVC.ScalingList16x16");
assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingList32x32), 864, "VAIQMatrixBufferHEVC.ScalingList32x32");
assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingListDC16x16), 992, "VAIQMatrixBufferHEVC.ScalingListDC16x16");
assert_eq!(offset_of!(VAIQMatrixBufferHEVC, ScalingListDC32x32), 998, "VAIQMatrixBufferHEVC.ScalingListDC32x32");
assert_eq!(offset_of!(VAIQMatrixBufferHEVC, va_reserved), 1000, "VAIQMatrixBufferHEVC.va_reserved");
}
#[test]
fn constants_match_the_header() {
assert_eq!(VA_STATUS_SUCCESS as i64, 0, "VA_STATUS_SUCCESS");
assert_eq!(VA_STATUS_ERROR_UNSUPPORTED_PROFILE as i64, 12, "VA_STATUS_ERROR_UNSUPPORTED_PROFILE");
assert_eq!(VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT as i64, 13, "VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT");
assert_eq!(VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT as i64, 14, "VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT");
assert_eq!(VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED as i64, 19, "VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED");
assert_eq!(VA_STATUS_ERROR_DECODING_ERROR as i64, 23, "VA_STATUS_ERROR_DECODING_ERROR");
assert_eq!(VA_INVALID_ID as i64, 4294967295, "VA_INVALID_ID");
assert_eq!(VA_INVALID_SURFACE as i64, 4294967295, "VA_INVALID_SURFACE");
assert_eq!(VAProfileH264Main as i64, 6, "VAProfileH264Main");
assert_eq!(VAProfileH264High as i64, 7, "VAProfileH264High");
assert_eq!(VAProfileH264ConstrainedBaseline as i64, 13, "VAProfileH264ConstrainedBaseline");
assert_eq!(VAEntrypointVLD as i64, 1, "VAEntrypointVLD");
assert_eq!(VAConfigAttribRTFormat as i64, 0, "VAConfigAttribRTFormat");
assert_eq!(VA_RT_FORMAT_YUV420 as i64, 1, "VA_RT_FORMAT_YUV420");
assert_eq!(VA_PROGRESSIVE as i64, 1, "VA_PROGRESSIVE");
assert_eq!(VAPictureParameterBufferType as i64, 0, "VAPictureParameterBufferType");
assert_eq!(VAIQMatrixBufferType as i64, 1, "VAIQMatrixBufferType");
assert_eq!(VASliceParameterBufferType as i64, 4, "VASliceParameterBufferType");
assert_eq!(VASliceDataBufferType as i64, 5, "VASliceDataBufferType");
assert_eq!(VA_SLICE_DATA_FLAG_ALL as i64, 0, "VA_SLICE_DATA_FLAG_ALL");
assert_eq!(VA_PICTURE_H264_INVALID as i64, 1, "VA_PICTURE_H264_INVALID");
assert_eq!(VA_PICTURE_H264_SHORT_TERM_REFERENCE as i64, 8, "VA_PICTURE_H264_SHORT_TERM_REFERENCE");
assert_eq!(VA_PICTURE_H264_LONG_TERM_REFERENCE as i64, 16, "VA_PICTURE_H264_LONG_TERM_REFERENCE");
assert_eq!(VA_FOURCC_NV12 as i64, 842094158, "VA_FOURCC_NV12");
assert_eq!(VA_FOURCC_P010 as i64, 808530000, "VA_FOURCC_P010");
assert_eq!(VAProfileHEVCMain as i64, 17, "VAProfileHEVCMain");
assert_eq!(VAProfileHEVCMain10 as i64, 18, "VAProfileHEVCMain10");
assert_eq!(VA_RT_FORMAT_YUV420_10 as i64, 256, "VA_RT_FORMAT_YUV420_10");
assert_eq!(VA_PICTURE_HEVC_INVALID as i64, 1, "VA_PICTURE_HEVC_INVALID");
assert_eq!(VA_PICTURE_HEVC_LONG_TERM_REFERENCE as i64, 8, "VA_PICTURE_HEVC_LONG_TERM_REFERENCE");
assert_eq!(VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE as i64, 16, "VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE");
assert_eq!(VA_PICTURE_HEVC_RPS_ST_CURR_AFTER as i64, 32, "VA_PICTURE_HEVC_RPS_ST_CURR_AFTER");
assert_eq!(VA_PICTURE_HEVC_RPS_LT_CURR as i64, 64, "VA_PICTURE_HEVC_RPS_LT_CURR");
assert_eq!(VA_PADDING_LOW as i64, 4, "VA_PADDING_LOW");
assert_eq!(VA_PADDING_MEDIUM as i64, 8, "VA_PADDING_MEDIUM");
}
/// Each bit-field set alone to its largest value: the word the C compiler makes of it.
#[test]
fn bit_fields_match_the_header() {
assert_eq!(3u32 << seq_bits::CHROMA_FORMAT_IDC, 0x3, "seq_fields.chroma_format_idc");
assert_eq!(1u32 << seq_bits::RESIDUAL_COLOUR_TRANSFORM, 0x4, "seq_fields.residual_colour_transform_flag");
assert_eq!(1u32 << seq_bits::GAPS_IN_FRAME_NUM_ALLOWED, 0x8, "seq_fields.gaps_in_frame_num_value_allowed_flag");
assert_eq!(1u32 << seq_bits::FRAME_MBS_ONLY, 0x10, "seq_fields.frame_mbs_only_flag");
assert_eq!(1u32 << seq_bits::MB_ADAPTIVE_FRAME_FIELD, 0x20, "seq_fields.mb_adaptive_frame_field_flag");
assert_eq!(1u32 << seq_bits::DIRECT_8X8_INFERENCE, 0x40, "seq_fields.direct_8x8_inference_flag");
assert_eq!(1u32 << seq_bits::MIN_LUMA_BI_PRED_SIZE_8X8, 0x80, "seq_fields.MinLumaBiPredSize8x8");
assert_eq!(15u32 << seq_bits::LOG2_MAX_FRAME_NUM_MINUS4, 0xf00, "seq_fields.log2_max_frame_num_minus4");
assert_eq!(3u32 << seq_bits::PIC_ORDER_CNT_TYPE, 0x3000, "seq_fields.pic_order_cnt_type");
assert_eq!(15u32 << seq_bits::LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4, 0x3c000, "seq_fields.log2_max_pic_order_cnt_lsb_minus4");
assert_eq!(1u32 << seq_bits::DELTA_PIC_ORDER_ALWAYS_ZERO, 0x40000, "seq_fields.delta_pic_order_always_zero_flag");
assert_eq!(1u32 << pic_bits::ENTROPY_CODING_MODE, 0x1, "pic_fields.entropy_coding_mode_flag");
assert_eq!(1u32 << pic_bits::WEIGHTED_PRED, 0x2, "pic_fields.weighted_pred_flag");
assert_eq!(3u32 << pic_bits::WEIGHTED_BIPRED_IDC, 0xc, "pic_fields.weighted_bipred_idc");
assert_eq!(1u32 << pic_bits::TRANSFORM_8X8_MODE, 0x10, "pic_fields.transform_8x8_mode_flag");
assert_eq!(1u32 << pic_bits::FIELD_PIC, 0x20, "pic_fields.field_pic_flag");
assert_eq!(1u32 << pic_bits::CONSTRAINED_INTRA_PRED, 0x40, "pic_fields.constrained_intra_pred_flag");
assert_eq!(1u32 << pic_bits::PIC_ORDER_PRESENT, 0x80, "pic_fields.pic_order_present_flag");
assert_eq!(1u32 << pic_bits::DEBLOCKING_FILTER_CONTROL_PRESENT, 0x100, "pic_fields.deblocking_filter_control_present_flag");
assert_eq!(1u32 << pic_bits::REDUNDANT_PIC_CNT_PRESENT, 0x200, "pic_fields.redundant_pic_cnt_present_flag");
assert_eq!(1u32 << pic_bits::REFERENCE_PIC, 0x400, "pic_fields.reference_pic_flag");
assert_eq!(3u32 << hevc_pic_bits::CHROMA_FORMAT_IDC, 0x3, "hevc pic_fields.chroma_format_idc");
assert_eq!(1u32 << hevc_pic_bits::SEPARATE_COLOUR_PLANE, 0x4, "hevc pic_fields.separate_colour_plane_flag");
assert_eq!(1u32 << hevc_pic_bits::PCM_ENABLED, 0x8, "hevc pic_fields.pcm_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::SCALING_LIST_ENABLED, 0x10, "hevc pic_fields.scaling_list_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::TRANSFORM_SKIP_ENABLED, 0x20, "hevc pic_fields.transform_skip_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::AMP_ENABLED, 0x40, "hevc pic_fields.amp_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::STRONG_INTRA_SMOOTHING_ENABLED, 0x80, "hevc pic_fields.strong_intra_smoothing_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::SIGN_DATA_HIDING_ENABLED, 0x100, "hevc pic_fields.sign_data_hiding_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::CONSTRAINED_INTRA_PRED, 0x200, "hevc pic_fields.constrained_intra_pred_flag");
assert_eq!(1u32 << hevc_pic_bits::CU_QP_DELTA_ENABLED, 0x400, "hevc pic_fields.cu_qp_delta_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::WEIGHTED_PRED, 0x800, "hevc pic_fields.weighted_pred_flag");
assert_eq!(1u32 << hevc_pic_bits::WEIGHTED_BIPRED, 0x1000, "hevc pic_fields.weighted_bipred_flag");
assert_eq!(1u32 << hevc_pic_bits::TRANSQUANT_BYPASS_ENABLED, 0x2000, "hevc pic_fields.transquant_bypass_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::TILES_ENABLED, 0x4000, "hevc pic_fields.tiles_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::ENTROPY_CODING_SYNC_ENABLED, 0x8000, "hevc pic_fields.entropy_coding_sync_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED, 0x10000, "hevc pic_fields.pps_loop_filter_across_slices_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::LOOP_FILTER_ACROSS_TILES_ENABLED, 0x20000, "hevc pic_fields.loop_filter_across_tiles_enabled_flag");
assert_eq!(1u32 << hevc_pic_bits::PCM_LOOP_FILTER_DISABLED, 0x40000, "hevc pic_fields.pcm_loop_filter_disabled_flag");
assert_eq!(1u32 << hevc_pic_bits::NO_PIC_REORDERING, 0x80000, "hevc pic_fields.NoPicReorderingFlag");
assert_eq!(1u32 << hevc_pic_bits::NO_BI_PRED, 0x100000, "hevc pic_fields.NoBiPredFlag");
assert_eq!(1u32 << hevc_slice_parsing_bits::LISTS_MODIFICATION_PRESENT, 0x1, "hevc slice_parsing_fields.lists_modification_present_flag");
assert_eq!(1u32 << hevc_slice_parsing_bits::LONG_TERM_REF_PICS_PRESENT, 0x2, "hevc slice_parsing_fields.long_term_ref_pics_present_flag");
assert_eq!(1u32 << hevc_slice_parsing_bits::SPS_TEMPORAL_MVP_ENABLED, 0x4, "hevc slice_parsing_fields.sps_temporal_mvp_enabled_flag");
assert_eq!(1u32 << hevc_slice_parsing_bits::CABAC_INIT_PRESENT, 0x8, "hevc slice_parsing_fields.cabac_init_present_flag");
assert_eq!(1u32 << hevc_slice_parsing_bits::OUTPUT_FLAG_PRESENT, 0x10, "hevc slice_parsing_fields.output_flag_present_flag");
assert_eq!(1u32 << hevc_slice_parsing_bits::DEPENDENT_SLICE_SEGMENTS_ENABLED, 0x20, "hevc slice_parsing_fields.dependent_slice_segments_enabled_flag");
assert_eq!(
1u32 << hevc_slice_parsing_bits::PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT,
0x40,
"hevc slice_parsing_fields.pps_slice_chroma_qp_offsets_present_flag"
);
assert_eq!(1u32 << hevc_slice_parsing_bits::SAMPLE_ADAPTIVE_OFFSET_ENABLED, 0x80, "hevc slice_parsing_fields.sample_adaptive_offset_enabled_flag");
assert_eq!(1u32 << hevc_slice_parsing_bits::DEBLOCKING_FILTER_OVERRIDE_ENABLED, 0x100, "hevc slice_parsing_fields.deblocking_filter_override_enabled_flag");
assert_eq!(1u32 << hevc_slice_parsing_bits::PPS_DISABLE_DEBLOCKING_FILTER, 0x200, "hevc slice_parsing_fields.pps_disable_deblocking_filter_flag");
assert_eq!(
1u32 << hevc_slice_parsing_bits::SLICE_SEGMENT_HEADER_EXTENSION_PRESENT,
0x400,
"hevc slice_parsing_fields.slice_segment_header_extension_present_flag"
);
assert_eq!(1u32 << hevc_slice_parsing_bits::RAP_PIC, 0x800, "hevc slice_parsing_fields.RapPicFlag");
assert_eq!(1u32 << hevc_slice_parsing_bits::IDR_PIC, 0x1000, "hevc slice_parsing_fields.IdrPicFlag");
assert_eq!(1u32 << hevc_slice_parsing_bits::INTRA_PIC, 0x2000, "hevc slice_parsing_fields.IntraPicFlag");
assert_eq!(1u32 << hevc_slice_bits::LAST_SLICE_OF_PIC, 0x1, "LongSliceFlags.LastSliceOfPic");
assert_eq!(1u32 << hevc_slice_bits::DEPENDENT_SLICE_SEGMENT, 0x2, "LongSliceFlags.dependent_slice_segment_flag");
assert_eq!(3u32 << hevc_slice_bits::SLICE_TYPE, 0xc, "LongSliceFlags.slice_type");
assert_eq!(3u32 << hevc_slice_bits::COLOR_PLANE_ID, 0x30, "LongSliceFlags.color_plane_id");
assert_eq!(1u32 << hevc_slice_bits::SLICE_SAO_LUMA, 0x40, "LongSliceFlags.slice_sao_luma_flag");
assert_eq!(1u32 << hevc_slice_bits::SLICE_SAO_CHROMA, 0x80, "LongSliceFlags.slice_sao_chroma_flag");
assert_eq!(1u32 << hevc_slice_bits::MVD_L1_ZERO, 0x100, "LongSliceFlags.mvd_l1_zero_flag");
assert_eq!(1u32 << hevc_slice_bits::CABAC_INIT, 0x200, "LongSliceFlags.cabac_init_flag");
assert_eq!(1u32 << hevc_slice_bits::SLICE_TEMPORAL_MVP_ENABLED, 0x400, "LongSliceFlags.slice_temporal_mvp_enabled_flag");
assert_eq!(1u32 << hevc_slice_bits::SLICE_DEBLOCKING_FILTER_DISABLED, 0x800, "LongSliceFlags.slice_deblocking_filter_disabled_flag");
assert_eq!(1u32 << hevc_slice_bits::COLLOCATED_FROM_L0, 0x1000, "LongSliceFlags.collocated_from_l0_flag");
assert_eq!(1u32 << hevc_slice_bits::SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED, 0x2000, "LongSliceFlags.slice_loop_filter_across_slices_enabled_flag");
}

519
crates/platform/src/vaapi/ffi.rs Normal file
View File

@ -0,0 +1,519 @@
// The declarations in this file are transcribed from libva's `va/va.h`, `va/va_dec_hevc.h` and `va/va_drm.h`
// (VA-API 1.23, libva 2.23), which carry this notice:
//
// Copyright (c) 2007-2009 Intel Corporation. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sub license, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice (including the
// next paragraph) shall be included in all copies or substantial portions
// of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
// IN NO EVENT SHALL INTEL AND/OR ITS SUPPLIERS BE LIABLE FOR
// ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//! VA-API (`va.h`, `va_dec_hevc.h`, `va_drm.h`; MIT-licensed headers by Intel) as `libva.so.2` /
//! `libva-drm.so.2` expect it: the H.264 and HEVC decode buffers this backend fills in, the image structures it reads back
//! through, and the entry points it calls. Written from the public header; sizes and field offsets
//! are checked against a C compiler's by `abi_tests.rs`.
//!
//! The structures are plain data and compile on every target, so the code that fills them in
//! (`vaapi::h264`) is tested everywhere; only `vaapi::va` loads the library (Linux).
//!
//! FFI module (docs/adr/0001-platform-ffi.md): plain `repr(C)` data, no logic.
#![allow(non_snake_case, non_camel_case_types, non_upper_case_globals, dead_code)]
use std::ffi::{c_char, c_int, c_uint, c_void};
pub type VADisplay = *mut c_void;
pub type VAStatus = c_int;
pub type VAGenericID = c_uint;
pub type VAConfigID = VAGenericID;
pub type VAContextID = VAGenericID;
pub type VASurfaceID = VAGenericID;
pub type VABufferID = VAGenericID;
pub type VAImageID = VAGenericID;
/// `VAProfile` (a C enum: `int`).
pub type VAProfile = c_int;
/// `VAEntrypoint` (a C enum: `int`).
pub type VAEntrypoint = c_int;
/// `VABufferType` (a C enum: `int`).
pub type VABufferType = c_int;
/// `VAConfigAttribType` (a C enum: `int`).
pub type VAConfigAttribType = c_int;
pub type VAMessageCallback = Option<unsafe extern "C" fn(user_context: *mut c_void, message: *const c_char)>;
pub const VA_STATUS_SUCCESS: VAStatus = 0;
pub const VA_STATUS_ERROR_UNSUPPORTED_PROFILE: VAStatus = 0x0c;
pub const VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT: VAStatus = 0x0d;
pub const VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT: VAStatus = 0x0e;
pub const VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED: VAStatus = 0x13;
pub const VA_STATUS_ERROR_DECODING_ERROR: VAStatus = 0x17;
pub const VA_INVALID_ID: VAGenericID = 0xffff_ffff;
pub const VA_INVALID_SURFACE: VASurfaceID = VA_INVALID_ID;
pub const VAProfileH264Main: VAProfile = 6;
pub const VAProfileH264High: VAProfile = 7;
pub const VAProfileH264ConstrainedBaseline: VAProfile = 13;
pub const VAProfileHEVCMain: VAProfile = 17;
pub const VAProfileHEVCMain10: VAProfile = 18;
pub const VAEntrypointVLD: VAEntrypoint = 1;
pub const VAConfigAttribRTFormat: VAConfigAttribType = 0;
pub const VA_RT_FORMAT_YUV420: c_uint = 0x0000_0001;
pub const VA_RT_FORMAT_YUV420_10: c_uint = 0x0000_0100;
/// `vaCreateContext` flag: progressive pictures only.
pub const VA_PROGRESSIVE: c_int = 0x1;
pub const VAPictureParameterBufferType: VABufferType = 0;
pub const VAIQMatrixBufferType: VABufferType = 1;
pub const VASliceParameterBufferType: VABufferType = 4;
pub const VASliceDataBufferType: VABufferType = 5;
/// `VASliceParameterBuffer*::slice_data_flag`: the whole slice is in the buffer.
pub const VA_SLICE_DATA_FLAG_ALL: u32 = 0x00;
pub const VA_PICTURE_H264_INVALID: u32 = 0x0000_0001;
pub const VA_PICTURE_H264_SHORT_TERM_REFERENCE: u32 = 0x0000_0008;
pub const VA_PICTURE_H264_LONG_TERM_REFERENCE: u32 = 0x0000_0010;
pub const VA_FOURCC_NV12: u32 = 0x3231_564E;
pub const VA_FOURCC_P010: u32 = 0x3031_3050;
pub const VA_PICTURE_HEVC_INVALID: u32 = 0x0000_0001;
pub const VA_PICTURE_HEVC_LONG_TERM_REFERENCE: u32 = 0x0000_0008;
pub const VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE: u32 = 0x0000_0010;
pub const VA_PICTURE_HEVC_RPS_ST_CURR_AFTER: u32 = 0x0000_0020;
pub const VA_PICTURE_HEVC_RPS_LT_CURR: u32 = 0x0000_0040;
pub const VA_PADDING_LOW: usize = 4;
pub const VA_PADDING_MEDIUM: usize = 8;
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct VAConfigAttrib {
pub type_: VAConfigAttribType,
pub value: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct VAPictureH264 {
pub picture_id: VASurfaceID,
pub frame_idx: u32,
pub flags: u32,
pub TopFieldOrderCnt: i32,
pub BottomFieldOrderCnt: i32,
pub va_reserved: [u32; VA_PADDING_LOW],
}
impl VAPictureH264 {
/// An unused entry (`VA_INVALID_SURFACE`, `VA_PICTURE_H264_INVALID`).
pub const INVALID: Self = Self {
picture_id: VA_INVALID_SURFACE,
frame_idx: 0,
flags: VA_PICTURE_H264_INVALID,
TopFieldOrderCnt: 0,
BottomFieldOrderCnt: 0,
va_reserved: [0; VA_PADDING_LOW],
};
}
/// `VAPictureParameterBufferH264`. `seq_fields` and `pic_fields` are C bit-field unions, stored
/// here as their `value` word (bits allocated from the least significant end, as GCC and Clang do
/// on the little-endian targets libva runs on).
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct VAPictureParameterBufferH264 {
pub CurrPic: VAPictureH264,
pub ReferenceFrames: [VAPictureH264; 16],
pub picture_width_in_mbs_minus1: u16,
pub picture_height_in_mbs_minus1: u16,
pub bit_depth_luma_minus8: u8,
pub bit_depth_chroma_minus8: u8,
pub num_ref_frames: u8,
pub seq_fields: u32,
pub num_slice_groups_minus1: u8,
pub slice_group_map_type: u8,
pub slice_group_change_rate_minus1: u16,
pub pic_init_qp_minus26: i8,
pub pic_init_qs_minus26: i8,
pub chroma_qp_index_offset: i8,
pub second_chroma_qp_index_offset: i8,
pub pic_fields: u32,
pub frame_num: u16,
pub va_reserved: [u32; VA_PADDING_MEDIUM],
}
/// `seq_fields` bit positions (`chroma_format_idc` 2 bits, the log2 fields 4, `pic_order_cnt_type` 2).
pub mod seq_bits {
pub const CHROMA_FORMAT_IDC: u32 = 0;
pub const RESIDUAL_COLOUR_TRANSFORM: u32 = 2;
pub const GAPS_IN_FRAME_NUM_ALLOWED: u32 = 3;
pub const FRAME_MBS_ONLY: u32 = 4;
pub const MB_ADAPTIVE_FRAME_FIELD: u32 = 5;
pub const DIRECT_8X8_INFERENCE: u32 = 6;
pub const MIN_LUMA_BI_PRED_SIZE_8X8: u32 = 7;
pub const LOG2_MAX_FRAME_NUM_MINUS4: u32 = 8;
pub const PIC_ORDER_CNT_TYPE: u32 = 12;
pub const LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4: u32 = 14;
pub const DELTA_PIC_ORDER_ALWAYS_ZERO: u32 = 18;
}
/// `pic_fields` bit positions (`weighted_bipred_idc` 2 bits).
pub mod pic_bits {
pub const ENTROPY_CODING_MODE: u32 = 0;
pub const WEIGHTED_PRED: u32 = 1;
pub const WEIGHTED_BIPRED_IDC: u32 = 2;
pub const TRANSFORM_8X8_MODE: u32 = 4;
pub const FIELD_PIC: u32 = 5;
pub const CONSTRAINED_INTRA_PRED: u32 = 6;
pub const PIC_ORDER_PRESENT: u32 = 7;
pub const DEBLOCKING_FILTER_CONTROL_PRESENT: u32 = 8;
pub const REDUNDANT_PIC_CNT_PRESENT: u32 = 9;
pub const REFERENCE_PIC: u32 = 10;
}
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct VAIQMatrixBufferH264 {
/// 4x4 scaling lists in raster scan order: Intra Y, Cb, Cr, Inter Y, Cb, Cr.
pub ScalingList4x4: [[u8; 16]; 6],
/// 8x8 scaling lists in raster scan order: Intra Y, Inter Y.
pub ScalingList8x8: [[u8; 64]; 2],
pub va_reserved: [u32; VA_PADDING_LOW],
}
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct VASliceParameterBufferH264 {
pub slice_data_size: u32,
pub slice_data_offset: u32,
pub slice_data_flag: u32,
/// Bits from the start of the NAL unit (its header byte included) to `slice_data()`, counted
/// without emulation prevention bytes; the data buffer itself keeps them.
pub slice_data_bit_offset: u16,
pub first_mb_in_slice: u16,
pub slice_type: u8,
pub direct_spatial_mv_pred_flag: u8,
pub num_ref_idx_l0_active_minus1: u8,
pub num_ref_idx_l1_active_minus1: u8,
pub cabac_init_idc: u8,
pub slice_qp_delta: i8,
pub disable_deblocking_filter_idc: u8,
pub slice_alpha_c0_offset_div2: i8,
pub slice_beta_offset_div2: i8,
pub RefPicList0: [VAPictureH264; 32],
pub RefPicList1: [VAPictureH264; 32],
pub luma_log2_weight_denom: u8,
pub chroma_log2_weight_denom: u8,
pub luma_weight_l0_flag: u8,
pub luma_weight_l0: [i16; 32],
pub luma_offset_l0: [i16; 32],
pub chroma_weight_l0_flag: u8,
pub chroma_weight_l0: [[i16; 2]; 32],
pub chroma_offset_l0: [[i16; 2]; 32],
pub luma_weight_l1_flag: u8,
pub luma_weight_l1: [i16; 32],
pub luma_offset_l1: [i16; 32],
pub chroma_weight_l1_flag: u8,
pub chroma_weight_l1: [[i16; 2]; 32],
pub chroma_offset_l1: [[i16; 2]; 32],
pub va_reserved: [u32; VA_PADDING_LOW],
}
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct VAPictureHEVC {
pub picture_id: VASurfaceID,
pub pic_order_cnt: i32,
pub flags: u32,
pub va_reserved: [u32; VA_PADDING_LOW],
}
impl VAPictureHEVC {
/// An unused entry (`VA_INVALID_SURFACE`, `VA_PICTURE_HEVC_INVALID`).
pub const INVALID: Self = Self { picture_id: VA_INVALID_SURFACE, pic_order_cnt: 0, flags: VA_PICTURE_HEVC_INVALID, va_reserved: [0; VA_PADDING_LOW] };
}
/// `VAPictureParameterBufferHEVC`; `pic_fields` and `slice_parsing_fields` are C bit-field unions
/// stored as their `value` word (see [`VAPictureParameterBufferH264`]).
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct VAPictureParameterBufferHEVC {
pub CurrPic: VAPictureHEVC,
pub ReferenceFrames: [VAPictureHEVC; 15],
pub pic_width_in_luma_samples: u16,
pub pic_height_in_luma_samples: u16,
pub pic_fields: u32,
pub sps_max_dec_pic_buffering_minus1: u8,
pub bit_depth_luma_minus8: u8,
pub bit_depth_chroma_minus8: u8,
pub pcm_sample_bit_depth_luma_minus1: u8,
pub pcm_sample_bit_depth_chroma_minus1: u8,
pub log2_min_luma_coding_block_size_minus3: u8,
pub log2_diff_max_min_luma_coding_block_size: u8,
pub log2_min_transform_block_size_minus2: u8,
pub log2_diff_max_min_transform_block_size: u8,
pub log2_min_pcm_luma_coding_block_size_minus3: u8,
pub log2_diff_max_min_pcm_luma_coding_block_size: u8,
pub max_transform_hierarchy_depth_intra: u8,
pub max_transform_hierarchy_depth_inter: u8,
pub init_qp_minus26: i8,
pub diff_cu_qp_delta_depth: u8,
pub pps_cb_qp_offset: i8,
pub pps_cr_qp_offset: i8,
pub log2_parallel_merge_level_minus2: u8,
pub num_tile_columns_minus1: u8,
pub num_tile_rows_minus1: u8,
pub column_width_minus1: [u16; 19],
pub row_height_minus1: [u16; 21],
pub slice_parsing_fields: u32,
pub log2_max_pic_order_cnt_lsb_minus4: u8,
pub num_short_term_ref_pic_sets: u8,
pub num_long_term_ref_pic_sps: u8,
pub num_ref_idx_l0_default_active_minus1: u8,
pub num_ref_idx_l1_default_active_minus1: u8,
pub pps_beta_offset_div2: i8,
pub pps_tc_offset_div2: i8,
pub num_extra_slice_header_bits: u8,
/// Bits of `short_term_ref_pic_set()` in the slice header (0 when the SPS's set is used),
/// counted without emulation prevention bytes.
pub st_rps_bits: u32,
pub va_reserved: [u32; VA_PADDING_MEDIUM],
}
/// `VAPictureParameterBufferHEVC::pic_fields` bit positions (`chroma_format_idc` 2 bits).
pub mod hevc_pic_bits {
pub const CHROMA_FORMAT_IDC: u32 = 0;
pub const SEPARATE_COLOUR_PLANE: u32 = 2;
pub const PCM_ENABLED: u32 = 3;
pub const SCALING_LIST_ENABLED: u32 = 4;
pub const TRANSFORM_SKIP_ENABLED: u32 = 5;
pub const AMP_ENABLED: u32 = 6;
pub const STRONG_INTRA_SMOOTHING_ENABLED: u32 = 7;
pub const SIGN_DATA_HIDING_ENABLED: u32 = 8;
pub const CONSTRAINED_INTRA_PRED: u32 = 9;
pub const CU_QP_DELTA_ENABLED: u32 = 10;
pub const WEIGHTED_PRED: u32 = 11;
pub const WEIGHTED_BIPRED: u32 = 12;
pub const TRANSQUANT_BYPASS_ENABLED: u32 = 13;
pub const TILES_ENABLED: u32 = 14;
pub const ENTROPY_CODING_SYNC_ENABLED: u32 = 15;
pub const PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED: u32 = 16;
pub const LOOP_FILTER_ACROSS_TILES_ENABLED: u32 = 17;
pub const PCM_LOOP_FILTER_DISABLED: u32 = 18;
pub const NO_PIC_REORDERING: u32 = 19;
pub const NO_BI_PRED: u32 = 20;
}
/// `VAPictureParameterBufferHEVC::slice_parsing_fields` bit positions.
pub mod hevc_slice_parsing_bits {
pub const LISTS_MODIFICATION_PRESENT: u32 = 0;
pub const LONG_TERM_REF_PICS_PRESENT: u32 = 1;
pub const SPS_TEMPORAL_MVP_ENABLED: u32 = 2;
pub const CABAC_INIT_PRESENT: u32 = 3;
pub const OUTPUT_FLAG_PRESENT: u32 = 4;
pub const DEPENDENT_SLICE_SEGMENTS_ENABLED: u32 = 5;
pub const PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT: u32 = 6;
pub const SAMPLE_ADAPTIVE_OFFSET_ENABLED: u32 = 7;
pub const DEBLOCKING_FILTER_OVERRIDE_ENABLED: u32 = 8;
pub const PPS_DISABLE_DEBLOCKING_FILTER: u32 = 9;
pub const SLICE_SEGMENT_HEADER_EXTENSION_PRESENT: u32 = 10;
pub const RAP_PIC: u32 = 11;
pub const IDR_PIC: u32 = 12;
pub const INTRA_PIC: u32 = 13;
}
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct VASliceParameterBufferHEVC {
pub slice_data_size: u32,
pub slice_data_offset: u32,
pub slice_data_flag: u32,
/// Bytes from the start of the NAL unit (its two-byte header included) to
/// `slice_segment_data()`, counted without emulation prevention bytes; the data buffer itself
/// keeps them.
pub slice_data_byte_offset: u32,
pub slice_segment_address: u32,
/// Indices into `ReferenceFrames` (0xFF: none).
pub RefPicList: [[u8; 15]; 2],
pub LongSliceFlags: u32,
pub collocated_ref_idx: u8,
pub num_ref_idx_l0_active_minus1: u8,
pub num_ref_idx_l1_active_minus1: u8,
pub slice_qp_delta: i8,
pub slice_cb_qp_offset: i8,
pub slice_cr_qp_offset: i8,
pub slice_beta_offset_div2: i8,
pub slice_tc_offset_div2: i8,
pub luma_log2_weight_denom: u8,
pub delta_chroma_log2_weight_denom: i8,
pub delta_luma_weight_l0: [i8; 15],
pub luma_offset_l0: [i8; 15],
pub delta_chroma_weight_l0: [[i8; 2]; 15],
pub ChromaOffsetL0: [[i8; 2]; 15],
pub delta_luma_weight_l1: [i8; 15],
pub luma_offset_l1: [i8; 15],
pub delta_chroma_weight_l1: [[i8; 2]; 15],
pub ChromaOffsetL1: [[i8; 2]; 15],
pub five_minus_max_num_merge_cand: u8,
pub num_entry_point_offsets: u16,
pub entry_offset_to_subset_array: u16,
/// Emulation prevention bytes in the slice segment header.
pub slice_data_num_emu_prevn_bytes: u16,
pub va_reserved: [u32; VA_PADDING_LOW - 2],
}
/// `VASliceParameterBufferHEVC::LongSliceFlags` bit positions (`slice_type` and `color_plane_id` 2 bits).
pub mod hevc_slice_bits {
pub const LAST_SLICE_OF_PIC: u32 = 0;
pub const DEPENDENT_SLICE_SEGMENT: u32 = 1;
pub const SLICE_TYPE: u32 = 2;
pub const COLOR_PLANE_ID: u32 = 4;
pub const SLICE_SAO_LUMA: u32 = 6;
pub const SLICE_SAO_CHROMA: u32 = 7;
pub const MVD_L1_ZERO: u32 = 8;
pub const CABAC_INIT: u32 = 9;
pub const SLICE_TEMPORAL_MVP_ENABLED: u32 = 10;
pub const SLICE_DEBLOCKING_FILTER_DISABLED: u32 = 11;
pub const COLLOCATED_FROM_L0: u32 = 12;
pub const SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED: u32 = 13;
}
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct VAIQMatrixBufferHEVC {
/// `ScalingList[sizeId][matrixId][j]` of the spec for sizeId 0..3, raster order.
pub ScalingList4x4: [[u8; 16]; 6],
pub ScalingList8x8: [[u8; 64]; 6],
pub ScalingList16x16: [[u8; 64]; 6],
pub ScalingList32x32: [[u8; 64]; 2],
pub ScalingListDC16x16: [u8; 6],
pub ScalingListDC32x32: [u8; 2],
pub va_reserved: [u32; VA_PADDING_LOW],
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct VAImageFormat {
pub fourcc: u32,
pub byte_order: u32,
pub bits_per_pixel: u32,
pub depth: u32,
pub red_mask: u32,
pub green_mask: u32,
pub blue_mask: u32,
pub alpha_mask: u32,
pub va_reserved: [u32; VA_PADDING_LOW],
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct VAImage {
pub image_id: VAImageID,
pub format: VAImageFormat,
pub buf: VABufferID,
pub width: u16,
pub height: u16,
pub data_size: u32,
pub num_planes: u32,
pub pitches: [u32; 3],
pub offsets: [u32; 3],
pub num_palette_entries: i32,
pub entry_bytes: i32,
pub component_order: [i8; 4],
pub va_reserved: [u32; VA_PADDING_LOW],
}
/// The `libva.so.2` entry points used (all `VAStatus`-returning unless noted).
pub type vaInitialize = unsafe extern "C" fn(dpy: VADisplay, major: *mut c_int, minor: *mut c_int) -> VAStatus;
pub type vaTerminate = unsafe extern "C" fn(dpy: VADisplay) -> VAStatus;
pub type vaErrorStr = unsafe extern "C" fn(status: VAStatus) -> *const c_char;
pub type vaQueryVendorString = unsafe extern "C" fn(dpy: VADisplay) -> *const c_char;
pub type vaSetInfoCallback = unsafe extern "C" fn(dpy: VADisplay, callback: VAMessageCallback, user_context: *mut c_void) -> VAMessageCallback;
pub type vaSetErrorCallback = unsafe extern "C" fn(dpy: VADisplay, callback: VAMessageCallback, user_context: *mut c_void) -> VAMessageCallback;
pub type vaGetConfigAttributes =
unsafe extern "C" fn(dpy: VADisplay, profile: VAProfile, entrypoint: VAEntrypoint, attrib_list: *mut VAConfigAttrib, num_attribs: c_int) -> VAStatus;
pub type vaCreateConfig = unsafe extern "C" fn(
dpy: VADisplay,
profile: VAProfile,
entrypoint: VAEntrypoint,
attrib_list: *mut VAConfigAttrib,
num_attribs: c_int,
config_id: *mut VAConfigID,
) -> VAStatus;
pub type vaDestroyConfig = unsafe extern "C" fn(dpy: VADisplay, config_id: VAConfigID) -> VAStatus;
/// The last two arguments are `VASurfaceAttrib *attrib_list, unsigned int num_attribs`; this
/// backend passes none, so the attribute structure is not declared.
pub type vaCreateSurfaces = unsafe extern "C" fn(
dpy: VADisplay,
format: c_uint,
width: c_uint,
height: c_uint,
surfaces: *mut VASurfaceID,
num_surfaces: c_uint,
attrib_list: *mut c_void,
num_attribs: c_uint,
) -> VAStatus;
pub type vaDestroySurfaces = unsafe extern "C" fn(dpy: VADisplay, surfaces: *mut VASurfaceID, num_surfaces: c_int) -> VAStatus;
pub type vaCreateContext = unsafe extern "C" fn(
dpy: VADisplay,
config_id: VAConfigID,
picture_width: c_int,
picture_height: c_int,
flag: c_int,
render_targets: *mut VASurfaceID,
num_render_targets: c_int,
context: *mut VAContextID,
) -> VAStatus;
pub type vaDestroyContext = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID) -> VAStatus;
pub type vaCreateBuffer = unsafe extern "C" fn(
dpy: VADisplay,
context: VAContextID,
type_: VABufferType,
size: c_uint,
num_elements: c_uint,
data: *mut c_void,
buf_id: *mut VABufferID,
) -> VAStatus;
pub type vaDestroyBuffer = unsafe extern "C" fn(dpy: VADisplay, buffer_id: VABufferID) -> VAStatus;
pub type vaBeginPicture = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID, render_target: VASurfaceID) -> VAStatus;
pub type vaRenderPicture = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID, buffers: *mut VABufferID, num_buffers: c_int) -> VAStatus;
pub type vaEndPicture = unsafe extern "C" fn(dpy: VADisplay, context: VAContextID) -> VAStatus;
pub type vaSyncSurface = unsafe extern "C" fn(dpy: VADisplay, render_target: VASurfaceID) -> VAStatus;
pub type vaCreateImage = unsafe extern "C" fn(dpy: VADisplay, format: *mut VAImageFormat, width: c_int, height: c_int, image: *mut VAImage) -> VAStatus;
pub type vaDestroyImage = unsafe extern "C" fn(dpy: VADisplay, image: VAImageID) -> VAStatus;
pub type vaGetImage =
unsafe extern "C" fn(dpy: VADisplay, surface: VASurfaceID, x: c_int, y: c_int, width: c_uint, height: c_uint, image: VAImageID) -> VAStatus;
pub type vaPutImage = unsafe extern "C" fn(
dpy: VADisplay,
surface: VASurfaceID,
image: VAImageID,
src_x: c_int,
src_y: c_int,
src_width: c_uint,
src_height: c_uint,
dest_x: c_int,
dest_y: c_int,
dest_width: c_uint,
dest_height: c_uint,
) -> VAStatus;
pub type vaMapBuffer = unsafe extern "C" fn(dpy: VADisplay, buf_id: VABufferID, pbuf: *mut *mut c_void) -> VAStatus;
pub type vaUnmapBuffer = unsafe extern "C" fn(dpy: VADisplay, buf_id: VABufferID) -> VAStatus;
/// `libva-drm.so.2`: the display of a DRM device (render node) file descriptor.
pub type vaGetDisplayDRM = unsafe extern "C" fn(fd: c_int) -> VADisplay;

512
crates/platform/src/vaapi/h264.rs Normal file
View File

@ -0,0 +1,512 @@
//! The H.264 side of the VA-API decoder: everything a stateless hardware decoder needs from the
//! host, done with the software decoder's own parsers (`filmcraft_h264::params`, `slice`) and its
//! decoded picture buffer (`filmcraft_h264::dpb`: reference marking, reference lists, output
//! order), so both decoders make the same decisions and output the same pictures in the same order.
//!
//! [`Front`] parses each access unit, fills the VA-API picture / matrix / slice buffers and hands
//! them to an [`Accel`] (the libva session on Linux, a recording stand-in in tests), then reads
//! back the pictures the DPB outputs. What the hardware path does not take is an error, so the
//! hybrid decoder replays the run in software: field / MBAFF coding, FMO, SP / SI and data
//! partitioning, stream changes the surfaces were not made for.
//!
//! Safe code; compiled on every target so its logic is tested everywhere.
use std::sync::Arc;
use filmcraft_bitstream::{length_prefixed_nals, unescape_rbsp};
use filmcraft_codecs::{CodecError, DecodedFrame, Result};
use filmcraft_h264::dpb::{Dpb, DpbEntry, Output, OutputMeta, RefMark};
use filmcraft_h264::params::{Pps, Sps};
use filmcraft_h264::slice::{NalHeader, Poc, PocState, SliceHeader, SliceType, is_new_picture, nal_type};
use super::Accel;
use super::ffi::{self, VAIQMatrixBufferH264, VAPictureH264, VAPictureParameterBufferH264, VASliceParameterBufferH264, pic_bits, seq_bits};
/// A picture in the DPB: its surface, field order counts (after an MMCO 5, relative to itself) and
/// place in decoding order.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Surface {
pub index: usize,
pub top: i32,
pub bottom: i32,
pub seq: u64,
}
/// A frame_num gap frame not given a surface yet.
const UNASSIGNED: usize = usize::MAX;
/// The picture whose slices are being collected.
struct Pending {
target: usize,
sps: Arc<Sps>,
pps: Arc<Pps>,
first: SliceHeader,
poc: Poc,
pts: i64,
slices: Vec<(VASliceParameterBufferH264, Vec<u8>)>,
}
/// H.264 front end of a stateless hardware decoder (see the module docs).
pub struct Front<A: Accel> {
accel: A,
length_size: usize,
/// Coded size (macroblocks) the surfaces were made for.
mbs: (u32, u32),
spss: Vec<Option<Arc<Sps>>>,
ppss: Vec<Option<Arc<Pps>>>,
dpb: Dpb<Surface>,
poc_state: PocState,
prev_ref_frame_num: u32,
active_sps: Option<Arc<Sps>>,
pending: Option<Pending>,
/// Pictures made so far (decoding order of [`Surface::seq`]).
seq: u64,
}
fn decode_err(e: impl std::fmt::Display) -> CodecError {
CodecError::Decode(e.to_string())
}
fn unsupported(what: &str) -> CodecError {
CodecError::Decode(format!("the hardware decoder does not take {what}"))
}
impl<A: Accel> Front<A> {
/// A front end for samples with `length_size`-byte NAL length prefixes whose sample entry
/// carried `parameter_sets` (SPS / PPS NAL units), decoding into `accel`'s surfaces of
/// `mbs` macroblocks.
pub fn new(accel: A, length_size: usize, mbs: (u32, u32), parameter_sets: &[Vec<u8>]) -> Result<Self> {
let mut f = Self {
accel,
length_size,
mbs,
spss: vec![None; 32],
ppss: vec![None; 256],
dpb: Dpb::new(),
poc_state: PocState::default(),
prev_ref_frame_num: 0,
active_sps: None,
pending: None,
seq: 0,
};
f.load_parameter_sets(parameter_sets)?;
Ok(f)
}
fn load_parameter_sets(&mut self, sets: &[Vec<u8>]) -> Result<()> {
let mut frames = Vec::new();
for nal in sets {
self.handle_nal(nal, 0, &mut frames)?;
}
Ok(())
}
pub fn accel(&self) -> &A {
&self.accel
}
/// Decode one access unit; pictures that left the DPB come back in output order.
pub fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let nals = length_prefixed_nals(sample, self.length_size).map_err(decode_err)?;
let mut frames = Vec::new();
for nal in nals {
self.handle_nal(nal, pts, &mut frames)?;
}
// the access unit is complete
self.submit_pending(&mut frames)?;
Ok(frames)
}
/// Output every picture still held for reordering (references stay for what follows).
pub fn flush(&mut self) -> Result<Vec<DecodedFrame>> {
let mut frames = Vec::new();
self.submit_pending(&mut frames)?;
let mut outs = Vec::new();
self.dpb.flush(&mut outs);
self.read_out(outs, &mut frames)?;
Ok(frames)
}
/// Forget every picture (after a seek).
pub fn reset(&mut self) {
self.dpb = Dpb::new();
self.poc_state = PocState::default();
self.prev_ref_frame_num = 0;
self.active_sps = None;
self.pending = None;
}
/// Read back the pictures the DPB output, right away: a surface leaving the DPB is reused by
/// the next picture.
fn read_out(&mut self, outs: Vec<Output<Surface>>, frames: &mut Vec<DecodedFrame>) -> Result<()> {
for o in outs {
let frame = self.accel.read(o.frame.index).map_err(decode_err)?;
frames.push(DecodedFrame { pts: o.meta.pts, frame, draft: false });
}
Ok(())
}
fn handle_nal(&mut self, nal: &[u8], pts: i64, frames: &mut Vec<DecodedFrame>) -> Result<()> {
let Some(&first) = nal.first() else { return Ok(()) };
let rest = nal.get(1..).unwrap_or_default();
let hdr = NalHeader::parse(first).map_err(decode_err)?;
match hdr.nal_unit_type {
nal_type::SPS => {
let sps = Sps::parse(&unescape_rbsp(rest)).map_err(decode_err)?;
let slot = self.spss.get_mut(sps.id as usize).ok_or_else(|| decode_err("SPS id out of range"))?;
*slot = Some(Arc::new(sps));
}
nal_type::PPS => {
let spss: Vec<Option<Sps>> = self.spss.iter().map(|s| s.as_ref().map(|s| (**s).clone())).collect();
let pps = Pps::parse(&unescape_rbsp(rest), &spss).map_err(decode_err)?;
let slot = self.ppss.get_mut(pps.id as usize).ok_or_else(|| decode_err("PPS id out of range"))?;
*slot = Some(Arc::new(pps));
}
nal_type::SLICE | nal_type::IDR => self.handle_slice(nal, hdr, pts, frames)?,
nal_type::SLICE_DPA | nal_type::SLICE_DPB | nal_type::SLICE_DPC => return Err(unsupported("data partitioning")),
nal_type::END_SEQ | nal_type::END_STREAM => self.submit_pending(frames)?,
// SEI, AUD, filler, extensions: nothing to decode
_ => {}
}
Ok(())
}
fn handle_slice(&mut self, nal: &[u8], hdr: NalHeader, pts: i64, frames: &mut Vec<DecodedFrame>) -> Result<()> {
let rbsp = unescape_rbsp(nal.get(1..).unwrap_or_default());
let (ppss, spss) = (&self.ppss, &self.spss);
let mut found: Option<(Arc<Pps>, Arc<Sps>)> = None;
let (sh, _, _) = SliceHeader::parse(&rbsp, hdr, |id| {
let missing = |what: String| filmcraft_h264::Error::MissingParameterSet(what);
let pps = ppss.get(id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("PPS {id}")))?;
let sps = spss.get(pps.sps_id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("SPS {}", pps.sps_id)))?;
found = Some((pps.clone(), sps.clone()));
Ok((&**pps, &**sps))
})
.map_err(decode_err)?;
let Some((pps, sps)) = found else { return Err(decode_err("slice without a PPS")) };
sps.check_supported().map_err(decode_err)?;
if pps.num_slice_groups > 1 {
return Err(unsupported("slice groups (FMO)"));
}
if sh.field_pic {
return Err(unsupported("field pictures"));
}
if matches!(sh.slice_type, SliceType::Sp | SliceType::Si) {
return Err(unsupported("SP / SI slices"));
}
if sh.redundant_pic_cnt > 0 {
return Ok(()); // redundant slices are ignored, as in the software decoder
}
let new_pic = match &self.pending {
None => true,
Some(p) => is_new_picture(&p.first, &sh, &sps) || sh.first_mb_in_slice == 0 || !Arc::ptr_eq(&p.sps, &sps),
};
if new_pic {
self.submit_pending(frames)?;
if sh.first_mb_in_slice != 0 && self.active_sps.is_none() {
return Err(decode_err("stream does not start with the first slice of a picture"));
}
self.start_picture(&sh, &sps, &pps, pts, frames)?;
}
let Some(pending) = self.pending.as_ref() else { return Err(decode_err("slice without a started picture")) };
if !Arc::ptr_eq(&pending.pps, &pps) {
return Err(unsupported("a picture whose slices use different PPSs"));
}
let surfaces = self.accel.surfaces();
let lists = self
.dpb
.build_ref_lists(&sh, pending.poc.frame(), sps.max_frame_num(), |e| (va_ref(e, surfaces), e.mark != RefMark::Unused))
.map_err(decode_err)?;
if !sh.slice_type.is_intra() && (lists[0].is_empty() || (sh.slice_type.is_b() && lists[1].is_empty())) {
return Err(decode_err("no reference pictures available for an inter slice"));
}
// a missing reference (the software decoder conceals it with another picture) is not
// something to hand the hardware
if lists.iter().flatten().any(|(_, is_ref)| !is_ref) {
return Err(decode_err("a reference picture is missing"));
}
let refs = lists.map(|l| l.into_iter().map(|(p, _)| p).collect::<Vec<_>>());
let params = slice_params(&sh, nal.len(), &refs)?;
if let Some(p) = self.pending.as_mut() {
p.slices.push((params, nal.to_vec()));
}
Ok(())
}
fn start_picture(&mut self, sh: &SliceHeader, sps: &Arc<Sps>, pps: &Arc<Pps>, pts: i64, frames: &mut Vec<DecodedFrame>) -> Result<()> {
if (sps.pic_width_in_mbs, sps.frame_height_in_mbs()) != self.mbs {
return Err(unsupported("a picture size change"));
}
if sps.max_dpb_frames() >= self.accel.surfaces().len() {
return Err(unsupported("a DPB larger than the decoder's surfaces"));
}
// a new SPS with another DPB size outputs and forgets what came before (as in software)
let changed = self.active_sps.as_ref().is_some_and(|a| a.max_dpb_frames() != sps.max_dpb_frames());
let mut outs = Vec::new();
if changed {
self.dpb.flush(&mut outs);
self.dpb.entries.clear();
}
self.active_sps = Some(sps.clone());
self.dpb.capacity = sps.max_dpb_frames();
self.dpb.max_reorder = sps.max_num_reorder_frames().min(self.dpb.capacity);
if sh.idr {
self.dpb.idr(sh.no_output_of_prior_pics, &mut outs);
self.prev_ref_frame_num = 0;
} else {
let max = sps.max_frame_num();
if sh.frame_num != self.prev_ref_frame_num && sh.frame_num != (self.prev_ref_frame_num + 1) % max {
self.fill_frame_num_gap(sh, sps, pts)?;
}
}
self.read_out(outs, frames)?;
let poc = self.poc_state.compute(sh, sps);
let target =
(0..self.accel.surfaces().len()).find(|&i| !self.dpb.entries.iter().any(|e| e.frame.index == i)).ok_or_else(|| decode_err("no free surface"))?;
self.pending = Some(Pending { target, sps: sps.clone(), pps: pps.clone(), first: sh.clone(), poc, pts, slices: Vec::new() });
Ok(())
}
/// frame_num gap (8.2.5.2; also every start at a non-IDR picture, an open-GOP seek): "non-existing"
/// frames as the software decoder makes them, copies of the latest decoded frame (mid-gray when
/// there is none), so pictures predicted from them come out the same.
fn fill_frame_num_gap(&mut self, sh: &SliceHeader, sps: &Arc<Sps>, pts: i64) -> Result<()> {
let max = sps.max_frame_num();
let last = self.dpb.entries.iter().filter(|e| !e.non_existing).max_by_key(|e| e.frame.seq).map(|e| e.frame.index);
let meta = Arc::new(output_meta(sps, pts, false));
let poc_state = &mut self.poc_state;
// surfaces are given afterwards, to the frames the sliding window keeps
let mut make = |_| (Surface { index: UNASSIGNED, top: 0, bottom: 0, seq: 0 }, 0, meta.clone());
let mut update = |frame_num| poc_state.update_gap_frame(frame_num, sps);
self.dpb.fill_frame_num_gap(self.prev_ref_frame_num, sh.frame_num, max, sps.max_num_ref_frames as usize, &mut make, &mut update);
self.prev_ref_frame_num = (sh.frame_num + max - 1) % max;
let used: Vec<usize> = self.dpb.entries.iter().map(|e| e.frame.index).filter(|&i| i != UNASSIGNED).collect();
let mut free = (0..self.accel.surfaces().len()).filter(|i| !used.contains(i));
let mut fills = Vec::new();
for e in self.dpb.entries.iter_mut().filter(|e| e.frame.index == UNASSIGNED) {
let index = free.next().ok_or_else(|| decode_err("no free surface for a frame_num gap"))?;
self.seq += 1;
e.frame = Surface { index, top: 0, bottom: 0, seq: self.seq };
fills.push(index);
}
for index in fills {
self.accel.fill(index, last).map_err(decode_err)?;
}
Ok(())
}
/// Decode the collected picture, then do its reference marking and DPB insertion.
fn submit_pending(&mut self, frames: &mut Vec<DecodedFrame>) -> Result<()> {
let Some(p) = self.pending.take() else { return Ok(()) };
if p.slices.is_empty() {
return Err(decode_err("a picture without slices"));
}
let surfaces = self.accel.surfaces();
let target_id = surfaces.get(p.target).copied().ok_or_else(|| decode_err("surface out of range"))?;
if self.dpb.entries.iter().filter(|e| e.mark != RefMark::Unused).count() > 16 {
return Err(decode_err("more than 16 reference frames"));
}
let mut pic = picture_params(&p.sps, &p.pps, &p.first);
pic.CurrPic = VAPictureH264 {
picture_id: target_id,
frame_idx: p.first.frame_num,
flags: 0,
TopFieldOrderCnt: p.poc.top,
BottomFieldOrderCnt: p.poc.bottom,
va_reserved: [0; ffi::VA_PADDING_LOW],
};
for (slot, e) in pic.ReferenceFrames.iter_mut().zip(self.dpb.entries.iter().filter(|e| e.mark != RefMark::Unused)) {
*slot = va_ref(e, surfaces);
}
let iq = iq_matrix(&p.pps);
self.accel.decode_h264(p.target, &pic, &iq, &p.slices).map_err(decode_err)?;
self.poc_state.update(&p.first, &p.poc);
let mmco5 = p.first.has_mmco5();
if p.first.nal_ref_idc != 0 {
self.prev_ref_frame_num = if mmco5 { 0 } else { p.first.frame_num };
}
// MMCO 5: the picture's order counts become relative to itself (tempPicOrderCnt, 8.2.1)
let temp = if mmco5 { p.poc.frame() } else { 0 };
self.seq += 1;
let surface = Surface { index: p.target, top: p.poc.top.wrapping_sub(temp), bottom: p.poc.bottom.wrapping_sub(temp), seq: self.seq };
let fpoc = if mmco5 { 0 } else { p.poc.frame() };
let meta = Arc::new(output_meta(&p.sps, p.pts, p.first.idr));
let mut outs = Vec::new();
self.dpb.store_picture(&p.first, surface, fpoc, p.sps.max_frame_num(), p.sps.max_num_ref_frames as usize, meta, &mut outs);
self.read_out(outs, frames)
}
}
/// A DPB entry as a VA-API reference picture.
fn va_ref(e: &DpbEntry<Surface>, surfaces: &[ffi::VASurfaceID]) -> VAPictureH264 {
let long = e.mark == RefMark::Long;
VAPictureH264 {
picture_id: surfaces.get(e.frame.index).copied().unwrap_or(ffi::VA_INVALID_SURFACE),
frame_idx: if long { e.long_term_frame_idx } else { e.frame_num },
flags: if long { ffi::VA_PICTURE_H264_LONG_TERM_REFERENCE } else { ffi::VA_PICTURE_H264_SHORT_TERM_REFERENCE },
TopFieldOrderCnt: e.frame.top,
BottomFieldOrderCnt: e.frame.bottom,
va_reserved: [0; ffi::VA_PADDING_LOW],
}
}
fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta {
let vui = sps.vui.clone().unwrap_or_default();
OutputMeta {
pts,
key,
crop: sps.crop_rect(),
full_range: vui.full_range,
colour_primaries: vui.colour_primaries,
transfer_characteristics: vui.transfer_characteristics,
matrix_coefficients: vui.matrix_coefficients,
sar: vui.sar,
draft: false,
}
}
fn flag(on: bool, bit: u32) -> u32 {
u32::from(on) << bit
}
/// The picture parameters of a picture coded with `sps` / `pps` whose first slice is `sh` (the
/// current and reference pictures are filled in by the caller).
pub fn picture_params(sps: &Sps, pps: &Pps, sh: &SliceHeader) -> VAPictureParameterBufferH264 {
let seq_fields = (sps.chroma_format_idc & 3) << seq_bits::CHROMA_FORMAT_IDC
| flag(sps.separate_colour_plane, seq_bits::RESIDUAL_COLOUR_TRANSFORM)
| flag(sps.gaps_in_frame_num_allowed, seq_bits::GAPS_IN_FRAME_NUM_ALLOWED)
| flag(sps.frame_mbs_only, seq_bits::FRAME_MBS_ONLY)
| flag(sps.mb_adaptive_frame_field, seq_bits::MB_ADAPTIVE_FRAME_FIELD)
| flag(sps.direct_8x8_inference, seq_bits::DIRECT_8X8_INFERENCE)
// A.3.3.2: bi-prediction of blocks under 8x8 is not allowed from level 3.1
| flag(sps.level_idc >= 31, seq_bits::MIN_LUMA_BI_PRED_SIZE_8X8)
| (sps.log2_max_frame_num.saturating_sub(4) & 15) << seq_bits::LOG2_MAX_FRAME_NUM_MINUS4
| (sps.pic_order_cnt_type & 3) << seq_bits::PIC_ORDER_CNT_TYPE
| (sps.log2_max_poc_lsb.saturating_sub(4) & 15) << seq_bits::LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4
| flag(sps.delta_pic_order_always_zero, seq_bits::DELTA_PIC_ORDER_ALWAYS_ZERO);
let pic_fields = flag(pps.entropy_coding_mode, pic_bits::ENTROPY_CODING_MODE)
| flag(pps.weighted_pred, pic_bits::WEIGHTED_PRED)
| (pps.weighted_bipred_idc & 3) << pic_bits::WEIGHTED_BIPRED_IDC
| flag(pps.transform_8x8_mode, pic_bits::TRANSFORM_8X8_MODE)
| flag(sh.field_pic, pic_bits::FIELD_PIC)
| flag(pps.constrained_intra_pred, pic_bits::CONSTRAINED_INTRA_PRED)
| flag(pps.bottom_field_pic_order_in_frame_present, pic_bits::PIC_ORDER_PRESENT)
| flag(pps.deblocking_filter_control_present, pic_bits::DEBLOCKING_FILTER_CONTROL_PRESENT)
| flag(pps.redundant_pic_cnt_present, pic_bits::REDUNDANT_PIC_CNT_PRESENT)
| flag(sh.nal_ref_idc != 0, pic_bits::REFERENCE_PIC);
// the parsers bound these (frame size by the level limits, qp offsets to ±12, qp to 0..51)
let narrow = |v: u32| u16::try_from(v.saturating_sub(1)).unwrap_or(u16::MAX);
let small = |v: i32| i8::try_from(v).unwrap_or(0);
VAPictureParameterBufferH264 {
CurrPic: VAPictureH264::INVALID,
ReferenceFrames: [VAPictureH264::INVALID; 16],
picture_width_in_mbs_minus1: narrow(sps.pic_width_in_mbs),
picture_height_in_mbs_minus1: narrow(sps.frame_height_in_mbs()),
bit_depth_luma_minus8: u8::try_from(sps.bit_depth_luma.saturating_sub(8)).unwrap_or(0),
bit_depth_chroma_minus8: u8::try_from(sps.bit_depth_chroma.saturating_sub(8)).unwrap_or(0),
num_ref_frames: u8::try_from(sps.max_num_ref_frames).unwrap_or(16),
seq_fields,
num_slice_groups_minus1: 0,
slice_group_map_type: 0,
slice_group_change_rate_minus1: 0,
pic_init_qp_minus26: small(pps.pic_init_qp - 26),
pic_init_qs_minus26: small(pps.pic_init_qs - 26),
chroma_qp_index_offset: small(pps.chroma_qp_index_offset),
second_chroma_qp_index_offset: small(pps.second_chroma_qp_index_offset),
pic_fields,
frame_num: u16::try_from(sh.frame_num).unwrap_or(0),
va_reserved: [0; ffi::VA_PADDING_MEDIUM],
}
}
/// The scaling matrices of `pps` (after the SPS / PPS fall-back rules), raster order.
pub fn iq_matrix(pps: &Pps) -> VAIQMatrixBufferH264 {
VAIQMatrixBufferH264 { ScalingList4x4: pps.scaling.m4, ScalingList8x8: [pps.scaling.m8[0], pps.scaling.m8[1]], va_reserved: [0; ffi::VA_PADDING_LOW] }
}
/// The slice parameters of the slice `sh` (a `nal_len`-byte NAL unit) with reference lists `refs`.
pub fn slice_params(sh: &SliceHeader, nal_len: usize, refs: &[Vec<VAPictureH264>; 2]) -> Result<VASliceParameterBufferH264> {
let bit_offset = sh.header_bits.checked_add(8).and_then(|b| u16::try_from(b).ok()).ok_or_else(|| decode_err("slice header too long"))?;
let mut s = VASliceParameterBufferH264 {
slice_data_size: u32::try_from(nal_len).map_err(|_| decode_err("slice too large"))?,
slice_data_offset: 0,
slice_data_flag: ffi::VA_SLICE_DATA_FLAG_ALL,
slice_data_bit_offset: bit_offset,
first_mb_in_slice: u16::try_from(sh.first_mb_in_slice).map_err(|_| decode_err("first_mb_in_slice out of range"))?,
slice_type: (sh.slice_type_raw % 5) as u8,
direct_spatial_mv_pred_flag: u8::from(sh.direct_spatial_mv_pred),
num_ref_idx_l0_active_minus1: u8::try_from(sh.num_ref_idx_active[0].saturating_sub(1)).unwrap_or(31).min(31),
num_ref_idx_l1_active_minus1: u8::try_from(sh.num_ref_idx_active[1].saturating_sub(1)).unwrap_or(31).min(31),
cabac_init_idc: u8::try_from(sh.cabac_init_idc).unwrap_or(0),
slice_qp_delta: i8::try_from(sh.slice_qp_delta).map_err(|_| decode_err("slice_qp_delta out of range"))?,
disable_deblocking_filter_idc: u8::try_from(sh.disable_deblocking_filter_idc).unwrap_or(0),
slice_alpha_c0_offset_div2: i8::try_from(sh.slice_alpha_c0_offset_div2).unwrap_or(0),
slice_beta_offset_div2: i8::try_from(sh.slice_beta_offset_div2).unwrap_or(0),
RefPicList0: [VAPictureH264::INVALID; 32],
RefPicList1: [VAPictureH264::INVALID; 32],
luma_log2_weight_denom: 0,
chroma_log2_weight_denom: 0,
luma_weight_l0_flag: 0,
luma_weight_l0: [0; 32],
luma_offset_l0: [0; 32],
chroma_weight_l0_flag: 0,
chroma_weight_l0: [[0; 2]; 32],
chroma_offset_l0: [[0; 2]; 32],
luma_weight_l1_flag: 0,
luma_weight_l1: [0; 32],
luma_offset_l1: [0; 32],
chroma_weight_l1_flag: 0,
chroma_weight_l1: [[0; 2]; 32],
chroma_offset_l1: [[0; 2]; 32],
va_reserved: [0; ffi::VA_PADDING_LOW],
};
for (dst, src) in s.RefPicList0.iter_mut().zip(&refs[0]) {
*dst = *src;
}
for (dst, src) in s.RefPicList1.iter_mut().zip(&refs[1]) {
*dst = *src;
}
// explicit weighted prediction: every entry, the default weight where none was coded
if let Some(w) = &sh.pred_weight_table {
s.luma_log2_weight_denom = u8::try_from(w.luma_log2_denom).unwrap_or(0);
s.chroma_log2_weight_denom = u8::try_from(w.chroma_log2_denom).unwrap_or(0);
let w16 = |v: i32| i16::try_from(v).unwrap_or(0);
let lists = [
(
&w.l0,
&mut s.luma_weight_l0,
&mut s.luma_offset_l0,
&mut s.chroma_weight_l0,
&mut s.chroma_offset_l0,
&mut s.luma_weight_l0_flag,
&mut s.chroma_weight_l0_flag,
),
(
&w.l1,
&mut s.luma_weight_l1,
&mut s.luma_offset_l1,
&mut s.chroma_weight_l1,
&mut s.chroma_offset_l1,
&mut s.luma_weight_l1_flag,
&mut s.chroma_weight_l1_flag,
),
];
for (entries, lw, lo, cw, co, lflag, cflag) in lists {
if entries.is_empty() {
continue;
}
*lflag = 1;
*cflag = 1;
for (i, e) in entries.iter().take(32).enumerate() {
lw[i] = w16(e.luma_weight);
lo[i] = w16(e.luma_offset);
for c in 0..2 {
cw[i][c] = w16(e.chroma_weight[c]);
co[i][c] = w16(e.chroma_offset[c]);
}
}
}
}
Ok(s)
}

612
crates/platform/src/vaapi/hevc.rs Normal file
View File

@ -0,0 +1,612 @@
//! The HEVC side of the VA-API decoder, like [`super::h264`]: the software decoder's own parsers
//! (`filmcraft_hevc::params`, `slice`) and decoded picture buffer (`filmcraft_hevc::dpb`: POC,
//! reference picture sets, reference lists, output order) drive a stateless hardware decoder, so
//! both decoders make the same decisions and output the same pictures in the same order, RASL
//! pictures of a CRA the run starts at left out as in software.
//!
//! What the hardware path does not take is an error, so the hybrid decoder replays the run in
//! software: missing reference pictures (the software decoder conceals them), more than 15
//! reference frames, stream changes the surfaces were not made for, range extensions.
//!
//! Safe code; compiled on every target so its logic is tested everywhere.
use std::sync::Arc;
use filmcraft_bitstream::{length_prefixed_nals, unescape_rbsp};
use filmcraft_codecs::{CodecError, DecodedFrame, Result};
use filmcraft_hevc::dpb::{Dpb, Marking, Output, OutputMeta, PocState, Ref, RefPicSet, build_ref_lists};
use filmcraft_hevc::params::{Layout, Pps, ScalingList, Sps};
use filmcraft_hevc::slice::{NalHeader, SliceHeader, SliceType, nal_type};
use super::Accel;
use super::ffi::{
self, VAIQMatrixBufferHEVC, VAPictureHEVC, VAPictureParameterBufferHEVC, VASliceParameterBufferHEVC, hevc_pic_bits, hevc_slice_bits,
hevc_slice_parsing_bits,
};
/// A picture in the DPB: its surface.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Surface {
pub index: usize,
}
/// A reference picture that was never decoded (8.3.3): not handed to the hardware.
const MISSING: usize = usize::MAX;
/// The picture whose slice segments are being collected.
struct Pending {
target: usize,
sps: Arc<Sps>,
pps: Arc<Pps>,
first: SliceHeader,
/// The latest independent slice segment's header (dependent ones continue it).
last: SliceHeader,
rps: RefPicSet<Surface>,
/// The surfaces of `pic.ReferenceFrames`, in order.
ref_frames: Vec<usize>,
pic: VAPictureParameterBufferHEVC,
poc: i32,
output: bool,
meta: Arc<OutputMeta>,
slices: Vec<(VASliceParameterBufferHEVC, Vec<u8>)>,
all_intra: bool,
any_b: bool,
}
/// HEVC front end of a stateless hardware decoder (see the module docs).
pub struct Front<A: Accel> {
accel: A,
length_size: usize,
/// Coded size (luma samples) and bit depth the surfaces were made for.
size: (u32, u32),
bits: u32,
spss: Vec<Option<Arc<Sps>>>,
ppss: Vec<Option<Arc<Pps>>>,
dpb: Dpb<Surface>,
poc_state: PocState,
active_sps: Option<Arc<Sps>>,
pending: Option<Pending>,
first_picture: bool,
skip_rasl: bool,
after_eos: bool,
}
fn decode_err(e: impl std::fmt::Display) -> CodecError {
CodecError::Decode(e.to_string())
}
fn unsupported(what: &str) -> CodecError {
CodecError::Decode(format!("the hardware decoder does not take {what}"))
}
impl<A: Accel> Front<A> {
/// A front end for samples with `length_size`-byte NAL length prefixes whose sample entry
/// carried `parameter_sets` (VPS / SPS / PPS NAL units), decoding into `accel`'s surfaces of
/// `size` luma samples and `bits` bits.
pub fn new(accel: A, length_size: usize, size: (u32, u32), bits: u32, parameter_sets: &[Vec<u8>]) -> Result<Self> {
let mut f = Self {
accel,
length_size,
size,
bits,
spss: vec![None; 16],
ppss: vec![None; 64],
dpb: Dpb::default(),
poc_state: PocState::default(),
active_sps: None,
pending: None,
first_picture: true,
skip_rasl: false,
after_eos: false,
};
let mut frames = Vec::new();
for nal in parameter_sets {
f.handle_nal(nal, 0, &mut frames)?;
}
Ok(f)
}
pub fn accel(&self) -> &A {
&self.accel
}
/// Decode one access unit; pictures that left the DPB come back in output order.
pub fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let nals = length_prefixed_nals(sample, self.length_size).map_err(decode_err)?;
let mut frames = Vec::new();
for nal in nals {
self.handle_nal(nal, pts, &mut frames)?;
}
self.submit_pending(&mut frames)?;
Ok(frames)
}
/// Output every picture still held for reordering; what follows starts like a new stream
/// (a CRA's RASL pictures are left out), as in software.
pub fn flush(&mut self) -> Result<Vec<DecodedFrame>> {
let mut frames = Vec::new();
self.submit_pending(&mut frames)?;
let mut outs = Vec::new();
self.dpb.flush(&mut outs);
self.read_out(outs, &mut frames)?;
self.first_picture = true;
Ok(frames)
}
/// Forget every picture (after a seek).
pub fn reset(&mut self) {
self.dpb = Dpb::default();
self.poc_state = PocState::default();
self.active_sps = None;
self.pending = None;
self.first_picture = true;
self.skip_rasl = false;
self.after_eos = false;
}
/// Read back the pictures the DPB output, right away: a surface leaving the DPB is reused.
fn read_out(&mut self, outs: Vec<Output<Surface>>, frames: &mut Vec<DecodedFrame>) -> Result<()> {
for o in outs {
let frame = self.accel.read(o.frame.index).map_err(decode_err)?;
frames.push(DecodedFrame { pts: o.meta.pts, frame, draft: false });
}
Ok(())
}
fn handle_nal(&mut self, nal: &[u8], pts: i64, frames: &mut Vec<DecodedFrame>) -> Result<()> {
if nal.len() < 2 {
return Ok(());
}
let hdr = NalHeader::parse(nal).map_err(decode_err)?;
if hdr.layer_id != 0 {
return Ok(());
}
let rbsp = || unescape_rbsp(nal.get(2..).unwrap_or_default());
match hdr.nal_type {
nal_type::SPS => {
let sps = Sps::parse(&rbsp()).map_err(decode_err)?;
let slot = self.spss.get_mut(sps.id as usize).ok_or_else(|| decode_err("SPS id out of range"))?;
*slot = Some(Arc::new(sps));
}
nal_type::PPS => {
let pps = Pps::parse(&rbsp()).map_err(decode_err)?;
let slot = self.ppss.get_mut(pps.id as usize).ok_or_else(|| decode_err("PPS id out of range"))?;
*slot = Some(Arc::new(pps));
}
nal_type::EOS | nal_type::EOB => {
self.submit_pending(frames)?;
self.after_eos = true;
}
0..=9 | 16..=21 => self.handle_slice(nal, hdr, pts, frames)?,
// VPS, AUD, SEI, filler, reserved: nothing to decode
_ => {}
}
Ok(())
}
fn handle_slice(&mut self, nal: &[u8], hdr: NalHeader, pts: i64, frames: &mut Vec<DecodedFrame>) -> Result<()> {
let rbsp = unescape_rbsp(nal.get(2..).unwrap_or_default());
let first = rbsp.first().is_some_and(|b| b & 0x80 != 0);
if first {
self.submit_pending(frames)?;
}
if hdr.is_rasl() && (self.skip_rasl || self.first_picture) {
return Ok(());
}
let (ppss, spss) = (&self.ppss, &self.spss);
let prev = self.pending.as_ref().map(|p| &p.last);
let (sh, pps, sps) = SliceHeader::parse(
&rbsp,
hdr,
|id| {
let missing = |what: String| filmcraft_hevc::Error::MissingParameterSet(what);
let pps = ppss.get(id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("PPS {id}")))?;
let sps = spss.get(pps.sps_id as usize).and_then(Option::as_ref).ok_or_else(|| missing(format!("SPS {}", pps.sps_id)))?;
Ok((pps.clone(), sps.clone()))
},
prev,
)
.map_err(decode_err)?;
sps.check_supported().map_err(decode_err)?;
pps.check_supported().map_err(decode_err)?;
if first {
self.start_picture(&sh, &sps, &pps, pts, frames)?;
}
let Some(p) = self.pending.as_mut() else { return Err(decode_err("slice segment without the first slice of its picture")) };
if !Arc::ptr_eq(&p.pps, &pps) {
return Err(decode_err("PPS changed within a picture"));
}
let refs = build_ref_lists(&sh, &p.rps).map_err(decode_err)?;
let params = slice_params(&sh, nal, &refs, &p.ref_frames)?;
if !sh.dependent {
p.last = sh.clone();
}
p.all_intra &= sh.is_intra();
p.any_b |= sh.is_b();
p.slices.push((params, nal.to_vec()));
Ok(())
}
fn start_picture(&mut self, sh: &SliceHeader, sps: &Arc<Sps>, pps: &Arc<Pps>, pts: i64, frames: &mut Vec<DecodedFrame>) -> Result<()> {
if !sh.first_slice_segment_in_pic || sh.dependent {
return Err(decode_err("invalid first slice segment"));
}
if (sps.width, sps.height) != self.size || sps.bit_depth_luma != self.bits || sps.bit_depth_chroma != self.bits || sps.chroma_format_idc != 1 {
return Err(unsupported("a picture size or format change"));
}
if sps.max_dec_pic_buffering as usize >= self.accel.surfaces().len() {
return Err(unsupported("a DPB larger than the decoder's surfaces"));
}
let hdr = sh.nal;
let changed = self.active_sps.as_ref().is_some_and(|a| a.log2_ctb != sps.log2_ctb);
let mut outs = Vec::new();
if changed {
self.dpb.flush(&mut outs);
}
self.active_sps = Some(sps.clone());
self.dpb.max_dec_pic_buffering = sps.max_dec_pic_buffering as usize;
self.dpb.max_num_reorder = sps.max_num_reorder as usize;
self.dpb.max_latency = if sps.max_latency_increase_plus1 != 0 { sps.max_num_reorder + sps.max_latency_increase_plus1 - 1 } else { 0 };
let irap_no_rasl = hdr.is_irap() && (hdr.is_idr() || hdr.is_bla() || self.first_picture || self.after_eos);
if hdr.is_irap() {
self.skip_rasl = irap_no_rasl;
}
let max_lsb = sps.max_poc_lsb();
let poc = self.poc_state.compute(sh, max_lsb, irap_no_rasl);
let mut make_missing = |_| Surface { index: MISSING };
let rps = self.dpb.apply_rps(sh, poc, max_lsb, irap_no_rasl, &mut make_missing).map_err(decode_err)?;
if [&rps.st_curr_before, &rps.st_curr_after, &rps.lt_curr].iter().any(|l| l.iter().any(|r| r.frame.index == MISSING)) {
// the software decoder conceals them with mid-gray pictures
return Err(unsupported("missing reference pictures"));
}
if irap_no_rasl && !self.first_picture {
if sh.no_output_of_prior_pics && hdr.is_idr() {
self.dpb.clear();
} else {
self.dpb.flush(&mut outs);
}
} else {
self.dpb.bump_before_decode(&mut outs);
}
self.read_out(outs, frames)?;
let surfaces = self.accel.surfaces();
let target = (0..surfaces.len()).find(|&i| !self.dpb.entries.iter().any(|e| e.frame.index == i)).ok_or_else(|| decode_err("no free surface"))?;
let target_id = surfaces.get(target).copied().ok_or_else(|| decode_err("surface out of range"))?;
let mut pic = picture_params(sps, pps, sh)?;
pic.CurrPic = VAPictureHEVC { picture_id: target_id, pic_order_cnt: poc, flags: 0, va_reserved: [0; ffi::VA_PADDING_LOW] };
// the reference frames: every picture the RPS kept, flagged with its part of the RPS
let rps_flag = |index: usize| {
let in_list = |l: &[Ref<Surface>]| l.iter().any(|r| r.frame.index == index);
if in_list(&rps.st_curr_before) {
ffi::VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE
} else if in_list(&rps.st_curr_after) {
ffi::VA_PICTURE_HEVC_RPS_ST_CURR_AFTER
} else if in_list(&rps.lt_curr) {
ffi::VA_PICTURE_HEVC_RPS_LT_CURR
} else {
0
}
};
let refs: Vec<_> = self.dpb.entries.iter().filter(|e| e.marking != Marking::Unused).collect();
if refs.len() > pic.ReferenceFrames.len() {
return Err(unsupported("more than 15 reference frames"));
}
let mut ref_frames = Vec::with_capacity(refs.len());
for (slot, e) in pic.ReferenceFrames.iter_mut().zip(&refs) {
let long = if e.marking == Marking::Long { ffi::VA_PICTURE_HEVC_LONG_TERM_REFERENCE } else { 0 };
let picture_id = surfaces.get(e.frame.index).copied().ok_or_else(|| decode_err("surface out of range"))?;
*slot = VAPictureHEVC { picture_id, pic_order_cnt: e.poc, flags: long | rps_flag(e.frame.index), va_reserved: [0; ffi::VA_PADDING_LOW] };
ref_frames.push(e.frame.index);
}
let output = sh.pic_output && !(hdr.is_rasl() && self.skip_rasl);
let meta = Arc::new(output_meta(sps, pts, hdr.is_irap()));
self.pending = Some(Pending {
target,
sps: sps.clone(),
pps: pps.clone(),
first: sh.clone(),
last: sh.clone(),
rps,
ref_frames,
pic,
poc,
output,
meta,
slices: Vec::new(),
all_intra: true,
any_b: false,
});
self.first_picture = false;
self.after_eos = false;
Ok(())
}
/// Decode the collected picture, then insert it into the DPB.
fn submit_pending(&mut self, frames: &mut Vec<DecodedFrame>) -> Result<()> {
let Some(mut p) = self.pending.take() else { return Ok(()) };
let Some(last) = p.slices.last_mut() else { return Err(decode_err("a picture without slices")) };
last.0.LongSliceFlags |= 1 << hevc_slice_bits::LAST_SLICE_OF_PIC;
p.pic.slice_parsing_fields |= u32::from(p.all_intra) << hevc_slice_parsing_bits::INTRA_PIC;
p.pic.pic_fields |= u32::from(!p.any_b) << hevc_pic_bits::NO_BI_PRED;
let iq = if p.sps.scaling_list_enabled { Some(iq_matrix(p.pps.scaling_list.as_ref().or(p.sps.scaling_list.as_ref()))) } else { None };
self.accel.decode_hevc(p.target, &p.pic, iq.as_ref(), &p.slices).map_err(decode_err)?;
self.poc_state.update(&p.first, p.poc);
let mut outs = Vec::new();
self.dpb.insert(Surface { index: p.target }, p.poc, p.output, p.meta, &mut outs);
self.read_out(outs, frames)
}
}
fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta {
let vui = sps.vui.clone().unwrap_or_default();
OutputMeta {
pts,
key,
crop: sps.crop_rect(),
full_range: vui.full_range,
colour_primaries: vui.colour_primaries,
transfer_characteristics: vui.transfer_characteristics,
matrix_coefficients: vui.matrix_coefficients,
sar: vui.sar,
bit_depth: sps.bit_depth_luma,
draft: false,
}
}
fn flag(on: bool, bit: u32) -> u32 {
u32::from(on) << bit
}
fn u8_of(v: u32) -> u8 {
u8::try_from(v).unwrap_or(u8::MAX)
}
fn i8_of(v: i32) -> i8 {
i8::try_from(v).unwrap_or(0)
}
/// The picture parameters of a picture coded with `sps` / `pps` whose first slice segment is `sh`
/// (the current and reference pictures, `IntraPicFlag` and `NoBiPredFlag` are filled in by the caller).
pub fn picture_params(sps: &Sps, pps: &Pps, sh: &SliceHeader) -> Result<VAPictureParameterBufferHEVC> {
use hevc_pic_bits as p;
use hevc_slice_parsing_bits as s;
let pic_fields = (sps.chroma_format_idc & 3) << p::CHROMA_FORMAT_IDC
| flag(sps.separate_colour_plane, p::SEPARATE_COLOUR_PLANE)
| flag(sps.pcm, p::PCM_ENABLED)
| flag(sps.scaling_list_enabled, p::SCALING_LIST_ENABLED)
| flag(pps.transform_skip, p::TRANSFORM_SKIP_ENABLED)
| flag(sps.amp, p::AMP_ENABLED)
| flag(sps.strong_intra_smoothing, p::STRONG_INTRA_SMOOTHING_ENABLED)
| flag(pps.sign_data_hiding, p::SIGN_DATA_HIDING_ENABLED)
| flag(pps.constrained_intra_pred, p::CONSTRAINED_INTRA_PRED)
| flag(pps.cu_qp_delta_enabled, p::CU_QP_DELTA_ENABLED)
| flag(pps.weighted_pred, p::WEIGHTED_PRED)
| flag(pps.weighted_bipred, p::WEIGHTED_BIPRED)
| flag(pps.transquant_bypass, p::TRANSQUANT_BYPASS_ENABLED)
| flag(pps.tiles_enabled, p::TILES_ENABLED)
| flag(pps.entropy_coding_sync, p::ENTROPY_CODING_SYNC_ENABLED)
| flag(pps.loop_filter_across_slices, p::PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED)
| flag(pps.loop_filter_across_tiles, p::LOOP_FILTER_ACROSS_TILES_ENABLED)
| flag(sps.pcm_loop_filter_disabled, p::PCM_LOOP_FILTER_DISABLED)
| flag(sps.max_num_reorder == 0, p::NO_PIC_REORDERING);
let slice_parsing_fields = flag(pps.lists_modification_present, s::LISTS_MODIFICATION_PRESENT)
| flag(sps.long_term_refs_present, s::LONG_TERM_REF_PICS_PRESENT)
| flag(sps.temporal_mvp, s::SPS_TEMPORAL_MVP_ENABLED)
| flag(pps.cabac_init_present, s::CABAC_INIT_PRESENT)
| flag(pps.output_flag_present, s::OUTPUT_FLAG_PRESENT)
| flag(pps.dependent_slice_segments_enabled, s::DEPENDENT_SLICE_SEGMENTS_ENABLED)
| flag(pps.slice_chroma_qp_offsets_present, s::PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT)
| flag(sps.sao, s::SAMPLE_ADAPTIVE_OFFSET_ENABLED)
| flag(pps.deblocking_override_enabled, s::DEBLOCKING_FILTER_OVERRIDE_ENABLED)
| flag(pps.deblocking_disabled, s::PPS_DISABLE_DEBLOCKING_FILTER)
| flag(pps.slice_header_extension_present, s::SLICE_SEGMENT_HEADER_EXTENSION_PRESENT)
| flag(sh.nal.is_irap(), s::RAP_PIC)
| flag(sh.nal.is_idr(), s::IDR_PIC);
// tile sizes in CTBs, uniform spacing resolved
let layout = Layout::new(sps, pps).map_err(decode_err)?;
let mut column_width_minus1 = [0u16; 19];
let mut row_height_minus1 = [0u16; 21];
if pps.tiles_enabled {
let sizes = |bd: &[u32], out: &mut [u16]| -> Result<()> {
let n = bd.len().saturating_sub(1);
if n > out.len() {
return Err(unsupported("that many tiles"));
}
for (o, w) in out.iter_mut().zip(bd.windows(2)) {
*o = u16::try_from(w[1].saturating_sub(w[0]).saturating_sub(1)).unwrap_or(0);
}
Ok(())
};
sizes(&layout.col_bd, &mut column_width_minus1)?;
sizes(&layout.row_bd, &mut row_height_minus1)?;
}
let narrow = |v: u32| u16::try_from(v).map_err(|_| decode_err("picture too large"));
Ok(VAPictureParameterBufferHEVC {
CurrPic: VAPictureHEVC::INVALID,
ReferenceFrames: [VAPictureHEVC::INVALID; 15],
pic_width_in_luma_samples: narrow(sps.width)?,
pic_height_in_luma_samples: narrow(sps.height)?,
pic_fields,
sps_max_dec_pic_buffering_minus1: u8_of(sps.max_dec_pic_buffering.saturating_sub(1)),
bit_depth_luma_minus8: u8_of(sps.bit_depth_luma.saturating_sub(8)),
bit_depth_chroma_minus8: u8_of(sps.bit_depth_chroma.saturating_sub(8)),
pcm_sample_bit_depth_luma_minus1: u8_of(sps.pcm_bit_depth_luma.saturating_sub(1)),
pcm_sample_bit_depth_chroma_minus1: u8_of(sps.pcm_bit_depth_chroma.saturating_sub(1)),
log2_min_luma_coding_block_size_minus3: u8_of(sps.log2_min_cb.saturating_sub(3)),
log2_diff_max_min_luma_coding_block_size: u8_of(sps.log2_ctb.saturating_sub(sps.log2_min_cb)),
log2_min_transform_block_size_minus2: u8_of(sps.log2_min_tb.saturating_sub(2)),
log2_diff_max_min_transform_block_size: u8_of(sps.log2_max_tb.saturating_sub(sps.log2_min_tb)),
log2_min_pcm_luma_coding_block_size_minus3: u8_of(sps.log2_min_pcm.saturating_sub(3)),
log2_diff_max_min_pcm_luma_coding_block_size: u8_of(sps.log2_max_pcm.saturating_sub(sps.log2_min_pcm)),
max_transform_hierarchy_depth_intra: u8_of(sps.max_th_depth_intra),
max_transform_hierarchy_depth_inter: u8_of(sps.max_th_depth_inter),
init_qp_minus26: i8_of(pps.init_qp - 26),
diff_cu_qp_delta_depth: u8_of(pps.diff_cu_qp_delta_depth),
pps_cb_qp_offset: i8_of(pps.cb_qp_offset),
pps_cr_qp_offset: i8_of(pps.cr_qp_offset),
log2_parallel_merge_level_minus2: u8_of(pps.log2_parallel_merge_level.saturating_sub(2)),
num_tile_columns_minus1: u8_of(pps.num_tile_columns.saturating_sub(1)),
num_tile_rows_minus1: u8_of(pps.num_tile_rows.saturating_sub(1)),
column_width_minus1,
row_height_minus1,
slice_parsing_fields,
log2_max_pic_order_cnt_lsb_minus4: u8_of(sps.log2_max_poc_lsb.saturating_sub(4)),
num_short_term_ref_pic_sets: u8_of(u32::try_from(sps.st_rps.len()).unwrap_or(u32::MAX)),
num_long_term_ref_pic_sps: u8_of(u32::try_from(sps.lt_ref_pics.len()).unwrap_or(u32::MAX)),
num_ref_idx_l0_default_active_minus1: u8_of(pps.num_ref_idx_l0_default.saturating_sub(1)),
num_ref_idx_l1_default_active_minus1: u8_of(pps.num_ref_idx_l1_default.saturating_sub(1)),
pps_beta_offset_div2: i8_of(pps.beta_offset_div2),
pps_tc_offset_div2: i8_of(pps.tc_offset_div2),
num_extra_slice_header_bits: u8_of(pps.num_extra_slice_header_bits),
st_rps_bits: u32::try_from(sh.st_rps_bits).unwrap_or(0),
va_reserved: [0; ffi::VA_PADDING_MEDIUM],
})
}
/// The 8x8 base list `coded` (up-right diagonal order, as coded and stored) in raster order.
fn raster8(list: &ScalingList, size_id: usize, matrix_id: usize) -> [u8; 64] {
let mut l = list.clone();
let row = |s: usize| l.lists.get(s).and_then(|m| m.get(matrix_id)).copied();
if let Some(src) = row(size_id)
&& let Some(dst) = l.lists.get_mut(1).and_then(|m| m.get_mut(matrix_id))
{
*dst = src;
}
let mut out = [16u8; 64];
for (o, v) in out.iter_mut().zip(l.factor(1, matrix_id)) {
*o = v;
}
out
}
/// The scaling lists in force (`list`, or the default lists), in raster order.
pub fn iq_matrix(list: Option<&ScalingList>) -> VAIQMatrixBufferHEVC {
let defaults = ScalingList::default_lists();
let l = list.unwrap_or(&defaults);
let mut iq = VAIQMatrixBufferHEVC {
ScalingList4x4: [[16; 16]; 6],
ScalingList8x8: [[16; 64]; 6],
ScalingList16x16: [[16; 64]; 6],
ScalingList32x32: [[16; 64]; 2],
ScalingListDC16x16: [16; 6],
ScalingListDC32x32: [16; 2],
va_reserved: [0; ffi::VA_PADDING_LOW],
};
for m in 0..6 {
for (o, v) in iq.ScalingList4x4[m].iter_mut().zip(l.factor(0, m)) {
*o = v;
}
iq.ScalingList8x8[m] = raster8(l, 1, m);
iq.ScalingList16x16[m] = raster8(l, 2, m);
iq.ScalingListDC16x16[m] = l.dc[0][m];
}
// 32x32: matrixId 0 (intra) and 3 (inter)
for (i, m) in [0, 3].into_iter().enumerate() {
iq.ScalingList32x32[i] = raster8(l, 3, m);
iq.ScalingListDC32x32[i] = l.dc[1][m];
}
iq
}
/// Emulation prevention bytes among the first `rbsp_len` RBSP bytes of a NAL unit payload.
fn emulation_bytes_before(payload: &[u8], rbsp_len: usize) -> usize {
let (mut zeros, mut out, mut ep) = (0usize, 0usize, 0usize);
for &b in payload {
if out >= rbsp_len {
break;
}
if zeros >= 2 && b == 3 {
ep += 1;
zeros = 0;
continue;
}
zeros = if b == 0 { zeros + 1 } else { 0 };
out += 1;
}
ep
}
/// The slice parameters of the slice segment `sh` (NAL unit `nal`) with reference lists `refs`,
/// whose pictures are `ref_frames` (the surfaces of `ReferenceFrames`, in order).
pub fn slice_params(sh: &SliceHeader, nal: &[u8], refs: &[Vec<Ref<Surface>>; 2], ref_frames: &[usize]) -> Result<VASliceParameterBufferHEVC> {
use hevc_slice_bits as b;
let slice_type = match sh.slice_type {
SliceType::B => 0,
SliceType::P => 1,
SliceType::I => 2,
};
let flags = flag(sh.dependent, b::DEPENDENT_SLICE_SEGMENT)
| slice_type << b::SLICE_TYPE
| flag(sh.sao_luma, b::SLICE_SAO_LUMA)
| flag(sh.sao_chroma, b::SLICE_SAO_CHROMA)
| flag(sh.mvd_l1_zero, b::MVD_L1_ZERO)
| flag(sh.cabac_init_flag, b::CABAC_INIT)
| flag(sh.temporal_mvp, b::SLICE_TEMPORAL_MVP_ENABLED)
| flag(sh.deblocking_disabled, b::SLICE_DEBLOCKING_FILTER_DISABLED)
| flag(sh.collocated_from_l0, b::COLLOCATED_FROM_L0)
| flag(sh.loop_filter_across_slices, b::SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED);
let header_bytes = sh.data_offset.checked_add(2).ok_or_else(|| decode_err("slice header too long"))?;
let ep = emulation_bytes_before(nal.get(2..).unwrap_or_default(), sh.data_offset);
let mut s = VASliceParameterBufferHEVC {
slice_data_size: u32::try_from(nal.len()).map_err(|_| decode_err("slice too large"))?,
slice_data_offset: 0,
slice_data_flag: ffi::VA_SLICE_DATA_FLAG_ALL,
slice_data_byte_offset: u32::try_from(header_bytes).map_err(|_| decode_err("slice header too long"))?,
slice_segment_address: sh.segment_address,
RefPicList: [[0xFF; 15]; 2],
LongSliceFlags: flags,
collocated_ref_idx: u8_of(sh.collocated_ref_idx),
num_ref_idx_l0_active_minus1: u8_of(sh.num_ref_idx[0].saturating_sub(1)),
num_ref_idx_l1_active_minus1: u8_of(sh.num_ref_idx[1].saturating_sub(1)),
slice_qp_delta: i8_of(sh.qp_delta),
slice_cb_qp_offset: i8_of(sh.cb_qp_offset),
slice_cr_qp_offset: i8_of(sh.cr_qp_offset),
slice_beta_offset_div2: i8_of(sh.beta_offset_div2),
slice_tc_offset_div2: i8_of(sh.tc_offset_div2),
luma_log2_weight_denom: 0,
delta_chroma_log2_weight_denom: 0,
delta_luma_weight_l0: [0; 15],
luma_offset_l0: [0; 15],
delta_chroma_weight_l0: [[0; 2]; 15],
ChromaOffsetL0: [[0; 2]; 15],
delta_luma_weight_l1: [0; 15],
luma_offset_l1: [0; 15],
delta_chroma_weight_l1: [[0; 2]; 15],
ChromaOffsetL1: [[0; 2]; 15],
five_minus_max_num_merge_cand: u8_of(5u32.saturating_sub(sh.max_num_merge_cand)),
num_entry_point_offsets: u16::try_from(sh.entry_points.len()).map_err(|_| decode_err("too many entry points"))?,
entry_offset_to_subset_array: 0,
slice_data_num_emu_prevn_bytes: u16::try_from(ep).unwrap_or(u16::MAX),
va_reserved: [0; ffi::VA_PADDING_LOW - 2],
};
for (l, list) in refs.iter().enumerate() {
if list.len() > 15 {
return Err(decode_err("reference list longer than 15"));
}
for (i, r) in list.iter().enumerate() {
let idx = ref_frames.iter().position(|&f| f == r.frame.index).ok_or_else(|| decode_err("a reference is not among the reference frames"))?;
s.RefPicList[l][i] = u8::try_from(idx).unwrap_or(0xFF);
}
}
if let Some(w) = &sh.pwt {
let luma_denom = i32::try_from(w.luma_log2_denom).unwrap_or(0);
let chroma_denom = i32::try_from(w.chroma_log2_denom).unwrap_or(0);
s.luma_log2_weight_denom = u8_of(w.luma_log2_denom);
s.delta_chroma_log2_weight_denom = i8_of(chroma_denom - luma_denom);
let lists = [
(&w.l[0], &mut s.delta_luma_weight_l0, &mut s.luma_offset_l0, &mut s.delta_chroma_weight_l0, &mut s.ChromaOffsetL0),
(&w.l[1], &mut s.delta_luma_weight_l1, &mut s.luma_offset_l1, &mut s.delta_chroma_weight_l1, &mut s.ChromaOffsetL1),
];
for (entries, lw, lo, cw, co) in lists {
for (i, e) in entries.iter().take(15).enumerate() {
lw[i] = i8_of(e.luma.0 - (1 << luma_denom));
lo[i] = i8_of(e.luma.1);
for c in 0..2 {
cw[i][c] = i8_of(e.chroma[c].0 - (1 << chroma_denom));
co[i][c] = i8_of(e.chroma[c].1);
}
}
}
}
Ok(s)
}

217
crates/platform/src/vaapi/mod.rs Normal file
View File

@ -0,0 +1,217 @@
//! Hardware decoding through VA-API (Linux: Intel, AMD and other Mesa / vendor drivers).
//!
//! VA-API decoding is stateless: the application parses the stream and keeps the decoded picture
//! buffer, and the GPU decodes one picture's slice data at a time into a surface. [`h264`] does the
//! host side with the software decoder's own parsers and DPB (so the two decide alike), [`va`]
//! drives libva (loaded at run time; no libva means no hardware decoder, never a failure to start),
//! and [`VaDecoder`] puts them together as a [`VideoDecoder`]. H.264 8-bit 4:2:0 progressive
//! (Constrained Baseline / Main / High) and HEVC 4:2:0 (Main, Main 10, Main Still Picture), the
//! latter through [`hevc`] the same way; everything else stays in software.
//!
//! The FFI is in [`ffi`] (declarations) and `va` (calls); the rest of this module is safe code.
pub mod ffi;
pub mod h264;
pub mod hevc;
#[cfg(target_os = "linux")]
#[allow(unsafe_code)]
pub mod va;
#[cfg(test)]
mod abi_tests;
#[cfg(test)]
mod tests;
#[cfg(target_os = "linux")]
pub use linux::VaDecoder;
use filmcraft_frame::VideoFrame;
/// The hardware a front end ([`h264::Front`], [`hevc::Front`]) drives: a fixed set of surfaces,
/// picture decoding into one of them, and reading one back or filling it.
pub trait Accel: Send {
/// The VA surface ids of the decoder's surfaces; front ends refer to them by index.
fn surfaces(&self) -> &[ffi::VASurfaceID];
/// Decode one H.264 picture (its slices: parameters and the NAL unit, header byte and
/// emulation prevention included) into surface `target` (an index into [`Accel::surfaces`]).
fn decode_h264(
&mut self,
target: usize,
pic: &ffi::VAPictureParameterBufferH264,
iq: &ffi::VAIQMatrixBufferH264,
slices: &[(ffi::VASliceParameterBufferH264, Vec<u8>)],
) -> std::result::Result<(), String>;
/// Decode one HEVC picture (its slice segments: parameters and the NAL unit as stored) into
/// surface `target`; `iq` when scaling lists are enabled.
fn decode_hevc(
&mut self,
target: usize,
pic: &ffi::VAPictureParameterBufferHEVC,
iq: Option<&ffi::VAIQMatrixBufferHEVC>,
slices: &[(ffi::VASliceParameterBufferHEVC, Vec<u8>)],
) -> std::result::Result<(), String>;
/// The picture in surface `index`, once decoded.
fn read(&mut self, index: usize) -> std::result::Result<VideoFrame, String>;
/// Make surface `index` a copy of surface `from`, or mid-gray (every sample half the range)
/// when `None`: the stand-ins for reference pictures that were never decoded, as the software
/// decoders make them.
fn fill(&mut self, index: usize, from: Option<usize>) -> std::result::Result<(), String>;
}
#[cfg(target_os = "linux")]
mod linux {
use filmcraft_bitstream::unescape_rbsp;
use filmcraft_codecs::hw::{NalCodec, NalStreamInfo};
use filmcraft_codecs::{CodecError, DecodedFrame, Result, VideoDecoder};
use super::va::Session;
use super::{ffi, h264, hevc};
use crate::biplanar::Geometry;
/// The VA-API profiles that decode an H.264 stream of `profile_idc`, best fit first (a Main or
/// High decoder also decodes what the smaller profiles allow).
pub(crate) fn h264_profiles(profile_idc: u8) -> Option<&'static [ffi::VAProfile]> {
Some(match profile_idc {
66 => &[ffi::VAProfileH264ConstrainedBaseline, ffi::VAProfileH264Main, ffi::VAProfileH264High],
77 => &[ffi::VAProfileH264Main, ffi::VAProfileH264High],
100 => &[ffi::VAProfileH264High],
_ => return None,
})
}
/// The VA-API profiles that decode an HEVC stream of `general_profile_idc` (Main, Main 10,
/// Main Still Picture) at `bits`, best fit first.
pub(crate) fn hevc_profiles(profile_idc: u8, bits: u32) -> Option<&'static [ffi::VAProfile]> {
Some(match (profile_idc, bits) {
(1 | 3, 8) => &[ffi::VAProfileHEVCMain, ffi::VAProfileHEVCMain10],
(2, 8) => &[ffi::VAProfileHEVCMain10, ffi::VAProfileHEVCMain],
(2, 10) => &[ffi::VAProfileHEVCMain10],
_ => return None,
})
}
enum Front {
H264(Box<h264::Front<Session>>),
Hevc(Box<hevc::Front<Session>>),
}
/// A VA-API decoder for one H.264 or HEVC stream.
pub struct VaDecoder {
front: Front,
info: NalStreamInfo,
name: String,
fed: u64,
fail_after: Option<u64>,
}
/// The first parameter set of NAL unit type `ty` among `sets`, without its header (`header`
/// bytes) and emulation prevention.
fn parameter_set(sets: &[Vec<u8>], header: usize, ty: impl Fn(u8) -> bool) -> Option<Vec<u8>> {
sets.iter().find(|n| n.first().is_some_and(|&h| ty(h))).and_then(|n| Some(unescape_rbsp(n.get(header..)?)))
}
impl VaDecoder {
/// A decoder for the stream `info` describes, or why VA-API does not take it on this system.
pub fn new(info: NalStreamInfo) -> std::result::Result<Self, String> {
let bits = info.bit_depth_luma;
if info.chroma_format_idc != 1 || info.bit_depth_chroma != bits || info.interlaced {
return Err(format!(
"{} chroma format {} at {bits}/{}-bit is not taken",
if info.interlaced { "interlaced" } else { "progressive" },
info.chroma_format_idc,
info.bit_depth_chroma
));
}
let geometry = Geometry { crop: info.crop, bits, color: info.color, par: info.par };
let (front, codec, vendor) = match info.codec {
NalCodec::H264 => {
if bits != 8 {
return Err(format!("{bits}-bit H.264 is not taken"));
}
let profiles = h264_profiles(info.profile_idc).ok_or_else(|| format!("H.264 profile_idc {} is not taken", info.profile_idc))?;
let rbsp = parameter_set(&info.parameter_sets, 1, |h| h & 0x1f == 7).ok_or("no SPS")?;
let sps = filmcraft_h264::params::Sps::parse(&rbsp).map_err(|e| e.to_string())?;
sps.check_supported().map_err(|e| e.to_string())?;
let mbs = (sps.pic_width_in_mbs, sps.frame_height_in_mbs());
// the DPB, the picture being decoded, and one spare
let count = sps.max_dpb_frames().saturating_add(2).min(18);
let session = Session::new(profiles, (sps.width(), sps.height()), count, geometry)?;
let vendor = session.vendor().to_string();
(Front::H264(Box::new(h264::Front::new(session, info.length_size, mbs, &info.parameter_sets).map_err(|e| e.to_string())?)), "H.264", vendor)
}
NalCodec::Hevc => {
let profiles =
hevc_profiles(info.profile_idc, bits).ok_or_else(|| format!("HEVC profile {} at {bits}-bit is not taken", info.profile_idc))?;
let rbsp = parameter_set(&info.parameter_sets, 2, |h| (h >> 1) & 0x3f == 33).ok_or("no SPS")?;
let sps = filmcraft_hevc::params::Sps::parse(&rbsp).map_err(|e| e.to_string())?;
sps.check_supported().map_err(|e| e.to_string())?;
if sps.range_extension {
return Err("HEVC range extensions are not taken".into());
}
let count = (sps.max_dec_pic_buffering as usize).saturating_add(2).min(18);
let size = (sps.width, sps.height);
let session = Session::new(profiles, size, count, geometry)?;
let vendor = session.vendor().to_string();
(
Front::Hevc(Box::new(hevc::Front::new(session, info.length_size, size, bits, &info.parameter_sets).map_err(|e| e.to_string())?)),
"HEVC",
vendor,
)
}
};
let name = format!("VA-API {codec} ({vendor})");
Ok(Self { front, info, name, fed: 0, fail_after: None })
}
/// Test hook: fail every decode call after the first `n` (a GPU lost mid-stream).
pub fn fail_after(&mut self, n: u64) {
self.fail_after = Some(n);
}
}
impl VideoDecoder for VaDecoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
self.fed += 1;
if self.fail_after.is_some_and(|n| self.fed > n) {
return Err(CodecError::Decode("VA-API decoder failed (test hook)".into()));
}
match &mut self.front {
Front::H264(f) => f.decode(sample, pts),
Front::Hevc(f) => f.decode(sample, pts),
}
}
fn flush(&mut self) -> Vec<DecodedFrame> {
let flushed = match &mut self.front {
Front::H264(f) => f.flush(),
Front::Hevc(f) => f.flush(),
};
match flushed {
Ok(out) => out,
Err(e) => {
log::warn!("{}: {e} while flushing", self.name);
Vec::new()
}
}
}
fn reset(&mut self) {
match &mut self.front {
Front::H264(f) => f.reset(),
Front::Hevc(f) => f.reset(),
}
}
fn name(&self) -> &str {
&self.name
}
fn is_random_access(&self, sample: &[u8]) -> Option<bool> {
self.info.is_random_access(sample)
}
fn is_disposable(&self, sample: &[u8]) -> bool {
self.info.is_disposable(sample)
}
}
}

381
crates/platform/src/vaapi/tests.rs Normal file
View File

@ -0,0 +1,381 @@
//! The H.264 and HEVC front ends against a recording stand-in for the hardware, on every system:
//! for x264 streams with B-pyramids, several references, weighted prediction, several slices and
//! open GOPs, and x265 streams (Main and Main 10, open GOP with RASL pictures, B-pyramids),
//! the pictures come out in the software decoder's order with its pts (whole stream, after seeks,
//! after a mid-stream flush), and every buffer handed to the hardware is consistent: the current
//! picture's surface is free, every reference is a decoded surface listed in the DPB, reference
//! lists have the active number of entries, and slice data offsets lie inside the slice.
use std::collections::BTreeSet;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use filmcraft_codecs::DecodedFrame;
use filmcraft_codecs::hw::NalStreamInfo;
use filmcraft_frame::VideoFrame;
use super::Accel;
use super::ffi::{self, VAIQMatrixBufferH264, VAPictureH264, VAPictureParameterBufferH264, VASliceParameterBufferH264};
use super::h264::Front;
use crate::biplanar::{self, Biplanar, Geometry};
/// Records what it is asked to do and checks it; "decodes" nothing.
struct Recorder {
ids: Vec<ffi::VASurfaceID>,
decoded: Vec<bool>,
pictures: usize,
frame: VideoFrame,
}
impl Recorder {
fn new(count: usize) -> Self {
let g = Geometry { crop: (0, 0, 2, 2), bits: 8, color: Default::default(), par: (1, 1) };
let frame = biplanar::to_frame(&Biplanar { luma: &[128; 4], chroma: &[128; 2], stride: 2, rows: 2 }, &g).unwrap();
Self { ids: (0..count as u32).map(|i| 0x100 + i).collect(), decoded: vec![false; count], pictures: 0, frame }
}
}
fn valid(p: &VAPictureH264) -> bool {
p.flags & ffi::VA_PICTURE_H264_INVALID == 0
}
impl Accel for Recorder {
fn surfaces(&self) -> &[ffi::VASurfaceID] {
&self.ids
}
fn decode_hevc(
&mut self,
target: usize,
pic: &ffi::VAPictureParameterBufferHEVC,
_iq: Option<&ffi::VAIQMatrixBufferHEVC>,
slices: &[(ffi::VASliceParameterBufferHEVC, Vec<u8>)],
) -> Result<(), String> {
assert_eq!(pic.CurrPic.picture_id, self.ids[target], "current picture surface");
let valid = |p: &ffi::VAPictureHEVC| p.flags & ffi::VA_PICTURE_HEVC_INVALID == 0;
let n = pic.ReferenceFrames.iter().take_while(|p| valid(p)).count();
assert!(pic.ReferenceFrames[n..].iter().all(|p| !valid(p)), "reference frames packed at the front");
let ids: BTreeSet<_> = pic.ReferenceFrames[..n].iter().map(|r| r.picture_id).collect();
assert_eq!(ids.len(), n, "each reference frame once");
assert!(!ids.contains(&pic.CurrPic.picture_id), "the target is not a reference");
for r in &pic.ReferenceFrames[..n] {
let i = self.ids.iter().position(|&id| id == r.picture_id).expect("reference is a decoder surface");
assert!(self.decoded[i], "reference surface {i} holds a picture");
}
let rps = ffi::VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE | ffi::VA_PICTURE_HEVC_RPS_ST_CURR_AFTER | ffi::VA_PICTURE_HEVC_RPS_LT_CURR;
assert!(!slices.is_empty());
let last = ffi::hevc_slice_bits::LAST_SLICE_OF_PIC;
for (k, (s, data)) in slices.iter().enumerate() {
assert_eq!(s.slice_data_size as usize, data.len());
assert!((s.slice_data_byte_offset as usize) <= data.len(), "slice data starts inside the slice");
assert_eq!(s.LongSliceFlags >> last & 1 == 1, k + 1 == slices.len(), "LastSliceOfPic on the last slice only");
let ty = (s.LongSliceFlags >> ffi::hevc_slice_bits::SLICE_TYPE) & 3;
let lists: &[(usize, u8)] = match ty {
2 => &[],
1 => &[(0, s.num_ref_idx_l0_active_minus1)],
_ => &[(0, s.num_ref_idx_l0_active_minus1), (1, s.num_ref_idx_l1_active_minus1)],
};
for &(l, minus1) in lists {
let len = minus1 as usize + 1;
for &i in &s.RefPicList[l][..len] {
assert!((i as usize) < n, "list entry {i} is a reference frame");
assert!(pic.ReferenceFrames[i as usize].flags & rps != 0, "list entries are in the current RPS");
}
assert!(s.RefPicList[l][len..].iter().all(|&i| i == 0xFF), "nothing past the active entries");
}
}
self.decoded[target] = true;
self.pictures += 1;
Ok(())
}
fn decode_h264(
&mut self,
target: usize,
pic: &VAPictureParameterBufferH264,
_iq: &VAIQMatrixBufferH264,
slices: &[(VASliceParameterBufferH264, Vec<u8>)],
) -> Result<(), String> {
assert_eq!(pic.CurrPic.picture_id, self.ids[target], "current picture surface");
let refs: Vec<_> = pic.ReferenceFrames.iter().filter(|r| valid(r)).collect();
let ref_ids: BTreeSet<_> = refs.iter().map(|r| r.picture_id).collect();
assert_eq!(ref_ids.len(), refs.len(), "each reference frame once");
assert!(!ref_ids.contains(&pic.CurrPic.picture_id), "the target is not a reference");
for r in &refs {
let i = self.ids.iter().position(|&id| id == r.picture_id).expect("reference is a decoder surface");
assert!(self.decoded[i], "reference surface {i} holds a picture");
assert!(r.flags & (ffi::VA_PICTURE_H264_SHORT_TERM_REFERENCE | ffi::VA_PICTURE_H264_LONG_TERM_REFERENCE) != 0, "reference flags");
}
assert!(!slices.is_empty());
for (s, data) in slices {
assert_eq!(s.slice_data_size as usize, data.len());
assert!((s.slice_data_bit_offset as usize) < data.len() * 8, "slice data starts inside the slice");
let ty = s.slice_type;
let lists: &[(&[VAPictureH264; 32], u8)] = match ty {
2 => &[],
0 => &[(&s.RefPicList0, s.num_ref_idx_l0_active_minus1)],
_ => &[(&s.RefPicList0, s.num_ref_idx_l0_active_minus1), (&s.RefPicList1, s.num_ref_idx_l1_active_minus1)],
};
for (list, minus1) in lists {
let n = *minus1 as usize + 1;
assert!(list[..n].iter().all(valid), "{n} active entries");
assert!(list[n..].iter().all(|p| !valid(p)), "nothing past the active entries");
assert!(list[..n].iter().all(|p| ref_ids.contains(&p.picture_id)), "list entries are reference frames");
}
}
self.decoded[target] = true;
self.pictures += 1;
Ok(())
}
fn read(&mut self, index: usize) -> Result<VideoFrame, String> {
assert!(self.decoded[index], "read back a decoded surface");
Ok(self.frame.clone())
}
fn fill(&mut self, index: usize, from: Option<usize>) -> Result<(), String> {
assert!(from.is_none_or(|f| self.decoded[f]));
self.decoded[index] = true;
Ok(())
}
}
/// (file, x264 profile, x264 parameters).
const STREAMS: &[(&str, &str, &str)] = &[
("va_front_high.mp4", "high", "bframes=3:b-pyramid=normal:keyint=12:min-keyint=12:scenecut=0:ref=4:weightp=2"),
("va_front_slices.mp4", "baseline", "keyint=12:min-keyint=12:scenecut=0:slices=4:ref=2"),
("va_front_open_gop.mp4", "high", "bframes=3:b-pyramid=strict:keyint=12:min-keyint=12:scenecut=0:open-gop=1:direct=temporal"),
];
fn fixture(ff: &std::path::Path, name: &str, profile: &str, params: &str) -> Option<PathBuf> {
let out = filmcraft_testkit::fixtures_dir("platform").join(name);
let src = "testsrc2=s=320x184:r=24:d=2";
filmcraft_testkit::fixtures::generate(&out, |tmp| {
Command::new(ff)
.args(["-y", "-v", "error", "-f", "lavfi", "-i", src, "-c:v", "libx264", "-profile:v", profile, "-x264-params", params, "-pix_fmt", "yuv420p"])
.arg(tmp)
.stdin(Stdio::null())
.status()
.is_ok_and(|s| s.success())
})
}
struct Stream {
entry: filmcraft_isobmff::SampleEntry,
samples: Vec<(Vec<u8>, i64)>,
sync: Vec<bool>,
}
fn read_stream(path: &std::path::Path) -> Stream {
let bytes = std::fs::read(path).unwrap();
let file = filmcraft_isobmff::open(bytes.clone()).unwrap();
let t = file.track_of_kind(filmcraft_isobmff::TrackKind::Video).unwrap();
let track = &file.tracks[t];
let samples = (0..track.samples.len()).map(|i| (file.read_sample(&bytes, t, i).unwrap(), track.samples[i].pts)).collect();
Stream { entry: track.entries[0].clone(), samples, sync: track.samples.iter().map(|s| s.is_sync).collect() }
}
fn front(s: &Stream) -> Front<Recorder> {
let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap();
let mbs = (info.coded.0 / 16, info.coded.1 / 16);
Front::new(Recorder::new(18), info.length_size, mbs, &info.parameter_sets).unwrap()
}
fn pts(frames: &[DecodedFrame]) -> Vec<i64> {
frames.iter().map(|f| f.pts).collect()
}
fn run_front(f: &mut Front<Recorder>, samples: &[(Vec<u8>, i64)]) -> Vec<i64> {
let mut out = Vec::new();
for (s, p) in samples {
out.extend(pts(&f.decode(s, *p).unwrap()));
}
out.extend(pts(&f.flush().unwrap()));
out
}
fn run_software(d: &mut dyn filmcraft_codecs::VideoDecoder, samples: &[(Vec<u8>, i64)]) -> Vec<i64> {
let mut out = Vec::new();
for (s, p) in samples {
out.extend(pts(&d.decode(s, *p).unwrap()));
}
out.extend(pts(&d.flush()));
out
}
#[test]
fn output_order_and_buffers_match_the_software_decoder() {
let ff = filmcraft_testkit::require_ffmpeg!();
for (name, profile, params) in STREAMS {
let Some(path) = fixture(&ff, name, profile, params) else { continue };
let s = read_stream(&path);
let mut f = front(&s);
let mut sw = filmcraft_codecs::software_video_decoder(&s.entry).unwrap();
let a = run_front(&mut f, &s.samples);
assert_eq!(a, run_software(sw.as_mut(), &s.samples), "{name}");
assert_eq!(f.accel().pictures, s.samples.len(), "{name}: every picture decoded");
// seeks to every sync sample (open-GOP I pictures included: frame_num gap stand-ins)
for k in (1..s.samples.len()).filter(|&i| s.sync[i]) {
f.reset();
sw.reset();
let end = (k + 13).min(s.samples.len());
assert_eq!(run_front(&mut f, &s.samples[k..end]), run_software(sw.as_mut(), &s.samples[k..end]), "{name} from sample {k}");
}
// a flush mid-GOP, then on
f.reset();
sw.reset();
let mid = s.samples.len() / 2 + 1;
let mut a = run_front(&mut f, &s.samples[..mid]);
a.extend(run_front(&mut f, &s.samples[mid..]));
let mut b = run_software(sw.as_mut(), &s.samples[..mid]);
b.extend(run_software(sw.as_mut(), &s.samples[mid..]));
assert_eq!(a, b, "{name} with a flush at {mid}");
}
}
/// Damaged samples give errors (the hybrid decoder then continues in software), never a panic.
#[test]
fn damaged_samples_are_errors() {
let ff = filmcraft_testkit::require_ffmpeg!();
let (name, profile, params) = STREAMS[0];
let Some(path) = fixture(&ff, name, profile, params) else { return };
let s = read_stream(&path);
let mut seed = 0x2545_F491_4F6C_DD1Du64;
let mut next = || {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
seed
};
for round in 0..40 {
let mut f = front(&s);
for (i, (smp, p)) in s.samples.iter().enumerate() {
let mut m = smp.clone();
if i % 3 == round % 3 {
match next() % 3 {
0 => m.truncate(next() as usize % m.len().max(1)),
1 => {
let at = next() as usize % m.len().max(1);
if let Some(b) = m.get_mut(at) {
*b ^= 1 << (next() % 8);
}
}
_ => m.iter_mut().skip(4).take(6).for_each(|b| *b = next() as u8),
}
}
if f.decode(&m, *p).is_err() {
f.reset();
}
}
let _ = f.flush();
}
}
/// (file, pixel format, x265 parameters).
const HEVC_STREAMS: &[(&str, &str, &str)] = &[
("va_front_hevc_open_gop.mp4", "yuv420p", "log-level=error:bframes=4:b-pyramid=1:keyint=12:min-keyint=12:scenecut=0:open-gop=1:ref=3"),
("va_front_hevc_main10.mp4", "yuv420p10le", "log-level=error:bframes=3:keyint=12:min-keyint=12:scenecut=0:slices=2:weightp=1"),
];
fn hevc_fixture(ff: &std::path::Path, name: &str, pix_fmt: &str, params: &str) -> Option<PathBuf> {
let out = filmcraft_testkit::fixtures_dir("platform").join(name);
let src = "testsrc2=s=320x184:r=24:d=2";
let profile = if pix_fmt.contains("10") { "main10" } else { "main" };
filmcraft_testkit::fixtures::generate(&out, |tmp| {
Command::new(ff)
.args([
"-y",
"-v",
"error",
"-f",
"lavfi",
"-i",
src,
"-c:v",
"libx265",
"-profile:v",
profile,
"-tag:v",
"hvc1",
"-x265-params",
params,
"-pix_fmt",
pix_fmt,
])
.arg(tmp)
.stdin(Stdio::null())
.status()
.is_ok_and(|s| s.success())
})
}
fn hevc_front(s: &Stream) -> super::hevc::Front<Recorder> {
let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap();
super::hevc::Front::new(Recorder::new(18), info.length_size, info.coded, info.bit_depth_luma, &info.parameter_sets).unwrap()
}
fn run_hevc(f: &mut super::hevc::Front<Recorder>, samples: &[(Vec<u8>, i64)]) -> Vec<i64> {
let mut out = Vec::new();
for (s, p) in samples {
out.extend(pts(&f.decode(s, *p).unwrap()));
}
out.extend(pts(&f.flush().unwrap()));
out
}
#[test]
fn hevc_output_order_and_buffers_match_the_software_decoder() {
let ff = filmcraft_testkit::require_ffmpeg!();
for (name, pix_fmt, params) in HEVC_STREAMS {
let Some(path) = hevc_fixture(&ff, name, pix_fmt, params) else { continue };
let s = read_stream(&path);
let mut f = hevc_front(&s);
let mut sw = filmcraft_codecs::software_video_decoder(&s.entry).unwrap();
assert_eq!(run_hevc(&mut f, &s.samples), run_software(sw.as_mut(), &s.samples), "{name}");
// seeks to every sync sample (open-GOP CRA pictures included: RASL pictures left out)
for k in (1..s.samples.len()).filter(|&i| s.sync[i]) {
f.reset();
sw.reset();
let end = (k + 13).min(s.samples.len());
assert_eq!(run_hevc(&mut f, &s.samples[k..end]), run_software(sw.as_mut(), &s.samples[k..end]), "{name} from sample {k}");
}
}
}
/// Damaged HEVC samples give errors, never a panic.
#[test]
fn hevc_damaged_samples_are_errors() {
let ff = filmcraft_testkit::require_ffmpeg!();
let (name, pix_fmt, params) = HEVC_STREAMS[0];
let Some(path) = hevc_fixture(&ff, name, pix_fmt, params) else { return };
let s = read_stream(&path);
let mut seed = 0x9E37_79B9_7F4A_7C15u64;
let mut next = || {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
seed
};
for round in 0..40 {
let mut f = hevc_front(&s);
for (i, (smp, p)) in s.samples.iter().enumerate() {
let mut m = smp.clone();
if i % 3 == round % 3 {
match next() % 3 {
0 => m.truncate(next() as usize % m.len().max(1)),
1 => {
let at = next() as usize % m.len().max(1);
if let Some(b) = m.get_mut(at) {
*b ^= 1 << (next() % 8);
}
}
_ => m.iter_mut().skip(4).take(6).for_each(|b| *b = next() as u8),
}
}
if f.decode(&m, *p).is_err() {
f.reset();
}
}
let _ = f.flush();
}
}

525
crates/platform/src/vaapi/va.rs Normal file
View File

@ -0,0 +1,525 @@
//! libva itself (Linux): `libva.so.2` and `libva-drm.so.2` are loaded at run time (a system without
//! them simply has no hardware decoder), a display is opened on a DRM render node, and a [`Session`]
//! (decode configuration, context and surfaces for one stream) decodes pictures and reads them back
//! as NV12 images.
//!
//! FFI module (docs/adr/0001-platform-ffi.md): every `unsafe` block has a `// SAFETY:` comment, the
//! public API is safe, and the one callback into Rust (libva's error messages) runs under
//! `catch_unwind`.
use std::ffi::{CStr, c_char, c_int, c_uint, c_void};
use std::fs::{File, OpenOptions};
use std::os::fd::AsRawFd;
use std::sync::OnceLock;
use filmcraft_frame::VideoFrame;
use libloading::Library;
use super::Accel;
use super::ffi::{self, VAConfigAttrib, VAImage, VAImageFormat, VASurfaceID};
use crate::biplanar::{self, Biplanar, Geometry};
/// The libva entry points (the libraries stay loaded for the life of the process).
struct Api {
_va: Library,
_drm: Library,
initialize: ffi::vaInitialize,
terminate: ffi::vaTerminate,
error_str: ffi::vaErrorStr,
vendor_string: ffi::vaQueryVendorString,
set_info_callback: ffi::vaSetInfoCallback,
set_error_callback: ffi::vaSetErrorCallback,
get_config_attributes: ffi::vaGetConfigAttributes,
create_config: ffi::vaCreateConfig,
destroy_config: ffi::vaDestroyConfig,
create_surfaces: ffi::vaCreateSurfaces,
destroy_surfaces: ffi::vaDestroySurfaces,
create_context: ffi::vaCreateContext,
destroy_context: ffi::vaDestroyContext,
create_buffer: ffi::vaCreateBuffer,
destroy_buffer: ffi::vaDestroyBuffer,
begin_picture: ffi::vaBeginPicture,
render_picture: ffi::vaRenderPicture,
end_picture: ffi::vaEndPicture,
sync_surface: ffi::vaSyncSurface,
create_image: ffi::vaCreateImage,
destroy_image: ffi::vaDestroyImage,
get_image: ffi::vaGetImage,
put_image: ffi::vaPutImage,
map_buffer: ffi::vaMapBuffer,
unmap_buffer: ffi::vaUnmapBuffer,
get_display_drm: ffi::vaGetDisplayDRM,
}
static API: OnceLock<Result<Api, String>> = OnceLock::new();
fn api() -> Result<&'static Api, String> {
API.get_or_init(load).as_ref().map_err(Clone::clone)
}
fn load() -> Result<Api, String> {
// SAFETY: loading libva runs its library initialisers, which only set up libva's own state;
// it is the system's VA-API runtime, loaded the way every VA-API client loads it.
let va = unsafe { Library::new("libva.so.2") }.map_err(|e| format!("libva.so.2: {e}"))?;
// SAFETY: as above, for libva's DRM display library.
let drm = unsafe { Library::new("libva-drm.so.2") }.map_err(|e| format!("libva-drm.so.2: {e}"))?;
macro_rules! sym {
($lib:expr, $name:ident) => {{
// SAFETY: `ffi::$name` is the C signature of the libva function of that name,
// transcribed from va.h and checked against it (abi_tests.rs). The pointer is copied
// out; the library it points into is kept loaded in `Api` for the life of the process.
let s = unsafe { $lib.get::<ffi::$name>(concat!(stringify!($name), "\0").as_bytes()) };
*s.map_err(|e| format!("{}: {e}", stringify!($name)))?
}};
}
Ok(Api {
initialize: sym!(va, vaInitialize),
terminate: sym!(va, vaTerminate),
error_str: sym!(va, vaErrorStr),
vendor_string: sym!(va, vaQueryVendorString),
set_info_callback: sym!(va, vaSetInfoCallback),
set_error_callback: sym!(va, vaSetErrorCallback),
get_config_attributes: sym!(va, vaGetConfigAttributes),
create_config: sym!(va, vaCreateConfig),
destroy_config: sym!(va, vaDestroyConfig),
create_surfaces: sym!(va, vaCreateSurfaces),
destroy_surfaces: sym!(va, vaDestroySurfaces),
create_context: sym!(va, vaCreateContext),
destroy_context: sym!(va, vaDestroyContext),
create_buffer: sym!(va, vaCreateBuffer),
destroy_buffer: sym!(va, vaDestroyBuffer),
begin_picture: sym!(va, vaBeginPicture),
render_picture: sym!(va, vaRenderPicture),
end_picture: sym!(va, vaEndPicture),
sync_surface: sym!(va, vaSyncSurface),
create_image: sym!(va, vaCreateImage),
destroy_image: sym!(va, vaDestroyImage),
get_image: sym!(va, vaGetImage),
put_image: sym!(va, vaPutImage),
map_buffer: sym!(va, vaMapBuffer),
unmap_buffer: sym!(va, vaUnmapBuffer),
get_display_drm: sym!(drm, vaGetDisplayDRM),
_va: va,
_drm: drm,
})
}
/// libva's error messages, into our log (instead of stderr).
extern "C" fn on_error(_user: *mut c_void, message: *const c_char) {
let _ = std::panic::catch_unwind(|| {
if message.is_null() {
return;
}
// SAFETY: libva passes a NUL-terminated message that is valid for the duration of the call.
let text = unsafe { CStr::from_ptr(message) }.to_string_lossy();
log::warn!("libva: {}", text.trim_end());
});
}
/// An initialised VA display on a DRM render node.
pub struct Display {
api: &'static Api,
dpy: ffi::VADisplay,
vendor: String,
/// The render node; open for as long as the display uses it.
_device: File,
}
// SAFETY: a VA display is not tied to the thread that opened it, and a `Display` is only used
// through its single owner (`&mut` or `&` of one `Session` at a time), never concurrently.
unsafe impl Send for Display {}
impl Display {
/// The first DRM render node with a working VA-API driver.
pub fn open() -> Result<Self, String> {
let api = api()?;
let mut why = "no DRM render node (/dev/dri/renderD*)".to_string();
for n in 128..144 {
let path = format!("/dev/dri/renderD{n}");
let Ok(file) = OpenOptions::new().read(true).write(true).open(&path) else { continue };
match Self::init(api, file) {
Ok(d) => return Ok(d),
Err(e) => why = format!("{path}: {e}"),
}
}
Err(why)
}
fn init(api: &'static Api, device: File) -> Result<Self, String> {
// SAFETY: the descriptor is an open render node, kept open by `device` (moved into the
// returned display) for as long as the display exists.
let dpy = unsafe { (api.get_display_drm)(device.as_raw_fd()) };
if dpy.is_null() {
return Err("vaGetDisplayDRM failed".into());
}
// From here on `Drop` terminates the display.
let mut d = Self { api, dpy, vendor: String::new(), _device: device };
// SAFETY: `dpy` is a display from vaGetDisplayDRM; a null info callback turns libva's info
// messages off, and `on_error` matches `VAMessageCallback` and never unwinds.
unsafe {
(api.set_info_callback)(dpy, None, std::ptr::null_mut());
(api.set_error_callback)(dpy, Some(on_error), std::ptr::null_mut());
}
let (mut major, mut minor): (c_int, c_int) = (0, 0);
// SAFETY: `dpy` is valid; the version outputs point to live locals.
let st = unsafe { (api.initialize)(dpy, &mut major, &mut minor) };
d.check(st, "vaInitialize")?;
// SAFETY: `dpy` is an initialised display; the vendor string is NUL-terminated and owned by
// the driver for the life of the display (copied out here).
let vendor = unsafe { (api.vendor_string)(dpy) };
if !vendor.is_null() {
// SAFETY: non-null, NUL-terminated (see above).
d.vendor = unsafe { CStr::from_ptr(vendor) }.to_string_lossy().into_owned();
}
Ok(d)
}
/// The driver's description ("Intel iHD driver …").
pub fn vendor(&self) -> &str {
&self.vendor
}
fn check(&self, st: ffi::VAStatus, what: &str) -> Result<(), String> {
if st == ffi::VA_STATUS_SUCCESS {
return Ok(());
}
// SAFETY: vaErrorStr takes any status and returns a static NUL-terminated string.
let text = unsafe { (self.api.error_str)(st) };
let text = if text.is_null() {
String::new()
} else {
// SAFETY: non-null static NUL-terminated string (see above).
unsafe { CStr::from_ptr(text) }.to_string_lossy().into_owned()
};
Err(format!("{what}: {text} ({st:#x})"))
}
}
impl Drop for Display {
fn drop(&mut self) {
// SAFETY: `dpy` came from vaGetDisplayDRM and is terminated exactly once, here, after
// every session object made on it was destroyed (`Session` drops its objects first).
unsafe { (self.api.terminate)(self.dpy) };
}
}
/// One stream's decoder: a VLD configuration of a profile, its surfaces and context, and the image
/// pictures are read back through.
pub struct Session {
display: Display,
config: ffi::VAConfigID,
context: ffi::VAContextID,
surfaces: Vec<VASurfaceID>,
image: Option<VAImage>,
/// Coded size of the surfaces (luma samples).
size: (u32, u32),
geometry: Geometry,
}
/// The surface and image formats of 8-bit (NV12) or 10-bit (P010) 4:2:0 pictures:
/// (render-target format, image fourcc, bits per pixel of the image).
fn formats(bits: u32) -> Result<(c_uint, u32, u32), String> {
match bits {
8 => Ok((ffi::VA_RT_FORMAT_YUV420, ffi::VA_FOURCC_NV12, 12)),
10 => Ok((ffi::VA_RT_FORMAT_YUV420_10, ffi::VA_FOURCC_P010, 24)),
_ => Err(format!("{bits}-bit pictures are not decoded through VA-API")),
}
}
impl Session {
/// A decoder for the first of `profiles` the driver decodes (VLD, 4:2:0 of `geometry.bits`:
/// 8 or 10), with `count` surfaces of `size` (coded luma samples); pictures read back are cut
/// to `geometry`.
pub fn new(profiles: &[ffi::VAProfile], size: (u32, u32), count: usize, geometry: Geometry) -> Result<Self, String> {
let (rt_format, _, _) = formats(geometry.bits)?;
let display = Display::open()?;
let api = display.api;
let dpy = display.dpy;
let mut config = None;
let mut why = "no VA-API profile for the stream".to_string();
for &profile in profiles {
let mut attr = VAConfigAttrib { type_: ffi::VAConfigAttribRTFormat, value: 0 };
// SAFETY: `dpy` is initialised; `attr` is one live attribute (count 1).
let st = unsafe { (api.get_config_attributes)(dpy, profile, ffi::VAEntrypointVLD, &mut attr, 1) };
if let Err(e) = display.check(st, "vaGetConfigAttributes") {
why = e;
continue;
}
if attr.value & rt_format == 0 {
why = format!("profile {profile} does not decode to {}-bit 4:2:0", geometry.bits);
continue;
}
attr.value = rt_format;
let mut id = ffi::VA_INVALID_ID;
// SAFETY: as above; `id` is a live output.
let st = unsafe { (api.create_config)(dpy, profile, ffi::VAEntrypointVLD, &mut attr, 1, &mut id) };
match display.check(st, "vaCreateConfig") {
Ok(()) => {
config = Some(id);
break;
}
Err(e) => why = e,
}
}
let config = config.ok_or(why)?;
// From here on `Drop` releases what was made.
let mut s = Self { display, config, context: ffi::VA_INVALID_ID, surfaces: Vec::new(), image: None, size, geometry };
let count_c = c_uint::try_from(count).map_err(|_| "too many surfaces")?;
let mut surfaces = vec![ffi::VA_INVALID_SURFACE; count];
// SAFETY: `surfaces` has room for `count` ids; no attributes are passed (null, 0).
let st = unsafe { (api.create_surfaces)(dpy, rt_format, size.0, size.1, surfaces.as_mut_ptr(), count_c, std::ptr::null_mut(), 0) };
s.display.check(st, "vaCreateSurfaces")?;
s.surfaces = surfaces;
let (w, h) = (c_int::try_from(size.0).map_err(|_| "picture too wide")?, c_int::try_from(size.1).map_err(|_| "picture too tall")?);
let n = c_int::try_from(count).map_err(|_| "too many surfaces")?;
let mut context = ffi::VA_INVALID_ID;
// SAFETY: `config` and the `count` surfaces were made on this display; `context` is a live output.
let st = unsafe { (api.create_context)(dpy, s.config, w, h, ffi::VA_PROGRESSIVE, s.surfaces.as_mut_ptr(), n, &mut context) };
s.display.check(st, "vaCreateContext")?;
s.context = context;
Ok(s)
}
pub fn vendor(&self) -> &str {
self.display.vendor()
}
/// A buffer of `ty` holding a copy of `len` bytes at `data`.
fn buffer(&self, ty: ffi::VABufferType, data: *const c_void, len: usize) -> Result<ffi::VABufferID, String> {
let size = c_uint::try_from(len).map_err(|_| "buffer too large")?;
let mut id = ffi::VA_INVALID_ID;
// SAFETY: `data` points to `len` readable bytes (the callers pass a live value or slice);
// libva copies them into the new buffer and never writes through the pointer.
let st = unsafe { (self.display.api.create_buffer)(self.display.dpy, self.context, ty, size, 1, data.cast_mut(), &mut id) };
self.display.check(st, "vaCreateBuffer")?;
Ok(id)
}
fn value_buffer<T>(&self, ty: ffi::VABufferType, value: &T) -> Result<ffi::VABufferID, String> {
self.buffer(ty, (value as *const T).cast(), std::mem::size_of::<T>())
}
fn render(&self, buffers: &mut [ffi::VABufferID]) -> Result<(), String> {
let n = c_int::try_from(buffers.len()).map_err(|_| "too many buffers")?;
// SAFETY: `buffers` are live buffer ids made on this context, `n` of them.
let st = unsafe { (self.display.api.render_picture)(self.display.dpy, self.context, buffers.as_mut_ptr(), n) };
self.display.check(st, "vaRenderPicture")
}
/// Decode one picture into `surface`: the picture parameters `pic`, the scaling matrices `iq`
/// when given, and a parameter + data buffer per slice; the buffers made go into `buffers`.
fn decode_into<P, Q, S>(
&self,
surface: VASurfaceID,
buffers: &mut Vec<ffi::VABufferID>,
pic: &P,
iq: Option<&Q>,
slices: &[(S, Vec<u8>)],
) -> Result<(), String> {
buffers.push(self.value_buffer(ffi::VAPictureParameterBufferType, pic)?);
if let Some(iq) = iq {
buffers.push(self.value_buffer(ffi::VAIQMatrixBufferType, iq)?);
}
let head = buffers.len();
for (params, data) in slices {
buffers.push(self.value_buffer(ffi::VASliceParameterBufferType, params)?);
buffers.push(self.buffer(ffi::VASliceDataBufferType, data.as_ptr().cast(), data.len())?);
}
let api = self.display.api;
// SAFETY: `surface` is one of this context's render targets.
let st = unsafe { (api.begin_picture)(self.display.dpy, self.context, surface) };
self.display.check(st, "vaBeginPicture")?;
let (picture, slice_buffers) = buffers.split_at_mut(head);
let mut rendered = self.render(picture);
for pair in slice_buffers.chunks_mut(2) {
if rendered.is_err() {
break;
}
rendered = self.render(pair);
}
// a begun picture is always ended, so the context stays usable
// SAFETY: the picture was begun on this context above.
let st = unsafe { (api.end_picture)(self.display.dpy, self.context) };
rendered?;
self.display.check(st, "vaEndPicture")
}
fn image(&mut self) -> Result<VAImage, String> {
if let Some(image) = self.image {
return Ok(image);
}
let (_, fourcc, bits_per_pixel) = formats(self.geometry.bits)?;
let mut format = VAImageFormat { fourcc, byte_order: 1, bits_per_pixel, ..Default::default() };
let mut image = VAImage { image_id: ffi::VA_INVALID_ID, buf: ffi::VA_INVALID_ID, ..Default::default() };
let (w, h) = (c_int::try_from(self.size.0).map_err(|_| "picture too wide")?, c_int::try_from(self.size.1).map_err(|_| "picture too tall")?);
// SAFETY: `format` and `image` are live; the driver fills `image` in.
let st = unsafe { (self.display.api.create_image)(self.display.dpy, &mut format, w, h, &mut image) };
self.display.check(st, "vaCreateImage")?;
self.image = Some(image);
Ok(image)
}
/// The picture in surface `index`, copied into the session's image once decoded.
fn get_image(&mut self, index: usize) -> Result<VAImage, String> {
let surface = *self.surfaces.get(index).ok_or("surface out of range")?;
let image = self.image()?;
let api = self.display.api;
// SAFETY: `surface` is one of this session's surfaces.
let st = unsafe { (api.sync_surface)(self.display.dpy, surface) };
self.display.check(st, "vaSyncSurface")?;
// SAFETY: the image and the surface were made on this display, with the same size.
let st = unsafe { (api.get_image)(self.display.dpy, surface, 0, 0, self.size.0, self.size.1, image.image_id) };
self.display.check(st, "vaGetImage")?;
Ok(image)
}
/// Every sample of `image` set to mid-gray: 128 in NV12, 512 (0x8000 in the high bits of
/// little-endian 16-bit words) in P010.
fn write_gray(&self, image: &VAImage) -> Result<(), String> {
let api = self.display.api;
let mut ptr: *mut c_void = std::ptr::null_mut();
// SAFETY: `image.buf` is the data buffer of an image made on this display; `ptr` is a live output.
let st = unsafe { (api.map_buffer)(self.display.dpy, image.buf, &mut ptr) };
self.display.check(st, "vaMapBuffer")?;
if !ptr.is_null() {
// SAFETY: a mapped image buffer holds `data_size` writable bytes at `ptr` until it is
// unmapped below; the slice does not outlive this block.
let data = unsafe { std::slice::from_raw_parts_mut(ptr.cast::<u8>(), image.data_size as usize) };
if self.geometry.bits > 8 {
for w in data.as_chunks_mut::<2>().0 {
w.copy_from_slice(&0x8000u16.to_le_bytes());
}
} else {
data.fill(128);
}
}
// SAFETY: the buffer was mapped above.
let st = unsafe { (api.unmap_buffer)(self.display.dpy, image.buf) };
self.display.check(st, "vaUnmapBuffer")?;
if ptr.is_null() { Err("vaMapBuffer returned no data".into()) } else { Ok(()) }
}
fn decode_picture<P, Q, S>(&mut self, target: usize, pic: &P, iq: Option<&Q>, slices: &[(S, Vec<u8>)]) -> Result<(), String> {
let surface = *self.surfaces.get(target).ok_or("surface out of range")?;
let mut buffers = Vec::with_capacity(2 + 2 * slices.len());
let decoded = self.decode_into(surface, &mut buffers, pic, iq, slices);
for id in buffers {
// SAFETY: each id is a buffer made above on this display, destroyed once; vaEndPicture
// has returned, so the driver no longer reads it.
unsafe { (self.display.api.destroy_buffer)(self.display.dpy, id) };
}
decoded
}
fn read_image(&self, image: &VAImage) -> Result<VideoFrame, String> {
let api = self.display.api;
let mut ptr: *mut c_void = std::ptr::null_mut();
// SAFETY: `image.buf` is the data buffer of an image made on this display; `ptr` is a live output.
let st = unsafe { (api.map_buffer)(self.display.dpy, image.buf, &mut ptr) };
self.display.check(st, "vaMapBuffer")?;
let result = if ptr.is_null() {
Err("vaMapBuffer returned no data".to_string())
} else {
// SAFETY: a mapped image buffer holds `data_size` bytes at `ptr` until it is unmapped
// below; the slice does not outlive this block.
let data = unsafe { std::slice::from_raw_parts(ptr.cast::<u8>().cast_const(), image.data_size as usize) };
biplanar_view(image, data).and_then(|b| biplanar::to_frame(&b, &self.geometry))
};
// SAFETY: the buffer was mapped above; `data` is no longer used.
let st = unsafe { (api.unmap_buffer)(self.display.dpy, image.buf) };
let frame = result?;
self.display.check(st, "vaUnmapBuffer")?;
Ok(frame)
}
}
/// The two planes of a mapped NV12 / P010 image.
fn biplanar_view<'a>(image: &VAImage, data: &'a [u8]) -> Result<Biplanar<'a>, String> {
if !matches!(image.format.fourcc, ffi::VA_FOURCC_NV12 | ffi::VA_FOURCC_P010) || image.num_planes < 2 {
return Err(format!("the read-back image is not NV12 / P010 (fourcc {:#x}, {} planes)", image.format.fourcc, image.num_planes));
}
let [p0, p1, _] = image.pitches;
if p0 != p1 {
return Err(format!("NV12 planes with different pitches ({p0}, {p1})"));
}
let luma = data.get(image.offsets[0] as usize..).ok_or("NV12 luma offset out of range")?;
let chroma = data.get(image.offsets[1] as usize..).ok_or("NV12 chroma offset out of range")?;
Ok(Biplanar { luma, chroma, stride: p0 as usize, rows: usize::from(image.height) })
}
impl Accel for Session {
fn surfaces(&self) -> &[VASurfaceID] {
&self.surfaces
}
fn decode_h264(
&mut self,
target: usize,
pic: &ffi::VAPictureParameterBufferH264,
iq: &ffi::VAIQMatrixBufferH264,
slices: &[(ffi::VASliceParameterBufferH264, Vec<u8>)],
) -> Result<(), String> {
self.decode_picture(target, pic, Some(iq), slices)
}
fn decode_hevc(
&mut self,
target: usize,
pic: &ffi::VAPictureParameterBufferHEVC,
iq: Option<&ffi::VAIQMatrixBufferHEVC>,
slices: &[(ffi::VASliceParameterBufferHEVC, Vec<u8>)],
) -> Result<(), String> {
self.decode_picture(target, pic, iq, slices)
}
fn read(&mut self, index: usize) -> Result<VideoFrame, String> {
let image = self.get_image(index)?;
self.read_image(&image)
}
fn fill(&mut self, index: usize, from: Option<usize>) -> Result<(), String> {
let surface = *self.surfaces.get(index).ok_or("surface out of range")?;
let image = match from {
Some(from) => self.get_image(from)?,
None => {
let image = self.image()?;
self.write_gray(&image)?;
image
}
};
let (w, h) = self.size;
// SAFETY: the image and the surface were made on this display, with the same size.
let st = unsafe { (self.display.api.put_image)(self.display.dpy, surface, image.image_id, 0, 0, w, h, 0, 0, w, h) };
self.display.check(st, "vaPutImage")
}
}
impl Drop for Session {
fn drop(&mut self) {
let (api, dpy) = (self.display.api, self.display.dpy);
// SAFETY: each object below was made on this display and is destroyed once; the display
// itself is terminated afterwards (when the `display` field is dropped).
unsafe {
if let Some(image) = self.image.take() {
(api.destroy_image)(dpy, image.image_id);
}
if self.context != ffi::VA_INVALID_ID {
(api.destroy_context)(dpy, self.context);
}
if !self.surfaces.is_empty() {
let n = c_int::try_from(self.surfaces.len()).unwrap_or(0);
(api.destroy_surfaces)(dpy, self.surfaces.as_mut_ptr(), n);
}
if self.config != ffi::VA_INVALID_ID {
(api.destroy_config)(dpy, self.config);
}
}
}
}
/// Whether this system has a working VA-API driver on a DRM render node (checked once), with the
/// driver's description, or why not.
pub fn probe() -> &'static Result<String, String> {
static PROBE: OnceLock<Result<String, String>> = OnceLock::new();
PROBE.get_or_init(|| Display::open().map(|d| d.vendor().to_string()))
}

29
crates/platform/tests/hevc_probe.rs Normal file
View File

@ -0,0 +1,29 @@
//! `register()` answers the HEVC availability probe in the background, so the first draw of the
//! format list does not create a hardware session on the UI thread. One test in this file on
//! purpose: anything else that asked `hevc_available()` first would make the wait below meaningless.
#![cfg(target_os = "macos")]
use std::time::{Duration, Instant};
use filmcraft_platform::hardware_encode::{hevc_available, hevc_probed, warm_hevc_probe};
#[test]
fn register_answers_the_hevc_probe_in_the_background() {
assert!(!hevc_probed(), "nothing has asked yet");
filmcraft_platform::register();
// nobody calls `hevc_available()` here: only the thread `register` started can answer
let t0 = Instant::now();
while !hevc_probed() {
assert!(t0.elapsed() < Duration::from_secs(60), "register() did not start the probe");
std::thread::sleep(Duration::from_millis(5));
}
// asking again, or registering again, changes nothing
let answer = hevc_available();
warm_hevc_probe();
filmcraft_platform::register();
assert_eq!(hevc_available(), answer);
assert!(hevc_probed());
}

491
crates/platform/tests/vaapi.rs Normal file
View File

@ -0,0 +1,491 @@
//! VA-API decoding against our software decoders (Linux): every picture of H.264 and HEVC fixtures
//! covering the stream features the hardware path takes (H.264: CAVLC and CABAC, B-pyramids,
//! several references, weighted prediction, temporal direct, custom scaling matrices, several
//! slices per picture; HEVC Main and Main 10: open GOP with RASL pictures, scaling lists, several
//! slices with wavefront parallel processing, weighted prediction, transform skip and AMP; 1080p /
//! 2160p) must be identical (bit-exact planes, colour, pixel aspect, pts and presentation
//! order), also after `reset` + reseek and a mid-stream `flush`; a forced mid-stream failure must
//! continue in software with the software decoder's exact output; what the hardware path does not
//! take (interlaced and 10-bit H.264, 4:2:2 HEVC) is declined; damaged samples and parameter sets give
//! errors or fall back, never crash or hang. Skips without ffmpeg (fixture generator) or without a
//! VA-API driver.
#![cfg(target_os = "linux")]
mod common;
use std::time::Duration;
use common::*;
use filmcraft_codecs::VideoDecoder;
use filmcraft_codecs::hw::NalStreamInfo;
use filmcraft_platform::HybridDecoder;
use filmcraft_platform::vaapi::VaDecoder;
/// The hardware decoder for a stream, or `None` (skip) when this machine has none for it.
fn hardware(s: &Stream) -> Option<Box<dyn VideoDecoder>> {
match filmcraft_platform::vaapi_factory(&s.entry) {
Some(Ok(d)) => {
assert!(d.name().starts_with("VA-API"), "{}", d.name());
Some(d)
}
Some(Err(e)) => panic!("factory error: {e}"),
None => {
eprintln!("SKIPPED: no VA-API hardware decoder for {}", s.entry.codec.name());
None
}
}
}
fn software(s: &Stream) -> Box<dyn VideoDecoder> {
filmcraft_codecs::software_video_decoder(&s.entry).unwrap()
}
/// H.264 fixtures for the stream features the hardware path takes: (file, x264 profile, x264
/// parameters). All 640x360 (coded 368: cropping), three seconds at 24 fps, keyframes every second.
const H264: &[(&str, &str, &str)] = &[
("va_h264_cbp_slices.mp4", "baseline", "keyint=24:min-keyint=24:scenecut=0:slices=3:ref=3"),
("va_h264_main_weighted.mp4", "main", "bframes=3:b-pyramid=normal:keyint=24:min-keyint=24:scenecut=0:ref=4:weightp=2:weightb=1"),
("va_h264_temporal_direct.mp4", "high", "bframes=2:b-pyramid=strict:direct=temporal:keyint=24:min-keyint=24:scenecut=0:ref=3:weightb=0"),
("va_h264_cqm_jvt.mp4", "high", "bframes=3:keyint=24:min-keyint=24:scenecut=0:cqm=jvt:8x8dct=1:slices=2"),
("va_h264_open_gop.mp4", "high", "bframes=3:b-pyramid=normal:keyint=24:min-keyint=24:scenecut=0:open-gop=1"),
];
fn h264_fixture(ff: &std::path::Path, name: &str, profile: &str, params: &str) -> Option<std::path::PathBuf> {
let src = "testsrc2=s=640x360:r=24:d=3,noise=alls=12:allf=t";
fixture(ff, name, &["-f", "lavfi", "-i", src, "-c:v", "libx264", "-profile:v", profile, "-preset", "fast", "-x264-params", params, "-pix_fmt", "yuv420p"])
}
/// HEVC fixtures for the stream features the hardware path takes: (file, pixel format, x265
/// parameters). 640x360, three seconds at 24 fps, keyframes every second.
const HEVC: &[(&str, &str, &str)] = &[
("va_hevc_scaling_lists.mp4", "yuv420p", "log-level=error:bframes=3:keyint=24:min-keyint=24:scenecut=0:scaling-list=default"),
("va_hevc_slices_wpp.mp4", "yuv420p10le", "log-level=error:bframes=3:keyint=24:min-keyint=24:scenecut=0:slices=3:wpp=1"),
("va_hevc_weighted.mp4", "yuv420p", "log-level=error:bframes=3:keyint=24:min-keyint=24:scenecut=0:weightp=1:weightb=1:ref=4"),
("va_hevc_tskip_amp.mp4", "yuv420p10le", "log-level=error:bframes=4:b-pyramid=1:keyint=24:min-keyint=24:scenecut=0:tskip=1:amp=1:rect=1:open-gop=1"),
];
fn hevc_fixture(ff: &std::path::Path, name: &str, pix_fmt: &str, params: &str) -> Option<std::path::PathBuf> {
let src = "testsrc2=s=640x360:r=24:d=3,noise=alls=12:allf=t";
let profile = if pix_fmt.contains("10") { "main10" } else { "main" };
fixture(
ff,
name,
&["-f", "lavfi", "-i", src, "-c:v", "libx265", "-preset", "fast", "-profile:v", profile, "-tag:v", "hvc1", "-x265-params", params, "-pix_fmt", pix_fmt],
)
}
/// The small fixtures: the shared H.264 High, HEVC Main (open GOP) and Main 10 ones and the
/// feature fixtures above.
fn small_streams(ff: &std::path::Path) -> Vec<(String, Stream)> {
let mut out = Vec::new();
for name in ["h264_high.mp4", "hevc_main.mp4", "hevc_main10.mp4"] {
if let Some(path) = named(ff, name) {
out.push((name.to_string(), read_stream(&path)));
}
}
for (name, profile, params) in H264 {
if let Some(path) = h264_fixture(ff, name, profile, params) {
out.push((name.to_string(), read_stream(&path)));
}
}
for (name, pix_fmt, params) in HEVC {
if let Some(path) = hevc_fixture(ff, name, pix_fmt, params) {
out.push((name.to_string(), read_stream(&path)));
}
}
out
}
/// Every picture of `s` from the hardware decoder equals the software decoder's: whole stream,
/// after `reset` + reseek to each later sync sample, after a mid-stream `flush`, and a full pass
/// after all the resets.
fn parity(name: &str, s: &Stream) {
let Some(mut hw) = hardware(s) else { return };
let mut sw = software(s);
let a = decode_all(hw.as_mut(), &s.samples);
let b = decode_all(sw.as_mut(), &s.samples);
assert!(a.len() >= 40, "{name}: {} pictures", a.len());
assert_same(name, &a, &b);
assert!(hw.name().starts_with("VA-API"), "{name}: stayed in hardware ({})", hw.name());
// reset + reseek to each later sync sample, decode a stretch. After a seek to a non-IDR I picture
// (open GOP), the pictures shown before it are predicted from frames that were never decoded:
// both decoders conceal them (the software decoder from mid-gray stand-ins, the hardware also
// from motion data those stand-ins do not have), so only the pictures from the seek point on
// must match, as they must.
let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap();
let syncs: Vec<usize> = (1..s.samples.len()).filter(|&i| s.sync[i]).collect();
assert!(!syncs.is_empty(), "{name}: more than one GOP");
for &k in syncs.iter().rev() {
let end = (k + 17).min(s.samples.len());
for d in [&mut hw, &mut sw] {
d.reset();
}
let mut a = decode_all(hw.as_mut(), &s.samples[k..end]);
let mut b = decode_all(sw.as_mut(), &s.samples[k..end]);
assert_eq!(a.len(), b.len(), "{name} from sample {k}: pictures");
if info.codec == filmcraft_codecs::hw::NalCodec::H264 && !info.nal_types(&s.samples[k].0).contains(&5) {
let start = s.samples[k].1;
a.retain(|f| f.pts >= start);
b.retain(|f| f.pts >= start);
}
assert!(!a.is_empty(), "{name}: pictures after seeking to {k}");
assert_same(&format!("{name} from sample {k}"), &a, &b);
}
// a flush in the middle of a GOP, then carrying on: H.264 only, where references survive a
// flush (an HEVC flush ends the stream; the GOP cache always seeks after one)
if info.codec == filmcraft_codecs::hw::NalCodec::H264 {
for d in [&mut hw, &mut sw] {
d.reset();
}
let mid = s.samples.len() / 2;
let mut a = Vec::new();
let mut b = Vec::new();
for (d, out) in [(&mut hw, &mut a), (&mut sw, &mut b)] {
for (smp, pts) in &s.samples[..mid] {
out.extend(d.decode(smp, *pts).unwrap());
}
out.extend(d.flush());
out.extend(decode_all(d.as_mut(), &s.samples[mid..]));
}
assert_same(&format!("{name} with a flush at {mid}"), &a, &b);
}
// back to the start after seeking around: the whole stream again
hw.reset();
sw.reset();
assert_same(&format!("{name} after resets"), &decode_all(hw.as_mut(), &s.samples), &decode_all(sw.as_mut(), &s.samples));
assert!(hw.name().starts_with("VA-API"), "{name}: stayed in hardware ({})", hw.name());
// random-access / disposable answers match the software decoder's
for (smp, _) in &s.samples {
assert_eq!(hw.is_random_access(smp), sw.is_random_access(smp), "{name}");
assert_eq!(hw.is_disposable(smp), sw.is_disposable(smp), "{name}");
}
}
/// [`parity`] for large streams, with little memory: the two decoders run side by side and each
/// picture is compared and dropped as soon as both have output it (a 4K 10-bit stream held whole
/// takes gigabytes). The whole stream, then from the last sync sample after a reset.
fn parity_streaming(name: &str, s: &Stream) {
let Some(mut hw) = hardware(s) else { return };
let mut sw = software(s);
let last_sync = (1..s.samples.len()).rev().find(|&i| s.sync[i]).unwrap_or(0);
for start in [0, last_sync] {
hw.reset();
sw.reset();
let (mut a, mut b) = (std::collections::VecDeque::new(), std::collections::VecDeque::new());
let mut compared = 0usize;
let mut compare = |a: &mut std::collections::VecDeque<filmcraft_codecs::DecodedFrame>,
b: &mut std::collections::VecDeque<filmcraft_codecs::DecodedFrame>| {
while !a.is_empty() && !b.is_empty() {
let (Some(x), Some(y)) = (a.pop_front(), b.pop_front()) else { break };
assert_same(&format!("{name} from sample {start}"), &[x], &[y]);
compared += 1;
}
};
for (smp, pts) in &s.samples[start..] {
a.extend(hw.decode(smp, *pts).unwrap());
b.extend(sw.decode(smp, *pts).unwrap());
compare(&mut a, &mut b);
}
a.extend(hw.flush());
b.extend(sw.flush());
compare(&mut a, &mut b);
assert!(a.is_empty() && b.is_empty(), "{name} from sample {start}: picture counts differ");
assert!(compared >= s.samples.len().saturating_sub(start) / 2, "{name} from sample {start}: {compared} pictures");
assert!(hw.name().starts_with("VA-API"), "{name}: stayed in hardware ({})", hw.name());
}
}
#[test]
fn bit_exact_with_the_software_decoder() {
let ff = filmcraft_testkit::require_ffmpeg!();
let streams = small_streams(&ff);
assert!(streams.len() > H264.len(), "fixtures");
for (name, s) in streams {
parity(&name, &s);
}
}
fn large(size: &str) {
let ff = filmcraft_testkit::require_ffmpeg!();
let (_, h) = size.split_once('x').unwrap();
let src = format!("testsrc2=s={size}:r=24:d=2,noise=alls=12:allf=t");
let args = [
"-f",
"lavfi",
"-i",
&src,
"-c:v",
"libx264",
"-profile:v",
"high",
"-preset",
"veryfast",
"-x264-params",
"bframes=3:b-pyramid=normal:keyint=24:min-keyint=24:scenecut=0",
"-pix_fmt",
"yuv420p",
];
let Some(path) = fixture(&ff, &format!("h264_{h}.mp4"), &args) else { return };
parity_streaming(&format!("h264 {size}"), &read_stream(&path));
}
#[test]
fn h264_1080p_and_2160p_are_bit_exact() {
large("1920x1080");
large("3840x2160");
}
fn large_hevc(size: &str, pix_fmt: &str) {
let ff = filmcraft_testkit::require_ffmpeg!();
let (_, h) = size.split_once('x').unwrap();
let src = format!("testsrc2=s={size}:r=24:d=2,noise=alls=12:allf=t");
let ten = pix_fmt.contains("10");
let args = [
"-f",
"lavfi",
"-i",
&src,
"-c:v",
"libx265",
"-preset",
"ultrafast",
"-profile:v",
if ten { "main10" } else { "main" },
"-tag:v",
"hvc1",
"-x265-params",
"log-level=error:bframes=4:keyint=24:min-keyint=24:scenecut=0:open-gop=1",
"-pix_fmt",
pix_fmt,
];
let name = format!("va_hevc_{}_{h}.mp4", if ten { "main10" } else { "main" });
let Some(path) = fixture(&ff, &name, &args) else { return };
parity_streaming(&name, &read_stream(&path));
}
#[test]
fn hevc_main_and_main10_1080p_and_2160p_are_bit_exact() {
for size in ["1920x1080", "3840x2160"] {
large_hevc(size, "yuv420p");
large_hevc(size, "yuv420p10le");
}
}
#[test]
fn mid_stream_failure_continues_in_software() {
let ff = filmcraft_testkit::require_ffmpeg!();
for (name, s) in small_streams(&ff) {
let name = name.as_str();
let info = NalStreamInfo::from_entry(&s.entry).unwrap().unwrap();
let Ok(_) = VaDecoder::new(info.clone()) else {
eprintln!("SKIPPED: no VA-API hardware decoder for {name}");
continue;
};
let reference = decode_all(software(&s).as_mut(), &s.samples);
assert!(!reference.is_empty(), "{name}");
// fail at the first sample, inside the first GOP, at a sync sample and just after one
let k = (1..s.samples.len()).find(|&i| s.sync[i]).unwrap();
for fail_at in [0, 5, k, k + 1, k + 9] {
let mut va = VaDecoder::new(info.clone()).unwrap();
va.fail_after(fail_at as u64);
let mut d = HybridDecoder::new(Box::new(va), s.entry.clone(), info.clone());
let before = filmcraft_codecs::hw::hw_stats().fallbacks;
let out = decode_all(&mut d, &s.samples);
assert!(!d.is_hardware(), "{name}: switched to software");
assert!(filmcraft_codecs::hw::hw_stats().fallbacks > before, "{name}: fallback counted");
assert_same(&format!("{name} failing at sample {fail_at}"), &out, &reference);
d.reset();
assert_same(&format!("{name} after the fallback and a reset"), &decode_all(&mut d, &s.samples), &reference);
}
}
}
/// What the hardware path does not take is declined (the software decoders keep it): interlaced
/// and 10-bit H.264, and HEVC (not decoded through VA-API yet).
#[test]
fn declines_what_the_hardware_path_does_not_take() {
let ff = filmcraft_testkit::require_ffmpeg!();
let src = "testsrc2=s=640x360:r=24:d=1";
let cases: [(&str, Vec<&str>); 2] = [
("va_h264_interlaced.mp4", vec!["-f", "lavfi", "-i", src, "-c:v", "libx264", "-x264-params", "interlaced=1:keyint=12", "-pix_fmt", "yuv420p"]),
("va_h264_10bit.mp4", vec!["-f", "lavfi", "-i", src, "-c:v", "libx264", "-profile:v", "high10", "-pix_fmt", "yuv420p10le"]),
];
for (name, args) in cases {
// an ffmpeg whose x264 lacks 10-bit leaves the fixture out
let Some(path) = fixture(&ff, name, &args) else { continue };
let s = read_stream(&path);
assert!(filmcraft_platform::vaapi_factory(&s.entry).is_none(), "{name}: declined");
}
let args = ["-f", "lavfi", "-i", src, "-c:v", "libx265", "-tag:v", "hvc1", "-x265-params", "log-level=error:keyint=12", "-pix_fmt", "yuv422p10le"];
if let Some(path) = fixture(&ff, "va_hevc_422.mp4", &args) {
let s = read_stream(&path);
assert!(filmcraft_platform::vaapi_factory(&s.entry).is_none(), "HEVC 4:2:2: declined");
}
// Settings ▸ Playback ▸ Hardware decoding Off
let Some(path) = named(&ff, "h264_high.mp4") else { return };
let s = read_stream(&path);
filmcraft_codecs::hw::set_hardware_decoding(false);
let off = filmcraft_platform::vaapi_factory(&s.entry).is_none();
filmcraft_codecs::hw::set_hardware_decoding(true);
assert!(off, "declined while hardware decoding is Off");
}
/// Run `f` on a thread; fail if it panics or takes longer than `limit`.
fn bounded(what: &str, limit: Duration, f: impl FnOnce() + Send + 'static) {
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f));
let _ = tx.send(r.is_ok());
});
match rx.recv_timeout(limit) {
Ok(true) => {}
Ok(false) => panic!("{what}: panicked"),
Err(_) => panic!("{what}: hung"),
}
}
#[test]
fn damaged_samples_and_parameter_sets_never_crash_or_hang() {
let ff = filmcraft_testkit::require_ffmpeg!();
for (name, s) in small_streams(&ff) {
let name = name.as_str();
if hardware(&s).is_none() {
continue;
}
// damaged samples (bit flips, truncation, corrupt length prefixes): through the hybrid, and
// straight into the hardware decoder (no software to fall back to)
for seed in 1..=8u64 {
let samples = s.samples.clone();
let entry = s.entry.clone();
bounded(&format!("{name} seed {seed}"), Duration::from_secs(60), move || {
let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;
let mut d = filmcraft_platform::vaapi_factory(&entry).unwrap().unwrap();
let info = NalStreamInfo::from_entry(&entry).unwrap().unwrap();
let mut raw = VaDecoder::new(info).unwrap();
for (i, (smp, pts)) in samples.iter().enumerate().take(40) {
let mut m = smp.clone();
match xorshift(&mut seed) % 4 {
0 => {
for _ in 0..1 + xorshift(&mut seed) % 8 {
let at = (xorshift(&mut seed) as usize) % m.len().max(1);
if let Some(b) = m.get_mut(at) {
*b ^= 1 << (xorshift(&mut seed) % 8);
}
}
}
1 => m.truncate((xorshift(&mut seed) as usize) % m.len().max(1)),
2 if m.len() > 4 => m[..4].copy_from_slice(&(xorshift(&mut seed) as u32).to_be_bytes()),
_ => {}
}
let _ = d.decode(&m, *pts);
let _ = raw.decode(&m, *pts);
if i % 13 == 12 {
d.reset();
raw.reset();
}
}
let _ = d.flush();
let _ = raw.flush();
});
}
// damaged parameter sets: the decoder declines, errors or decodes; never crashes
let filmcraft_isobmff::CodecConfig::Avc(a) = &s.entry.codec else { continue };
let rec = a.to_bytes();
let mut seed = 0x1234_5678u64;
for round in 0..24 {
let mut r = rec.clone();
if round % 3 == 2 {
r.truncate(r.len() * (round + 1) / 30);
} else {
for _ in 0..1 + round % 4 {
let at = 6 + (xorshift(&mut seed) as usize) % r.len().saturating_sub(6).max(1);
if let Some(b) = r.get_mut(at) {
*b ^= 1 << (xorshift(&mut seed) % 8);
}
}
}
let samples: Vec<_> = s.samples.iter().take(12).cloned().collect();
bounded(&format!("{name} record round {round}"), Duration::from_secs(60), move || {
let Ok(info) = NalStreamInfo::from_avcc(&r) else { return };
let Ok(mut va) = VaDecoder::new(info) else { return };
for (smp, pts) in &samples {
if va.decode(smp, *pts).is_err() {
break;
}
}
let _ = va.flush();
});
}
}
}
/// Creating and dropping decoders again and again (every clip on a timeline makes one) neither
/// fails nor leaks VA-API objects into the next one.
#[test]
fn repeated_initialisation() {
let ff = filmcraft_testkit::require_ffmpeg!();
let Some(path) = named(&ff, "h264_high.mp4") else { return };
let s = read_stream(&path);
let reference = decode_all(software(&s).as_mut(), &s.samples[..13]);
for round in 0..40 {
let Some(mut hw) = hardware(&s) else { return };
if round % 8 == 0 {
assert_same(&format!("round {round}"), &decode_all(hw.as_mut(), &s.samples[..13]), &reference);
}
}
}
/// Several decoders at once on different threads (the frame workers) give the software decoder's
/// pictures.
#[test]
fn concurrent_decoders() {
let ff = filmcraft_testkit::require_ffmpeg!();
let mut paths: Vec<_> = named(&ff, "h264_high.mp4").into_iter().collect();
for (name, profile, params) in H264.iter().take(3) {
paths.extend(h264_fixture(&ff, name, profile, params));
}
let jobs: Vec<_> = paths
.into_iter()
.chain(named(&ff, "h264_high.mp4"))
.map(|path| {
std::thread::spawn(move || {
let s = read_stream(&path);
let Some(mut hw) = hardware(&s) else { return };
let n = s.samples.len().min(40);
let a = decode_all(hw.as_mut(), &s.samples[..n]);
hw.reset();
let b = decode_all(hw.as_mut(), &s.samples[..n]);
let c = decode_all(software(&s).as_mut(), &s.samples[..n]);
assert_same("concurrent", &a, &c);
assert_same("concurrent, after a reset", &b, &c);
})
})
.collect();
for j in jobs {
j.join().unwrap();
}
}
/// A decoder used and moved between threads (decoders migrate between frame workers).
#[test]
fn decoder_moves_between_threads() {
let ff = filmcraft_testkit::require_ffmpeg!();
let Some(path) = named(&ff, "h264_high.mp4") else { return };
let s = read_stream(&path);
let Some(mut hw) = hardware(&s) else { return };
let reference = decode_all(software(&s).as_mut(), &s.samples[..30]);
let samples = s.samples.clone();
let (hw, out) = std::thread::spawn(move || {
let out = decode_all(hw.as_mut(), &samples[..30]);
(hw, out)
})
.join()
.unwrap();
assert_same("first thread", &out, &reference);
let mut hw = hw;
hw.reset();
let samples = s.samples.clone();
let out = std::thread::spawn(move || decode_all(hw.as_mut(), &samples[..30])).join().unwrap();
assert_same("second thread", &out, &reference);
}

12
crates/render/src/audio.rs
View File

@ -338,6 +338,18 @@ pub fn clip_signal(item: &TrackItem, start: i64, frames: usize, sr: u32, sources
Some(out)
}
/// Whether a clip plays one source channel on both sides: a mono source, or one picked channel
/// (Modify ▸ Audio Channels, Breakout to Mono). Loudness Auto-Match measures such a clip as the one
/// channel it is, as BS.1770 (and ffmpeg's `ebur128`) measure a mono programme, not as the
/// dual-mono pair [`clip_signal`] plays, which reads 3 LU louder. `None` when the clip has no
/// audio source.
pub fn clip_is_mono(item: &TrackItem, sources: &dyn SourceProvider) -> Option<bool> {
let src = sources.source(item.item)?;
let available = src.info().audio.as_ref()?.channels as usize;
let (left, right) = source_pair(item, available);
Some(left == right)
}
/// The clip's audio (after speed/reverse) for timeline samples `[x0, x0 + len)` as `w` channels
/// (2: stereo, mono sources on both sides; 6: 5.1), with silence outside the clip.
fn raw_channels(item: &TrackItem, src: SourceAudio, x0: i64, len: usize, sr: u32, w: usize) -> Vec<Vec<f32>> {

15
crates/render/src/lib.rs
View File

@ -489,11 +489,24 @@ fn plain_layer(frame: &VideoFrame, n: usize, opacity: f32, blend: Blend) -> Laye
/// convert it with.
fn media_frame(item: &TrackItem, t: Tick, sources: &dyn SourceProvider, want: f32, src_size: (u32, u32)) -> Option<(SharedSource, Arc<VideoFrame>, usize)> {
let src = sources.source(item.item)?;
let frame = src.video_frame(FrameRequest { time: item.source_time_at(t), scale: want }).ok()?;
let frame = src.video_frame(FrameRequest { time: video_source_time(item, t, src.info().frame_rate()), scale: want }).ok()?;
let n = decimation(frame.width as f32, src_size.0 as f32 * want);
Some((src, frame, n))
}
/// The media time to ask a source for the video frame a clip shows at timeline `t`. A reversed
/// clip's source time is the instant just before its mirrored position; readers that round the
/// request to the nearest unit of a coarse timescale (MP4) land on the next frame from there,
/// showing one frame past the clip's source range and never its In (#318). Snapped down to the
/// start of the `media_rate` frame that contains it, every reader shows that frame.
pub(crate) fn video_source_time(item: &TrackItem, t: Tick, media_rate: filmcraft_time::FrameRate) -> Tick {
let ft = item.source_time_at(t);
if !item.reverse || item.frame_hold.is_some() || media_rate.frame_duration().0 <= 0 {
return ft;
}
media_rate.tick_of(media_rate.frame_at(ft)).clamp(item.source_in.min(ft), ft)
}
/// The linear image of a media clip's `frame`, colour managed, and blended with the next frame when
/// the clip's speed asks for it.
#[allow(clippy::too_many_arguments)]

6
crates/render/src/plan.rs
View File

@ -275,8 +275,7 @@ fn push_item(
}
// Graphic clips without standard effects: the layers are rasterised (cached) into one tight
// image the GPU places as a layer.
if !(opts.effects && item.has_standard_effects())
&& !item.has_opacity_masks()
if !(opts.effects && item.has_standard_effects() || item.has_opacity_masks())
&& project.item(item.item).is_some_and(|p| matches!(p.kind, ItemKind::Graphic { .. }))
{
let Some(size) = crate::source_size(project, item.item) else { return };
@ -293,7 +292,8 @@ fn push_item(
let motion = motion_matrix(seq, item, size, mt);
let lin = ((motion.a * motion.a + motion.b * motion.b).sqrt()).max((motion.c * motion.c + motion.d * motion.d).sqrt());
let want = (lin * opts.scale as f64).clamp(1.0 / 64.0, 1.0) as f32;
let Ok(frame) = src.video_frame(FrameRequest { time: ft, scale: want }) else { return };
let time = crate::video_source_time(item, t, src.info().frame_rate());
let Ok(frame) = src.video_frame(FrameRequest { time, scale: want }) else { return };
let cs = crate::colorman::source_space(project, item.item, &frame);
// log / HDR / wide-gamut media is converted on the CPU (below), and so are blended
// in-between frames (Frame Blending / Optical Flow on speed-changed clips)

84
crates/render/src/tests.rs
View File

@ -369,3 +369,87 @@ fn a_multicam_clip_showing_itself_plans_and_renders_to_an_end() {
let img = render_sequence(&p, a, Tick::ZERO, opts, &sources);
assert!(img.w > 0 && img.h > 0);
}
/// A 24 fps source that picks frames like the MP4 reader: the requested time is converted to the
/// track timescale rounding to the nearest unit, then the frame at or before it is shown. It
/// records the frames it was asked for.
struct NearestUnitSource {
inner: GeneratorSource,
asked: std::sync::Mutex<Vec<i64>>,
}
impl MediaSource for NearestUnitSource {
fn info(&self) -> &filmcraft_media::MediaInfo {
self.inner.info()
}
fn video_frame(&self, req: FrameRequest) -> filmcraft_media::Result<Arc<filmcraft_frame::VideoFrame>> {
// 12288 units a second: 512 a frame at 24 fps
let units = req.time.to_rational_round(1, 12_288);
self.asked.lock().unwrap().push(units.div_euclid(512));
self.inner.video_frame(req)
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<filmcraft_frame::AudioBuffer> {
self.inner.audio(start, frames, sample_rate)
}
}
/// #318: a trimmed clip (source frames 10–29) played in reverse shows 29 down to 10. Its source
/// time is the instant just before the mirrored position; a reader that rounds to the nearest
/// timescale unit used to land on the next frame, showing frame 30 (outside the clip) and never 10.
#[test]
fn a_reversed_clip_shows_exactly_its_source_range() {
let r = FrameRate::FPS_24;
let mut p = Project::new("r");
let g = GeneratorSource::new(Generator::ColorMatte { color: [0.2, 0.4, 0.6, 1.0] }, 64, 64, r, Tick(4 * TICKS_PER_SECOND));
let info = g.info().clone();
let item = p.add_item(
"ids",
Label::Iris,
ItemKind::Media(MediaClip {
media: MediaRef::Generator(g.generator.clone()),
info,
interpret: Default::default(),
mark_in: None,
mark_out: None,
markers: vec![],
offline: false,
proxy: None,
identity: None,
}),
None,
);
let src = Arc::new(NearestUnitSource { inner: g, asked: Default::default() });
let mut map = SourceMap::default();
map.0.insert(item, src.clone() as SharedSource);
let seq = p.new_sequence("s", SequenceSettings { width: 64, height: 64, frame_rate: r, ..Default::default() }, 1, 0, None);
let ti = p.make_track_item(item, TrackKind::Video, Tick::ZERO, TimeRange::new(r.tick_of(10), r.tick_of(20)), r).unwrap();
let id = ti.id;
p.sequence_mut(seq).unwrap().video_tracks[0].items.push(ti);
let opts = RenderOptions { scale: 1.0, ..Default::default() };
let shown = |p: &Project, f: i64, planned: bool| -> Vec<i64> {
src.asked.lock().unwrap().clear();
if planned {
plan::execute_cpu(&plan::plan_frame(p, seq, r.tick_of(f), opts, &map));
} else {
render_sequence(p, seq, r.tick_of(f), opts, &map);
}
let mut asked = src.asked.lock().unwrap().clone();
asked.dedup();
asked
};
// forward control: 10, 17, 29
for (f, want) in [(0, 10), (7, 17), (19, 29)] {
for planned in [false, true] {
assert_eq!(shown(&p, f, planned), [want], "forward, timeline frame {f}, planned {planned}");
}
}
p.sequence_mut(seq).unwrap().find_item_mut(id).unwrap().1.reverse = true;
for (f, want) in [(0, 29), (7, 22), (19, 10)] {
for planned in [false, true] {
assert_eq!(shown(&p, f, planned), [want], "reversed, timeline frame {f}, planned {planned}");
}
}
// every timeline frame stays inside the clip's source range, each source frame shown once
let all: Vec<i64> = (0..20).flat_map(|f| shown(&p, f, false)).collect();
assert_eq!(all, (10..=29).rev().collect::<Vec<_>>());
}

2
crates/text/Cargo.toml
View File

@ -19,3 +19,5 @@ harfrust = { version = "0.12", default-features = false, features = ["std"] }
unicode-bidi = "0.3.18"
# UAX #14 line breaking (Apache-2.0).
unicode-linebreak = "0.1.5"
# UAX #29 extended grapheme clusters (MIT OR Apache-2.0).
unicode-segmentation = "1.13"

17
crates/text/src/fonts.rs
View File

@ -215,6 +215,23 @@ impl Face {
pub fn has_char(&self, c: char) -> bool {
self.glyph(c).is_some()
}
/// Height of the glyph's vertical origin above its baseline, in pixels.
/// OpenType VORG takes precedence; TrueType uses the ink top plus vmtx's top bearing.
pub(crate) fn vertical_origin(&self, gid: u32, px: f32) -> Option<f32> {
use skrifa::raw::TableProvider;
let font = self.font()?;
let glyph = skrifa::GlyphId::new(gid);
let units = match font.vorg() {
Ok(table) => f32::from(table.vertical_origin_y(glyph)),
Err(_) => {
let bounds = font.glyph_metrics(Size::unscaled(), LocationRef::default()).bounds(glyph)?;
bounds.y_max + f32::from(font.vmtx().ok()?.side_bearing(glyph)?)
}
};
let origin = units * px / self.units_per_em();
(origin.is_finite() && origin.abs() < px * 4.0).then_some(origin)
}
pub fn metrics(&self, px: f32) -> VMetrics {
let Some(f) = self.font() else {
return VMetrics {

126
crates/text/src/layout.rs
View File

@ -20,6 +20,7 @@ use std::sync::{Arc, Mutex, OnceLock};
use harfrust::{Direction, Feature, Tag, UnicodeBuffer};
use unicode_bidi::{BidiInfo, Level};
use unicode_segmentation::UnicodeSegmentation;
use crate::fonts::{self, FaceId, Resolved};
@ -218,7 +219,7 @@ pub struct Layout {
/// Lines were left out because they do not fit the box height.
pub overflow: bool,
pub text_len: usize,
/// Laid out vertically (one line per character; columns right to left).
/// Laid out vertically (one line per grapheme; columns right to left).
pub vertical: bool,
}
@ -319,7 +320,11 @@ fn upper_single(c: char) -> char {
const SMALL_CAPS_SCALE: f32 = 0.78;
fn shape_item(text: &str, chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, style: &TextStyle) -> Vec<ShapedGlyph> {
fn uses_vertical_form(c: char) -> bool {
matches!(c as u32, 0x2E80..=0x9FFF | 0xF900..=0xFAFF | 0xFE10..=0xFE1F | 0xFE30..=0xFE4F | 0xFF00..=0xFFEF | 0x20000..=0x3FFFF)
}
fn shape_item(text: &str, chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, style: &TextStyle, vertical: bool) -> Vec<ShapedGlyph> {
let f = fonts::face(face);
let (Some(font), Some(data)) = (f.font(), f.shaper_data()) else { return Vec::new() };
let shaper = data.shaper(&font).build();
@ -330,6 +335,10 @@ fn shape_item(text: &str, chars: &[(usize, char, char)], rtl: bool, face: FaceId
buf.set_direction(if rtl { Direction::RightToLeft } else { Direction::LeftToRight });
buf.guess_segment_properties();
let mut feats = Vec::new();
if vertical && chars.first().is_some_and(|c| uses_vertical_form(c.1)) {
feats.push(Feature::new(Tag::new(b"vert"), 1, ..));
feats.push(Feature::new(Tag::new(b"vrt2"), 1, ..));
}
if !style.kerning {
feats.push(Feature::new(Tag::new(b"kern"), 0, ..));
}
@ -416,7 +425,7 @@ fn paragraph(text: &str, base: usize, sty: &Styles, para: &ParagraphStyle) -> Ve
let st = sty.list[si];
let sub: Vec<(usize, char, char)> = chars[i..j].iter().map(|c| (c.0, c.1, c.2)).collect();
let size = if small { st.size * SMALL_CAPS_SCALE } else { st.size };
let mut glyphs = shape_item(text, &sub, rtl, face, size, st);
let mut glyphs = shape_item(text, &sub, rtl, face, size, st, para.vertical);
// tracking after each cluster
if st.tracking != 0.0 {
let t = st.tracking * st.size / 1000.0;
@ -707,13 +716,13 @@ pub fn layout_rich_uncached(text: &str, style: &TextStyle, runs: &[StyleRun], pa
lay
}
/// Vertical layout: each character is its own line (so carets, hit testing and selection work
/// Vertical layout: each grapheme is its own line (so carets, hit testing and selection work
/// unchanged), stacked top to bottom and centred on its column; columns run right to left.
fn layout_vertical(text: &str, sty: &Styles, para: &ParagraphStyle, col_w: f32, mut lay: Layout) -> Layout {
let style = sty.list[0];
let pm = fonts::face(sty.prim[0].face).metrics(style.size);
let row = (pm.ascent + pm.descent).max(style.size * 0.5) + style.tracking * style.size / 1000.0;
let flat = ParagraphStyle::default();
let flat = ParagraphStyle { vertical: true, ..Default::default() };
lay.vertical = true;
let mut base = 0usize;
let mut ncols = 0usize;
@ -724,8 +733,8 @@ fn layout_vertical(text: &str, sty: &Styles, para: &ParagraphStyle, col_w: f32,
if p_clean.is_empty() {
cells.extend(paragraph("", base, sty, &flat));
}
for (off, ch) in p_clean.char_indices() {
cells.extend(paragraph(&p_clean[off..off + ch.len_utf8()], base + off, sty, &flat));
for (off, cluster) in p_clean.grapheme_indices(true) {
cells.extend(paragraph(cluster, base + off, sty, &flat));
}
let h = row * cells.len() as f32;
let y0 = match para.align {
@ -740,7 +749,16 @@ fn layout_vertical(text: &str, sty: &Styles, para: &ParagraphStyle, col_w: f32,
let g0 = lay.glyphs.len();
lay.glyphs.extend(pl.glyphs.into_iter().map(|mut g| {
g.x += shift;
g.y += baseline;
// The alternate glyph hangs from its vertical origin. A horizontal ascent
// is not the top bearing of a vertical punctuation glyph (OpenType VORG/vmtx).
let font = fonts::face(g.face);
if text.get(g.cluster..).and_then(|s| s.chars().next()).is_some_and(uses_vertical_form)
&& let Some(origin) = font.vertical_origin(g.id, g.size)
{
g.y += baseline - pl.ascent + origin;
} else {
g.y += baseline;
}
g
}));
lay.lines.push(Line {
@ -910,6 +928,53 @@ mod tests {
assert!(l.glyphs[3].x < l.glyphs[0].x);
}
#[test]
fn japanese_vertical_punctuation_uses_alternates_at_the_top_right() {
use skrifa::{
MetadataProvider,
instance::{LocationRef, Size},
};
fonts::scan_system();
let face = fonts::all_faces().into_iter().find(|f| {
((f.info.family.contains("Hiragino") && !f.info.family.contains(" GB") && !f.info.family.contains(" TC"))
|| f.info.family.contains("Shippori")
|| f.info.family.contains("BIZ UD"))
&& "国、。「」ー".chars().all(|c| f.has_char(c))
&& (f.has_feature(b"vert") || f.has_feature(b"vrt2"))
});
let Some(face) = face else {
eprintln!("SKIPPED: no Japanese font with vertical alternates available");
return;
};
let style = TextStyle { family: face.info.family.clone(), style: face.info.style.clone(), ..st(48.0) };
let text = "国、。「」ーABC";
let horizontal = layout_uncached(text, &style, &ParagraphStyle::default());
let vertical = layout_uncached(text, &style, &ParagraphStyle { vertical: true, ..Default::default() });
assert_eq!(vertical.glyphs.len(), 9);
assert_eq!(vertical.glyphs[0].id, horizontal.glyphs[0].id);
for i in 1..6 {
assert_ne!(vertical.glyphs[i].id, horizontal.glyphs[i].id, "{}: vertical form for glyph {i}", face.info.family);
}
for i in [1, 2] {
let g = &vertical.glyphs[i];
let line = &vertical.lines[i];
let font = fonts::face(g.face);
let bounds = font.font().unwrap().glyph_metrics(Size::unscaled(), LocationRef::default()).bounds(skrifa::GlyphId::new(g.id)).unwrap();
let scale = g.size / font.units_per_em();
let ink_x = g.x + (bounds.x_min + bounds.x_max) * scale / 2.0;
let ink_y = g.y - (bounds.y_min + bounds.y_max) * scale / 2.0;
let centre_x = line.x + line.width / 2.0;
let cell_top = line.baseline - line.ascent;
assert!(ink_x > centre_x, "{}: punctuation ink on the right ({ink_x}, {centre_x})", face.info.family);
assert!(ink_y < cell_top + g.size / 2.0, "{}: punctuation ink at the top ({ink_y}, {cell_top})", face.info.family);
}
for i in 6..9 {
assert_eq!(vertical.glyphs[i].id, horizontal.glyphs[i].id, "Latin shaping stays unchanged");
}
assert_eq!(layout_uncached(text, &style, &ParagraphStyle::default()), horizontal);
eprintln!("verified vertical punctuation with {} / {}", face.info.family, face.info.style);
}
#[test]
fn vertical_text_stacks_characters_in_columns() {
let p = ParagraphStyle { vertical: true, ..Default::default() };
@ -933,6 +998,51 @@ mod tests {
assert_eq!(h.lines.len(), 2);
}
#[test]
fn vertical_cells_keep_combining_marks_with_their_base() {
let style = st(32.0);
let para = ParagraphStyle { vertical: true, ..Default::default() };
let decomposed = layout_uncached("e\u{301}x", &style, &para);
let composed = layout_uncached("éx", &style, &para);
assert_eq!(decomposed.lines.len(), 2, "the combining acute must not get another vertical cell");
assert_eq!(decomposed.lines[0].range, 0..3);
assert_eq!(decomposed.lines[1].range, 3..4);
assert_eq!(decomposed.text_len, 4, "retain the original UTF-8 source offsets");
assert_eq!(
decomposed.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::<Vec<_>>(),
composed.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::<Vec<_>>()
);
assert_eq!(decomposed.bounds, composed.bounds);
assert_eq!(decomposed.caret(0).1, decomposed.caret(1).1, "base and combining mark occupy the same cell");
for text in ["葛\u{e0101}x", "👩\u{200d}👧x"] {
let lay = layout_uncached(text, &style, &para);
assert_eq!(lay.lines.len(), 2, "variation selectors and ZWJ sequences stay with their base");
assert_eq!(lay.lines[0].range, 0..text.len() - 1);
assert_eq!(lay.lines[1].range, text.len() - 1..text.len());
}
}
#[test]
fn vertical_kana_keep_decomposed_dakuten_in_the_same_cell() {
fonts::scan_system();
let Some(face) = fonts::all_faces().into_iter().find(|f| {
(f.info.family.contains("Hiragino") || f.info.family.contains("Shippori") || f.info.family.contains("BIZ UD"))
&& "かがはぱ\u{3099}\u{309a}".chars().all(|c| f.has_char(c))
}) else {
eprintln!("SKIPPED: no Japanese font covering combining dakuten/handakuten");
return;
};
let style = TextStyle { family: face.info.family.clone(), style: face.info.style.clone(), ..st(48.0) };
let para = ParagraphStyle { vertical: true, ..Default::default() };
let nfd = layout_uncached("か\u{3099}は\u{309a}", &style, &para);
let nfc = layout_uncached("がぱ", &style, &para);
assert_eq!(nfd.lines.len(), 2);
assert_eq!(nfd.lines[0].range, 0..6);
assert_eq!(nfd.lines[1].range, 6..12);
assert_eq!(nfd.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::<Vec<_>>(), nfc.glyphs.iter().map(|g| (g.id, g.x, g.y)).collect::<Vec<_>>());
assert_eq!(nfd.bounds, nfc.bounds);
}
#[test]
fn carets_and_hit_testing() {
let l = layout("abc\nde", &st(40.0), &ParagraphStyle::default());

102
crates/time/src/lib.rs
View File

@ -224,11 +224,20 @@ impl FrameRate {
(t.0 as i128 * r.num as i128).div_euclid(TICKS_PER_SECOND as i128 * r.den as i128) as i64
}
/// Start tick of frame `f`.
/// Start tick of frame `f` (ceiling division).
///
/// Uses ceiling (not floor) so that `frame_at(tick_of(f)) == f` holds for every
/// frame index, even when the frame duration is not an integer number of ticks
/// (exotic rates like 37.516 fps where `TICKS_PER_SECOND * den / num` has a
/// fractional remainder). Floor division truncates the remainder, causing each
/// frame boundary to drift backward by one tick until `frame_at` returns `f - 1`
/// instead of `f` — frame stepping then stalls or moves backward.
/// Ceiling is correct because the frame duration is always > 1 tick, so
/// `floor(ceil(f·D) / D) == f` for every integer `f`.
pub fn tick_of(self, f: i64) -> Tick {
let r = self.sane();
let n = f as i128 * TICKS_PER_SECOND as i128 * r.den as i128;
Tick(n.div_euclid(r.num as i128) as i64)
Tick((-(-n).div_euclid(r.num as i128)) as i64)
}
/// Snap `t` down to a frame boundary.
@ -571,6 +580,85 @@ mod tests {
assert_eq!(FrameRate::FPS_23_976.label(), "23.976");
}
#[test]
fn exotic_frame_rate_roundtrip() {
// Issue #301: 37.516 fps footage. The frame duration is
// 254_016_000_000 * 250 / 9379 = 6_770_871_094.999… ticks — not an
// integer. Floor division in `tick_of` truncated to 6_770_871_094,
// losing ~1 tick/frame so `frame_at(tick_of(f))` returned `f - 1`.
// Ceiling division makes the round-trip exact for all integer frames.
let rates: &[(FrameRate, &str)] = &[
(FrameRate::from_f64(37.516), "37.516"),
(FrameRate::from_f64(37.5161), "37.5161"),
(FrameRate::from_f64(12.345), "12.345"),
(FrameRate::from_f64(59.9401), "59.9401"),
(FrameRate::from_f64(23.9761), "23.9761"),
];
for &(r, name) in rates {
// Round-trip over a wide range including negatives.
let bad: Vec<i64> = (-20_000..20_000).filter(|&f| r.frame_at(r.tick_of(f)) != f).collect();
assert!(bad.is_empty(), "{name}: {} round-trip failures, first {bad:?}", r);
// snap(tick_of(f)) == tick_of(f): snapping a frame boundary is a no-op.
for f in -100i64..100 {
assert_eq!(r.snap(r.tick_of(f)), r.tick_of(f), "{name}: snap(tick_of({f}))");
}
// Step-back: the last tick of frame f is one tick before tick_of(f+1),
// and frame_at of that is still f. Also check the frame_duration path:
// frame_at(tick_of(f) - frame_duration()) == f - 1.
for f in 1..2000i64 {
let last_tick = r.tick_of(f + 1) - Tick(1);
assert_eq!(r.frame_at(last_tick), f, "{name}: last tick of frame {f}");
let prev = r.tick_of(f) - r.frame_duration();
assert_eq!(r.frame_at(prev), f - 1, "{name}: tick_of({f}) - frame_duration");
}
}
// Stepping simulation: frame_at(playhead) + 1, tick_of, snap.
// Before the fix this stalled at frame 0 (step forward did nothing).
let r = FrameRate::from_f64(37.516);
let mut tick = Tick::ZERO;
for expected in 0..50 {
let frame = r.frame_at(tick);
assert_eq!(frame, expected, "stepping forward at frame {expected}");
tick = r.tick_of(frame + 1);
tick = r.snap(tick); // set_playhead calls snap
}
// Stepping backward from frame 50 should land on 49, 48, … 0
tick = r.tick_of(50);
let mut last = 50;
while last > 0 {
let frame = r.frame_at(tick);
let target = frame - 1;
if target < 0 {
break;
}
tick = r.snap(r.tick_of(target));
let landed = r.frame_at(tick);
assert_eq!(landed, target, "stepping backward to frame {target}");
last = landed;
}
}
#[test]
fn exotic_frame_rate_durations() {
// frame_duration uses floor (not round) so that `end - frame_duration()`
// lands on the previous frame, not two frames back, for exotic rates.
let r = FrameRate::from_f64(37.516);
// 254_016_000_000 * 250 / 9379 = 6_770_871_094.999… → floor = 6_770_871_094
assert_eq!(r.frame_duration().0, 6_770_871_094);
// Standard rates must remain exact (unchanged).
assert_eq!(FrameRate::FPS_23_976.frame_duration().0, 10_594_584_000);
assert_eq!(FrameRate::FPS_24.frame_duration().0, 10_584_000_000);
assert_eq!(FrameRate::FPS_25.frame_duration().0, 10_160_640_000);
assert_eq!(FrameRate::FPS_29_97.frame_duration().0, 8_475_667_200);
assert_eq!(FrameRate::FPS_30.frame_duration().0, 8_467_200_000);
assert_eq!(FrameRate::FPS_60.frame_duration().0, 4_233_600_000);
}
proptest! {
#[test]
fn frame_tick_roundtrip(f in -1_000_000i64..10_000_000, ri in 0usize..11) {
@ -579,6 +667,16 @@ mod tests {
prop_assert_eq!(r.frame_at(r.tick_of(f) + r.frame_duration() - Tick(1)), f);
}
#[test]
fn exotic_frame_tick_roundtrip(f in -100_000i64..100_000, num in 1u64..200_000, den in 1u64..2_000) {
// Skip rates that divide evenly (those are covered by COMMON above).
let r = FrameRate::new(num as i64, den as i64);
if (TICKS_PER_SECOND as i128 * r.den as i128) % r.num as i128 == 0 {
return Ok(());
}
prop_assert_eq!(r.frame_at(r.tick_of(f)), f, "rate={}/{}, f={}", r.num, r.den, f);
}
#[test]
fn drop_frame_bijection(f in 0i64..(24 * 107_892)) {
let r = FrameRate::FPS_29_97;

88
crates/ui-egui/examples/thumb_burst.rs Normal file
View File

@ -0,0 +1,88 @@
//! Memory of the thumbnail burst that follows an import (issue: 6+ GB while importing 42 4K H.264
//! clips with hardware decoding off).
//!
//! Imports every `*.mp4` of a directory one by one, as the Project panel does, and asks for each
//! clip's poster thumbnail (priority 50, 160 px) through the real [`FrameServer`] right after its
//! import. Prints the wall time until every thumbnail is ready and the peak number of live
//! decoders. Measure the process' private bytes from outside (`stress.py`'s sampler, or any
//! process monitor).
//!
//! ```sh
//! cargo run --release -p filmcraft-ui-egui --example thumb_burst -- <clips dir> [--hw off|auto] [--limit N]
//! ```
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use filmcraft_engine::Session;
use filmcraft_project::ItemId;
use filmcraft_ui_egui::frames::{FrameKey, FrameServer, Target};
use serde_json::json;
fn main() {
let args: Vec<String> = std::env::args().skip(1).collect();
let value = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
let Some(dir) = args.first().filter(|a| !a.starts_with("--")).map(PathBuf::from) else {
eprintln!("usage: thumb_burst <clips dir> [--hw off|auto] [--limit N]");
return;
};
filmcraft_codecs::hw::set_hardware_decoding(value("--hw").as_deref() != Some("off"));
let limit = value("--limit").and_then(|v| v.parse().ok()).unwrap_or(usize::MAX);
let mut clips: Vec<PathBuf> = std::fs::read_dir(&dir).map(|d| d.filter_map(|e| e.ok().map(|e| e.path())).collect()).unwrap_or_default();
clips.retain(|p| p.extension().is_some_and(|e| e == "mp4"));
clips.sort();
clips.truncate(limit);
let mut s = Session::default();
let server = FrameServer::new(s.media.clone(), s.services.clone(), s.previews.clone(), FrameServer::default_workers());
let peak = std::sync::Arc::new(AtomicUsize::new(0));
let stop = std::sync::Arc::new(AtomicBool::new(false));
let sampler = {
let (peak, stop) = (peak.clone(), stop.clone());
std::thread::spawn(move || {
while !stop.load(Ordering::Relaxed) {
peak.fetch_max(filmcraft_codecs::live_decoders(), Ordering::Relaxed);
std::thread::sleep(Duration::from_millis(5));
}
})
};
let t0 = Instant::now();
let mut keys = Vec::new();
for path in &clips {
let r = s.execute("file.import", json!({"paths": [path.to_string_lossy()]})).expect("import");
let id = ItemId(r["items"][0].as_u64().expect("item"));
let (rate, width) = match s.project.item(id).map(|i| (i.frame_rate(), &i.kind)) {
Some((rate, filmcraft_project::ItemKind::Media(m))) => (rate, m.info.video.as_ref().map_or(1920, |v| v.width)),
_ => continue,
};
let rev = {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
if let Some(filmcraft_project::ItemKind::Media(m)) = s.project.item(id).map(|i| &i.kind) {
filmcraft_engine::media_pool::media_key(m).hash(&mut h);
false.hash(&mut h);
}
h.finish()
};
let key = FrameKey { target: Target::Item(id), frame: 0, size: 160, revision: rev, draft: false };
server.request(key, rate.tick_of(0), 160.0 / width.max(1) as f32, &s.project.clone(), 50);
keys.push(key);
}
let imported = t0.elapsed();
while keys.iter().any(|k| !server.is_ready(k)) && t0.elapsed() < Duration::from_secs(300) {
std::thread::sleep(Duration::from_millis(10));
}
stop.store(true, Ordering::Relaxed);
let _ = sampler.join();
println!(
"clips {} imported in {:.2}s, all thumbnails ready after {:.2}s, peak live decoders {}, gop frames cached {:.0} MB",
keys.len(),
imported.as_secs_f64(),
t0.elapsed().as_secs_f64(),
peak.load(Ordering::Relaxed),
filmcraft_codecs::cached_bytes() as f64 / 1048576.0
);
}

Some files were not shown because too many files have changed in this diff Show More