shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into fix-1198

This commit is contained in:
Brandon Thomas 2026-10-11 23:13:26 -04:00
parent 24f4ed00a8 ca36cf9a6a
commit a1726ac09b
362 changed files with 42552 additions and 2493 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ATTRIBUTION.md
View File

@ -9,7 +9,9 @@ iconography, images or other Adobe assets; open licences only; every asset attri
| Asset | Author | Source | Licence |
|---|---|---|---|
| `crates/ui-egui/src/i18n/id.tsv` | FilmCraft contributors | Original work: clean-room Indonesian interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/fr.tsv` | FilmCraft contributors | Original work: clean-room French interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/pt-br.tsv` | FilmCraft contributors | Original work: clean-room Brazilian Portuguese interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/ru.tsv` | FilmCraft contributors | Original work: clean-room Russian interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/uk.tsv` | FilmCraft contributors | Original work: clean-room Ukrainian interface translations | MIT OR Apache-2.0 |
| `assets/fonts/Inter-Regular.ttf` | The Inter Project Authors | https://github.com/rsms/inter | OFL-1.1 (`assets/fonts/OFL-Inter.txt`) |

54
Cargo.lock generated
View File

@ -1696,6 +1696,7 @@ dependencies = [
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2",
"open",
"pollster 0.4.0",
"rfd",
"serde_json",
"sys-locale",
@ -1766,6 +1767,13 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-avi"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
]
[[package]]
name = "filmcraft-bitstream"
version = "0.6.0"
@ -1806,6 +1814,7 @@ dependencies = [
"filmcraft-platform",
"filmcraft-project",
"filmcraft-render",
"filmcraft-testkit",
"filmcraft-time",
"serde_json",
"tokio",
@ -1819,6 +1828,7 @@ dependencies = [
"filmcraft-ac3",
"filmcraft-apv",
"filmcraft-av1",
"filmcraft-avi",
"filmcraft-bitstream",
"filmcraft-color",
"filmcraft-dnx",
@ -1828,6 +1838,7 @@ dependencies = [
"filmcraft-isobmff",
"filmcraft-matroska",
"filmcraft-media",
"filmcraft-mpa",
"filmcraft-mpeg2v",
"filmcraft-mpegts",
"filmcraft-mxf",
@ -1836,9 +1847,11 @@ dependencies = [
"filmcraft-prores",
"filmcraft-testkit",
"filmcraft-time",
"filmcraft-vp8",
"filmcraft-vp9",
"image",
"log",
"rayon",
"serde_json",
"symphonia",
"thiserror 2.0.21",
@ -1853,6 +1866,15 @@ dependencies = [
"serde",
]
[[package]]
name = "filmcraft-dng"
version = "0.6.0"
dependencies = [
"filmcraft-dng",
"rayon",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-dnx"
version = "0.6.0"
@ -1884,12 +1906,14 @@ dependencies = [
"filmcraft-captions",
"filmcraft-codecs",
"filmcraft-color",
"filmcraft-dng",
"filmcraft-edit",
"filmcraft-export",
"filmcraft-format",
"filmcraft-frame",
"filmcraft-geom",
"filmcraft-interchange",
"filmcraft-isobmff",
"filmcraft-media",
"filmcraft-mxf",
"filmcraft-project",
@ -1918,6 +1942,7 @@ dependencies = [
"filmcraft-codecs",
"filmcraft-color",
"filmcraft-dnx",
"filmcraft-flac",
"filmcraft-frame",
"filmcraft-geom",
"filmcraft-h264enc",
@ -1938,6 +1963,15 @@ dependencies = [
"web-time",
]
[[package]]
name = "filmcraft-flac"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-format"
version = "0.6.0"
@ -2076,14 +2110,26 @@ name = "filmcraft-media"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"filmcraft-dng",
"filmcraft-frame",
"filmcraft-testkit",
"filmcraft-time",
"image",
"png",
"rayon",
"serde",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-mpa"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-mpeg2v"
version = "0.6.0"
@ -2303,9 +2349,17 @@ dependencies = [
"serde",
"serde_json",
"toml 1.1.6+spec-1.1.0",
"unicode-bidi",
"web-time",
]
[[package]]
name = "filmcraft-vp8"
version = "0.6.0"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-vp9"
version = "0.6.0"

14
Cargo.toml
View File

@ -17,21 +17,26 @@ filmcraft-geom = { path = "crates/geom" }
filmcraft-color = { path = "crates/color" }
filmcraft-bitstream = { path = "crates/bitstream" }
filmcraft-isobmff = { path = "crates/isobmff" }
filmcraft-avi = { path = "crates/avi" }
filmcraft-matroska = { path = "crates/matroska" }
filmcraft-mxf = { path = "crates/mxf" }
filmcraft-cfb = { path = "crates/cfb" }
filmcraft-mpegts = { path = "crates/mpegts" }
filmcraft-mpeg2v = { path = "crates/mpeg2v" }
filmcraft-ac3 = { path = "crates/ac3" }
filmcraft-mpa = { path = "crates/mpa" }
filmcraft-ogg = { path = "crates/ogg" }
filmcraft-h264 = { path = "crates/h264" }
filmcraft-hevc = { path = "crates/hevc" }
filmcraft-vp8 = { path = "crates/vp8" }
filmcraft-vp9 = { path = "crates/vp9" }
filmcraft-prores = { path = "crates/prores" }
filmcraft-dnx = { path = "crates/dnx" }
filmcraft-apv = { path = "crates/apv" }
filmcraft-dng = { path = "crates/dng" }
filmcraft-av1 = { path = "crates/av1" }
filmcraft-aac = { path = "crates/aac" }
filmcraft-flac = { path = "crates/flac" }
filmcraft-opus = { path = "crates/opus" }
filmcraft-h264enc = { path = "crates/h264enc" }
filmcraft-audio-dsp = { path = "crates/audio-dsp" }
@ -109,12 +114,18 @@ opt-level = 3
[profile.dev.package.filmcraft-apv]
opt-level = 3
[profile.dev.package.filmcraft-dng]
opt-level = 3
[profile.dev.package.filmcraft-av1]
opt-level = 3
[profile.dev.package.filmcraft-aac]
opt-level = 3
[profile.dev.package.filmcraft-flac]
opt-level = 3
# Audio effects: realtime-factor tests and long convolution / spectral runs.
[profile.dev.package.filmcraft-audio-dsp]
opt-level = 3
@ -125,6 +136,9 @@ opt-level = 3
[profile.dev.package.filmcraft-opus]
opt-level = 3
[profile.dev.package.filmcraft-vp8]
opt-level = 3
[profile.dev.package.filmcraft-vp9]
opt-level = 3

6
README.md
View File

@ -175,6 +175,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus
| **HEVC / H.265** | ✓ | | Main and Main 10, bit-exact on 41 streams (tiles, WPP, PCM, long-term references), about 225 fps at 1080p |
| **Apple ProRes** | ✓ | ✓ | Decodes 422 Proxy to 4444 XQ; export writes 422 HQ |
| **AAC-LC** | ✓ | ✓ | |
| **VP8** | ✓ | | Every coding tool, bitstream versions 0–3, hidden altref frames, alpha layer of transparent WebM; bit-exact with libvpx on its published VP8 test vectors; in WebM/Matroska and MP4 |
| **VP9** | ✓ | | Profiles 0–3, 8/10/12-bit, 4:2:0 to 4:4:4, tiles and superframes; bit-exact on 50+ streams; in WebM/Matroska and MP4 |
| **Opus** | ✓ | | SILK, CELT, hybrid and multistream surround; range-exact on every RFC 8251 conformance vector; in WebM/Matroska and MP4 |
| **AV1** | ✓ | | Main profile, 8/10-bit 4:2:0 and monochrome, every coding tool incl. film grain, superres and spatial layers; bit-exact with libdav1d on the libaom vectors tested; in MP4 and WebM/Matroska |
@ -183,7 +184,8 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus
| **MP3, FLAC, ALAC, Vorbis** | ✓ | | Via the [symphonia](https://github.com/pdeljanov/Symphonia) crate (MPL-2.0) for now, to be replaced by our own |
| **MP4 / MOV** | ✓ | ✓ | Fragmented MP4, edit lists, timecode tracks |
| **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata |
| **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF |
| **AVI** | ✓ | | AVI 1.0 and OpenDML (files past 1 GB); Motion JPEG, H.264, HEVC and uncompressed RGB / YUV with PCM, MP3, MP2 or AC-3. DivX / Xvid and DV not yet |
| **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP (animated GIF and APNG as clips, with transparency); export PNG sequences and animated GIF |
Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows and Linux (opt-in), and H.265 (HEVC Main, 8-bit) export works through it (NVENC on Windows, VideoToolbox on macOS) ([#30](https://github.com/storytold/filmcraft/issues/30)).
@ -304,7 +306,7 @@ A layered Cargo workspace:
| Layer | Crates |
|---|---|
| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `bitstream` |
| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `avi`, `bitstream` |
| Foundations | `time`, `geom`, `color`, `frame`, `media`, `text`, `project`, `audio-dsp` |
| Editing and interchange | `edit`, `codecs`, `captions`, `format` (project files), `interchange` |
| Rendering and output | `render`, `gpu`, `export` |

9
ROADMAP.md
View File

@ -5,7 +5,7 @@
**Stage: alpha** · next: beta, ~19 points of ready-for-real-work and ~400–700 h away, plus opening Premiere's `.prproj` ([why](docs/target-app-parity.md#stage); passes the [core-workflow gate](docs/roadmap.md#alpha-gate))
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (full re-measure against Premiere Pro 2026 26.5.2; scorecard, gaps, milestones and localization moved into docs/ per the craftrules progress-docs standard) · **Target:** Adobe Premiere Pro 2026 (26.5.2)
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (language table: German ships the menus, #630) · **Target:** Adobe Premiere Pro 2026 (26.5.2)
Progress toward parity with Adobe Premiere Pro. This page is the summary; the detail is in
[docs/target-app-parity.md](docs/target-app-parity.md) (assessment, methodology, calibration),
@ -82,10 +82,10 @@ Premiere ships 10 interface languages. Measured from our catalogs (3,498 strings
| Hindi | none (no Devanagari shaping in the interface) | 0% |
| Arabic | none (no RTL interface) | 0% |
| French | partial (whole interface; native review welcome) | ~100% |
| Portuguese (Brazil) | menus only | 9.9% |
| Indonesian | none | 0% |
| Portuguese (Brazil) | partial (whole interface; native review welcome) | ~100% |
| Indonesian | partial (whole interface; native review welcome) | ~100% |
| Japanese | partial (whole interface) | 99.7% |
| German | none | 0% |
| German | menus only (native-reviewed) | 10.6% |
| Korean | none | 0% |
| Vietnamese | none | 0% |
@ -164,6 +164,7 @@ Ranked; detail and "done when" in [docs/roadmap.md](docs/roadmap.md) and [docs/g
| Date | Change | Summary |
|---|---|---|
| 2026-10-11 | minor | Language table: German ships the menus (#630, 10.6%) |
| 2026-10-10 | minor | Readiness-by-audience table with hours to ~95% per audience |
| 2026-10-10 | minor | Headline adds mainstream practitioner (~55%) and essentials user (~60%); ready for real work ~55% -> ~56% (formula written out, same inputs); beta distance ~19 points |
| 2026-10-10 | minor | Stage banner links the core-workflow gate (all six workflows pass on macOS; stays alpha) |

3
apps/filmcraft-cli/Cargo.toml
View File

@ -33,3 +33,6 @@ filmcraft-ogg = { workspace = true }
filmcraft-automation = { workspace = true }
serde_json = { workspace = true }
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
[dev-dependencies]
filmcraft-testkit = { workspace = true }

114
apps/filmcraft-cli/src/main.rs
View File

@ -76,7 +76,7 @@ SUBCOMMANDS
import <file>... import media into the project
export <out> [--preset name] [--format f] [--range r] [--start s --end s] [--settings json]
[--gpu-rendering auto|off]
[--scale f] [--quality 0-100] [--no-audio] [--queue]
[--scale f] [--quality 0-100] [--no-audio] [--queue] [--allow-offline]
export the active sequence and wait for it to finish: with an
export preset (`export --list-presets`; built-in or the user's), or
a format (h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff|aac, guessed
@ -85,9 +85,12 @@ SUBCOMMANDS
preset; --scale renders at a fraction of the frame size (0.5 =
half); --quality 0-100 for the formats that take one; --no-audio
leaves the sound out; --queue adds to the export queue and runs it
instead
instead. Offline media (missing, made offline, unreadable) renders
as placeholder frames: the export then lists it in `offlineMedia`,
warns on stderr and exits 1, unless --allow-offline accepts it
export --list-presets [query] list export presets (name, category, format) as JSON
render --seconds S --out f.png [--scale 0.5] render one Program frame to PNG
render --seconds S --out f.png [--scale 0.5] [--allow-offline]
render one Program frame to PNG (offline media: as for export)
probe <media> [--image-sequence]
media info as JSON, with MXF / Ogg / BWF details; with
--image-sequence <media> is the first numbered still of a sequence
@ -113,7 +116,8 @@ VALUES
Times: 254016000000 ticks per second; most commands also take seconds=, frame= or timecode=.
EXIT STATUS
0 success · 1 a command failed · 2 usage error
0 success · 1 a command failed, or export/render drew offline media without --allow-offline
· 2 usage error
";
/// Writes one diagnostic line to `w`, dropping any write error. `eprintln!` panics (exit 101) when stderr's
@ -244,6 +248,13 @@ fn register_hardware_decoders() -> filmcraft_platform::Availability {
filmcraft_platform::register()
}
/// `path` made absolute against the working directory, as the project stores it. Unlike
/// `canonicalize`, `std::path::absolute` keeps symlinks and adds no Windows verbatim `\\?\`
/// prefix (#850). A path it cannot resolve is kept as given.
fn absolute(path: &str) -> String {
std::path::absolute(path).ok().and_then(|p| p.into_os_string().into_string().ok()).unwrap_or_else(|| path.to_string())
}
#[tokio::main]
async fn main() {
cli().await;
@ -253,7 +264,8 @@ async fn main() {
}
async fn cli() {
let a = Args::parse(std::env::args().skip(1));
// `std::env::args` panics on an argument that is not Unicode (a Linux file name in Latin-1)
let a = Args::parse(std::env::args_os().skip(1).map(|a| a.to_string_lossy().into_owned()));
// `--help` anywhere (`filmcraft-cli --help`, `filmcraft-cli export --help`) prints the reference:
// the parser takes it as a flag option, so it never reaches the subcommand match.
if a.flag("--help") {
@ -373,8 +385,7 @@ async fn cli() {
print(&a, &v);
}
"import" => {
let paths: Vec<String> =
a.positionals[1..].iter().map(|p| std::fs::canonicalize(p).map(|c| c.to_string_lossy().into_owned()).unwrap_or_else(|_| p.clone())).collect();
let paths: Vec<String> = a.positionals.get(1..).unwrap_or_default().iter().map(|p| absolute(p)).collect();
if paths.is_empty() {
usage("import <file>...");
}
@ -442,7 +453,9 @@ async fn cli() {
match r {
Ok(v) => {
print(&a, &v);
diag(format_args!("exported {} in {:.1}s", written_paths(&v, out).join(", "), t0.elapsed().as_secs_f64()));
let written = written_paths(&v, out).join(", ");
check_offline(&a, &offline_media(&v), &format!("export: {written}"));
diag(format_args!("exported {written} in {:.1}s", t0.elapsed().as_secs_f64()));
}
Err(e) => fail(format!("export: {e}")),
}
@ -457,10 +470,12 @@ async fn cli() {
let out = a.opt("--out").unwrap_or("frame.png");
s.set_playhead(filmcraft_time::Tick::from_seconds_f64(secs));
let t0 = std::time::Instant::now();
let img = render_at_playhead(&s, scale).unwrap_or_else(|e| fail(e));
let (img, offline) = render_at_playhead(&s, scale).unwrap_or_else(|e| fail(e));
let dt = t0.elapsed();
let png = filmcraft_automation::png_rgba(img.w as u32, img.h as u32, img.over_black_rgba8(), 0).unwrap_or_else(|e| fail(e));
std::fs::write(out, png).unwrap_or_else(|e| fail(format!("{out}: {e}")));
let offline: Vec<Value> = offline.iter().filter_map(|o| serde_json::to_value(o).ok()).collect();
check_offline(&a, &offline, &format!("render: {out}"));
diag(format_args!("rendered {}x{} in {:.1} ms → {out}", img.w, img.h, dt.as_secs_f64() * 1000.0));
}
"mcp" => {
@ -468,7 +483,7 @@ async fn cli() {
usage(format!("unknown mcp option `{opt}`"));
}
let server = match a.opt("--bridge") {
Some(addr) => filmcraft_automation::FilmcraftMcp::bridge(addr).unwrap_or_else(|e| fail(e)),
Some(addr) => filmcraft_automation::FilmcraftMcp::bridge(addr).unwrap_or_else(|e| usage(e)),
None => match Backend::open(&a) {
Backend::Local(s) => filmcraft_automation::FilmcraftMcp::headless(*s),
Backend::Bridge(_) => usage("mcp: use --bridge ADDR to drive a running app"),
@ -500,9 +515,46 @@ fn written_paths(result: &Value, requested: &str) -> Vec<String> {
if paths.is_empty() { vec![requested.to_string()] } else { paths }
}
/// The active sequence at the playhead, or the reason it cannot be rendered (no sequence, bad scale).
fn render_at_playhead(s: &Session, scale: f32) -> Result<filmcraft_engine::render::Image, String> {
s.try_render_program_at(scale, s.playhead()).map_err(|e| e.to_string())
/// The offline items an export result reports (`offlineMedia` of a direct export, or of each
/// queued item), once each.
fn offline_media(result: &Value) -> Vec<Value> {
let lists = std::iter::once(result).chain(result.get("items").and_then(Value::as_array).into_iter().flatten());
let mut out: Vec<Value> = Vec::new();
for o in lists.filter_map(|r| r.get("offlineMedia").and_then(Value::as_array)).flatten() {
if !out.iter().any(|x| x.get("item") == o.get("item")) {
out.push(o.clone());
}
}
out
}
/// One stderr line per offline item that rendered as placeholder frames (#1054).
fn offline_warning(offline: &[Value]) -> String {
let s = |o: &Value, k: &str| o.get(k).and_then(Value::as_str).unwrap_or("").to_string();
let mut msg = format!("filmcraft-cli: warning: {} offline media item(s) rendered as placeholder frames:", offline.len());
for o in offline {
let id = o.get("item").and_then(Value::as_u64).map(|i| i.to_string()).unwrap_or_default();
msg.push_str(&format!("\n item {id} {} ({}): {}", s(o, "name"), s(o, "status"), s(o, "path")));
}
msg
}
/// Warn about offline media on stderr; without `--allow-offline` that fails the run (exit 1), after
/// the output (`what`) was written, so scripts don't take placeholder frames for the real picture.
fn check_offline(a: &Args, offline: &[Value], what: &str) {
if offline.is_empty() {
return;
}
diag(format_args!("{}", offline_warning(offline)));
if !a.flag("--allow-offline") {
fail(format!("{what} has placeholder frames for offline media; relink it (exec media.relink …) or pass --allow-offline"));
}
}
/// The active sequence at the playhead with the offline items it drew, or the reason it cannot be
/// rendered (no sequence, bad scale).
fn render_at_playhead(s: &Session, scale: f32) -> Result<(filmcraft_engine::render::Image, Vec<filmcraft_engine::media_pool::OfflineMedia>), String> {
s.try_render_program_with_offline_at(scale, s.playhead()).map_err(|e| e.to_string())
}
/// `--start` / `--end` ask for a custom range. A lone bound is forwarded too, so the engine reports the missing one
@ -531,7 +583,7 @@ mod format_tests {
let e = super::render_at_playhead(&s, scale).unwrap_err();
assert!(e.contains("finite and positive"), "{e}");
}
assert_eq!(super::render_at_playhead(&s, 0.5).unwrap().w, 8);
assert_eq!(super::render_at_playhead(&s, 0.5).unwrap().0.w, 8);
}
/// The CLI (and MCP / headless runs, which share its entry point) registers the hardware
@ -573,6 +625,19 @@ mod format_tests {
assert_eq!(super::written_paths(&json!({"items": []}), "x.mov"), ["x.mov"]);
}
#[test]
fn offline_media_from_direct_and_queued_results() {
use serde_json::json;
let a = json!({"item": 8, "name": "a.mp4", "path": "/m/a.mp4", "status": "missing", "error": ""});
let b = json!({"item": 9, "name": "b.mov", "path": "/m/b.mov", "status": "offline", "error": ""});
assert_eq!(super::offline_media(&json!({"path": "x.mp4", "offlineMedia": [a]})), std::slice::from_ref(&a));
let queued = json!({"items": [{"offlineMedia": [a, b]}, {"offlineMedia": [a]}, {"offlineMedia": []}]});
assert_eq!(super::offline_media(&queued), [a.clone(), b.clone()]);
assert!(super::offline_media(&json!({"path": "x.mp4"})).is_empty());
let w = super::offline_warning(&[a, b]);
assert!(w.contains("2 offline media item(s)") && w.contains("item 8 a.mp4 (missing): /m/a.mp4") && w.contains("item 9 b.mov (offline)"), "{w}");
}
#[test]
fn export_lone_bound_is_forwarded_as_custom_range() {
use serde_json::json;
@ -590,6 +655,27 @@ mod format_tests {
assert!(p.get("range").is_none());
}
/// `import` used `canonicalize`, which resolves symlinks and on Windows stores `\\?\C:\…`
/// verbatim paths in the project (#850).
#[test]
fn imported_paths_are_absolute_without_resolving_links() {
let cwd = std::env::current_dir().unwrap();
assert_eq!(super::absolute("clip.mov"), cwd.join("clip.mov").to_string_lossy());
assert!(!super::absolute("clip.mov").starts_with(r"\\?\"));
assert_eq!(super::absolute(""), "");
#[cfg(unix)]
{
let dir = std::env::temp_dir().join(format!("filmcraft-cli-abs-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("real.mov"), b"x").unwrap();
let link = dir.join("link.mov");
let _ = std::fs::remove_file(&link);
std::os::unix::fs::symlink(dir.join("real.mov"), &link).unwrap();
assert_eq!(super::absolute(&link.to_string_lossy()), link.to_string_lossy());
std::fs::remove_dir_all(&dir).unwrap();
}
}
#[test]
fn export_format_from_extension() {
assert_eq!(super::format_for("out/clip.MXF"), Some("mxf-op1a"));

55
apps/filmcraft-cli/tests/cli.rs
View File

@ -22,6 +22,9 @@ fn help_and_usage_errors() {
assert_eq!(cli(&[]).status.code(), Some(2));
assert_eq!(cli(&["frobnicate"]).status.code(), Some(2));
assert_eq!(cli(&["exec", "file.newBin", "notkv"]).status.code(), Some(2));
// Invalid --bridge is a usage error (exit 2) for both exec and mcp (#921).
assert_eq!(cli(&["--bridge", "0.0.0.0:9876", "exec", "inspect.project"]).status.code(), Some(2));
assert_eq!(cli(&["mcp", "--bridge", "0.0.0.0:9876"]).status.code(), Some(2));
}
#[test]
@ -291,3 +294,55 @@ fn closed_stdout_still_runs_the_script_and_saves() {
assert_eq!(out.status.code(), Some(1));
let _ = std::fs::remove_dir_all(&dir);
}
/// #1054: exporting or rendering a project whose media is missing fails (exit 1) and names the
/// offline items on stderr and in the result's `offlineMedia`, unless `--allow-offline` accepts the
/// placeholder frames; then it still warns.
#[test]
fn export_with_missing_media_fails_unless_allowed() {
let dir = std::env::temp_dir().join(format!("fc-cli-1054-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let f = |n: &str| dir.join(n).to_str().unwrap().to_string();
let (still, proj) = (f("still.png"), f("p.fcproj"));
let o = cli(&["--demo", "render", "--seconds", "1", "--scale", "0.1", "--out", &still]);
assert!(o.status.success(), "{}", String::from_utf8_lossy(&o.stderr));
json_out(&cli(&["exec", "file.newSequence", "name=s", "width=64", "height=36", "fps=24", "--save-as", &proj]));
let paths = format!("paths=[\"{}\"]", still.replace('\\', "\\\\"));
let item = json_out(&cli(&["--project", &proj, "--save", "exec", "file.import", &paths]))["items"][0].as_u64().unwrap();
json_out(&cli(&["--project", &proj, "--save", "exec", "timeline.place", &format!("item={item}"), "track=V1", "frame=0", "audio=false"]));
let export = |out: &str, extra: &[&str]| {
let mut args = vec!["--project", proj.as_str(), "--compact", "export", out, "--format", "png", "--start", "0", "--end", "0.05"];
args.extend_from_slice(extra);
cli(&args)
};
let on = export(&f("on.png"), &[]);
let v = json_out(&on);
assert_eq!(v["offlineMedia"], serde_json::json!([]), "{v}");
assert!(!String::from_utf8_lossy(&on.stderr).contains("warning"), "{}", String::from_utf8_lossy(&on.stderr));
std::fs::remove_file(&still).unwrap();
let off = export(&f("off.png"), &[]);
let err = String::from_utf8_lossy(&off.stderr);
assert_eq!(off.status.code(), Some(1), "{err}");
assert!(err.contains("still.png") && err.contains("missing") && err.contains("--allow-offline"), "{err}");
let v: Value = serde_json::from_slice(&off.stdout).expect("the result JSON is still printed");
assert_eq!(v["offlineMedia"][0]["item"], item, "{v}");
assert_eq!(v["offlineMedia"][0]["status"], "missing", "{v}");
let allowed = export(&f("allowed.png"), &["--allow-offline"]);
let err = String::from_utf8_lossy(&allowed.stderr);
assert!(allowed.status.success(), "{err}");
assert!(err.contains("still.png") && err.contains("missing"), "still warns: {err}");
assert_eq!(json_out(&allowed)["offlineMedia"][0]["status"], "missing");
let render = cli(&["--project", &proj, "render", "--seconds", "0", "--out", &f("r.png")]);
let err = String::from_utf8_lossy(&render.stderr);
assert_eq!(render.status.code(), Some(1), "{err}");
assert!(err.contains("still.png") && err.contains("--allow-offline"), "{err}");
let render = cli(&["--project", &proj, "render", "--seconds", "0", "--out", &f("r.png"), "--allow-offline"]);
assert!(render.status.success(), "{}", String::from_utf8_lossy(&render.stderr));
assert!(String::from_utf8_lossy(&render.stderr).contains("still.png"));
let _ = std::fs::remove_dir_all(&dir);
}

488
apps/filmcraft-cli/tests/workflows.rs Normal file
View File

@ -0,0 +1,488 @@
//! Workflow acceptance tests (docs/gaps.md G6): real editing jobs done end to end the way an agent
//! does them, through the headless MCP server of the `filmcraft-cli` binary (`filmcraft-cli mcp`,
//! JSON-RPC over stdio) and the CLI itself, on media that looks like what people shoot:
//!
//! - a phone clip: H.264 + AAC in MP4 with a variable frame rate (dropped frames);
//! - a camera clip: long-GOP H.264 (one keyframe) in Matroska with two audio tracks (AAC, AC-3).
//!
//! Each test imports, edits (extract, audio gain, title; ripple trims, colour matte, cross
//! dissolve), exports (H.264 MP4; save the project, reopen it in a new process and export ProRes)
//! and checks the file with the ffprobe / ffmpeg oracles: streams, frame count, duration, which
//! source frame lands where, A/V sync, the title's pixels, the dissolve's mix, audio levels.
//!
//! The media is generated with ffmpeg into `target/fixtures/cli/workflows` at test time (never
//! committed); without ffmpeg the tests skip (`FILMCRAFT_REQUIRE_ORACLES=1` makes that a failure).
//! ffmpeg and ffprobe are external test oracles only.
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use serde_json::{Value, json};
// ---------------------------------------------------------------------------------------------
// Fixtures
/// A phone-like clip: 4 s of 320×240 `testsrc2` at 30 fps with every frame `n % 7 == 3` dropped
/// (variable frame rate, the timestamps of the kept frames unchanged), H.264 + AAC stereo. The
/// whole picture flashes white on the frames at 1.0 s and 3.0 s, and a 1 kHz beep sounds from 1.0
/// to 1.1 s and from 3.0 to 3.1 s (silence otherwise): A/V sync markers.
fn phone_vfr_mp4(ffmpeg: &Path) -> Option<PathBuf> {
let out = filmcraft_testkit::fixtures_dir("cli/workflows").join("phone_vfr_h264_aac.mp4");
filmcraft_testkit::fixtures::generate(&out, |tmp| {
Command::new(ffmpeg)
.args(["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=30:duration=4"])
.args(["-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=48000:duration=4"])
.args([
"-filter_complex",
"[0:v]drawbox=x=0:y=0:w=iw:h=ih:color=white:t=fill:enable='between(t,1,1.02)+between(t,3,3.02)',\
select='not(eq(mod(n,7),3))'[v];\
[1:a]volume='if(between(t,1,1.1)+between(t,3,3.1),1,0)':eval=frame[a]",
])
.args(["-map", "[v]", "-map", "[a]", "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-fps_mode", "passthrough"])
.args(["-video_track_timescale", "30000", "-c:a", "aac", "-b:a", "128k", "-ac", "2"])
.arg(tmp)
.status()
.is_ok_and(|s| s.success())
})
}
/// A camera-like clip: 6 s of 320×240 `testsrc2` at 25 fps as long-GOP H.264 (a single keyframe,
/// B-frames) in Matroska, with two stereo audio tracks: 440 Hz in AAC and 1320 Hz in AC-3.
fn camera_long_gop_mkv(ffmpeg: &Path) -> Option<PathBuf> {
let out = filmcraft_testkit::fixtures_dir("cli/workflows").join("camera_long_gop_2audio.mkv");
filmcraft_testkit::fixtures::generate(&out, |tmp| {
Command::new(ffmpeg)
.args(["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=25:duration=6"])
.args(["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=6"])
.args(["-f", "lavfi", "-i", "sine=frequency=1320:sample_rate=48000:duration=6"])
.args(["-map", "0", "-map", "1", "-map", "2", "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p"])
.args(["-g", "300", "-keyint_min", "300", "-sc_threshold", "0", "-bf", "3"])
.args(["-c:a:0", "aac", "-ac:a:0", "2", "-c:a:1", "ac3", "-ac:a:1", "2"])
.arg(tmp)
.status()
.is_ok_and(|s| s.success())
})
}
/// A scratch directory under the workspace `target/`, removed when dropped.
struct Scratch(PathBuf);
impl Scratch {
fn new(name: &str) -> Self {
let d = filmcraft_testkit::workspace_root().join("target").join("workflow-tests").join(format!("{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
Scratch(d)
}
fn path(&self, name: &str) -> String {
self.0.join(name).to_string_lossy().into_owned()
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
// ---------------------------------------------------------------------------------------------
// A minimal MCP client for `filmcraft-cli mcp` (newline-delimited JSON-RPC over stdio)
struct Mcp {
child: Child,
stdin: ChildStdin,
stdout: BufReader<ChildStdout>,
next_id: u64,
}
impl Mcp {
/// Start the headless server (an empty project) and initialise the session. `data_dir` keeps
/// user presets away from the real data directory.
fn start(data_dir: &str) -> Self {
let mut child = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli"))
.args(["--data-dir", data_dir, "mcp"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.expect("spawn filmcraft-cli mcp");
let stdin = child.stdin.take().unwrap();
let stdout = BufReader::new(child.stdout.take().unwrap());
let mut m = Mcp { child, stdin, stdout, next_id: 1 };
let init =
m.request("initialize", json!({"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "workflow-tests", "version": "1"}}));
assert_eq!(init["result"]["serverInfo"]["name"], "filmcraft", "{init}");
m.send(&json!({"jsonrpc": "2.0", "method": "notifications/initialized"}));
m
}
fn send(&mut self, msg: &Value) {
writeln!(self.stdin, "{msg}").unwrap();
self.stdin.flush().unwrap();
}
/// Send a request and return its response (notifications in between are skipped).
fn request(&mut self, method: &str, params: Value) -> Value {
let id = self.next_id;
self.next_id += 1;
self.send(&json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}));
loop {
let mut line = String::new();
assert!(self.stdout.read_line(&mut line).unwrap() > 0, "the MCP server closed stdout while `{method}` was pending");
let v: Value = serde_json::from_str(&line).unwrap_or_else(|e| panic!("not JSON-RPC: {e}: {line}"));
if v["id"] == id {
return v;
}
}
}
/// Call a tool; its text result parsed as JSON. Panics on a protocol or tool error.
fn tool(&mut self, name: &str, arguments: Value) -> Value {
let r = self.request("tools/call", json!({"name": name, "arguments": arguments}));
let text = r["result"]["content"][0]["text"].as_str().unwrap_or_default().to_string();
assert!(r["error"].is_null() && r["result"]["isError"] == false, "{name} {arguments}: {r}");
serde_json::from_str(&text).unwrap_or(Value::String(text))
}
/// `command_run`: one engine command by id.
fn run(&mut self, id: &str, params: Value) -> Value {
self.tool("command_run", json!({"id": id, "params": params}))
}
}
impl Drop for Mcp {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
// ---------------------------------------------------------------------------------------------
// Oracles
fn probe(ffprobe: &Path, path: &str) -> Value {
let out = Command::new(ffprobe).args(["-v", "error", "-of", "json", "-count_frames", "-show_format", "-show_streams", path]).output().unwrap();
assert!(out.status.success(), "ffprobe {path}: {}", String::from_utf8_lossy(&out.stderr));
serde_json::from_slice(&out.stdout).unwrap()
}
fn streams<'a>(j: &'a Value, kind: &str) -> Vec<&'a Value> {
j["streams"].as_array().unwrap().iter().filter(|s| s["codec_type"] == kind).collect()
}
fn num(v: &Value) -> f64 {
v.as_str().and_then(|s| s.parse().ok()).or_else(|| v.as_f64()).unwrap_or(f64::NAN)
}
/// Run ffmpeg decoding `input` (with `pre` input options) to raw data on stdout.
fn ffmpeg_raw(ffmpeg: &Path, pre: &[&str], input: &str, args: &[&str]) -> Vec<u8> {
let out = Command::new(ffmpeg).args(["-v", "error"]).args(pre).args(["-i", input]).args(args).arg("-").output().unwrap();
assert!(out.status.success(), "ffmpeg {input}: {}", String::from_utf8_lossy(&out.stderr));
out.stdout
}
/// Size of the frames [`frames`] returns.
const FW: usize = 80;
const FH: usize = 60;
/// Every video frame of `input`, scaled to 80×60 in `pix_fmt` (`gray` or `rgb24`). `vf` goes
/// first in the filter chain (e.g. `fps=30` to see a VFR file as an editor shows it).
fn frames(ffmpeg: &Path, input: &str, vf: Option<&str>, pix_fmt: &str) -> Vec<Vec<u8>> {
let chain = match vf {
Some(f) => format!("{f},scale={FW}:{FH}:flags=area,format={pix_fmt}"),
None => format!("scale={FW}:{FH}:flags=area,format={pix_fmt}"),
};
let bpp = if pix_fmt == "gray" { 1 } else { 3 };
let raw = ffmpeg_raw(ffmpeg, &[], input, &["-map", "0:v:0", "-vf", &chain, "-fps_mode", "passthrough", "-f", "rawvideo"]);
raw.chunks_exact(FW * FH * bpp).map(<[u8]>::to_vec).collect()
}
/// Mean absolute difference of two equally sized frames.
fn diff(a: &[u8], b: &[u8]) -> f64 {
a.iter().zip(b).map(|(x, y)| f64::from(x.abs_diff(*y))).sum::<f64>() / a.len().max(1) as f64
}
/// Assert that `got` shows source frame `want`: it is (one of) the closest source frames, and close.
fn assert_shows(got: &[u8], src: &[Vec<u8>], want: usize, what: &str) {
let d: Vec<f64> = src.iter().map(|s| diff(got, s)).collect();
let best = d.iter().copied().fold(f64::INFINITY, f64::min);
let at = d.iter().position(|x| *x == best).unwrap_or(0);
assert!(d[want] < 6.0 && d[want] <= best + 0.25, "{what}: expected source frame {want} (diff {:.2}), closest is {at} (diff {best:.2})", d[want]);
}
/// The first channel of audio stream `stream` of `input`, mono-mixed, as f32 at 48 kHz.
fn pcm(ffmpeg: &Path, input: &str, stream: usize) -> Vec<f32> {
let map = format!("0:a:{stream}");
let raw = ffmpeg_raw(ffmpeg, &[], input, &["-map", &map, "-ac", "1", "-ar", "48000", "-f", "f32le"]);
raw.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
}
/// Times (s) where the signal rises above a quarter of its peak, at least 0.2 s apart.
fn onsets(x: &[f32]) -> Vec<f64> {
let peak = x.iter().fold(0f32, |m, v| m.max(v.abs()));
let mut out = Vec::new();
let mut last = f64::NEG_INFINITY;
for (i, block) in x.chunks(48).enumerate() {
let t = i as f64 * 48.0 / 48_000.0;
if block.iter().any(|v| v.abs() > peak / 4.0) && t - last > 0.2 {
out.push(t);
}
if block.iter().any(|v| v.abs() > peak / 4.0) {
last = t;
}
}
out
}
fn rms(x: &[f32], from: f64, to: f64) -> f64 {
let s = x.get((from * 48_000.0) as usize..((to * 48_000.0) as usize).min(x.len())).unwrap_or_default();
(s.iter().map(|v| f64::from(*v) * f64::from(*v)).sum::<f64>() / s.len().max(1) as f64).sqrt()
}
/// Amplitude of the `freq` component of 100 ms of `x` from `at` seconds (Goertzel).
fn tone(x: &[f32], at: f64, freq: f64) -> f64 {
let start = (at * 48_000.0) as usize;
let s = x.get(start..(start + 4800).min(x.len())).unwrap_or_default();
let c = 2.0 * (2.0 * std::f64::consts::PI * freq / 48_000.0).cos();
let (mut s1, mut s2) = (0.0, 0.0);
for v in s {
let s0 = f64::from(*v) + c * s1 - s2;
s2 = s1;
s1 = s0;
}
(s1 * s1 + s2 * s2 - c * s1 * s2).max(0.0).sqrt() * 2.0 / s.len().max(1) as f64
}
fn db(ratio: f64) -> f64 {
20.0 * ratio.log10()
}
fn clips_on<'a>(seq: &'a Value, kind: &str, track: usize) -> &'a Vec<Value> {
seq[kind][track]["items"].as_array().unwrap()
}
// ---------------------------------------------------------------------------------------------
// Workflows
/// Phone footage: import a VFR H.264/AAC clip, make a sequence from it, extract a second from the
/// middle (Mark In / Mark Out / Extract), lower the audio 6 dB, put a title over the start, and
/// export H.264 MP4 through MCP. The VFR clip plays at its 30 fps base rate (a dropped frame
/// holds the one before), the cut lands on the right frames, picture and sound stay in sync
/// across the cut, the title is in the picture only while it is on the timeline.
#[test]
fn phone_vfr_clip_extract_gain_title_to_h264_mp4() {
let (ffmpeg, ffprobe) = filmcraft_testkit::require_oracles!();
let Some(src) = phone_vfr_mp4(&ffmpeg) else {
return filmcraft_testkit::skip("phone workflow", "ffmpeg could not make the VFR H.264/AAC fixture (libx264?)");
};
let src = src.to_string_lossy().into_owned();
let dir = Scratch::new("phone");
let mut m = Mcp::start(&dir.path("data"));
let imported = m.tool("media_import", json!({"paths": [src]}));
assert_eq!(imported["errors"], json!([]), "{imported}");
let item = imported["items"][0].as_u64().unwrap();
let made = m.run("file.newSequenceFromClip", json!({"items": [item]}));
assert!(made["sequence"].is_u64(), "{made}");
let seq = m.tool("sequence_inspect", json!({}));
assert_eq!((seq["settings"]["width"].as_u64(), seq["settings"]["height"].as_u64()), (Some(320), Some(240)), "{}", seq["settings"]);
assert_eq!(seq["settings"]["frame_rate"], json!({"num": 30, "den": 1}), "a VFR phone clip makes a sequence at its base rate");
assert_eq!(seq["durationFrames"], 120, "4 s at 30 fps");
// Extract 1.5 s .. 2.5 s; Mark Out includes its frame, so 31 frames go and 89 remain.
m.run("markers.markIn", json!({"seconds": 1.5}));
m.run("markers.markOut", json!({"seconds": 2.5}));
m.run("sequence.extract", json!({}));
m.run("edit.selectAll", json!({}));
let gain = m.run("clip.audioGain", json!({"mode": "set", "db": -6.0}));
assert_eq!(gain["clips"], 2, "both audio pieces: {gain}");
m.run("edit.deselectAll", json!({}));
m.run("graphics.newText", json!({"text": "HELLO", "size": 40, "time": 0, "seconds": 0.5}));
let seq = m.tool("sequence_inspect", json!({}));
assert_eq!(seq["durationFrames"], 89, "{seq}");
assert_eq!(clips_on(&seq, "video", 0).len(), 2);
assert_eq!(clips_on(&seq, "audio", 0).len(), 2);
let out = dir.path("phone_edit.mp4");
let r = m.run("file.exportMedia", json!({"path": out, "format": "h264", "range": "entire", "wait": true}));
assert_eq!(r["result"]["frames"], 89, "{r}");
// container and streams
let j = probe(&ffprobe, &out);
let (v, a) = (streams(&j, "video"), streams(&j, "audio"));
assert_eq!((v.len(), a.len()), (1, 1), "{j}");
assert_eq!((v[0]["codec_name"].as_str(), v[0]["width"].as_u64(), v[0]["height"].as_u64()), (Some("h264"), Some(320), Some(240)));
assert_eq!(v[0]["r_frame_rate"], "30/1");
assert_eq!(v[0]["nb_read_frames"], "89");
assert_eq!((a[0]["codec_name"].as_str(), a[0]["sample_rate"].as_str(), a[0]["channels"].as_u64()), (Some("aac"), Some("48000"), Some(2)));
let duration = num(&j["format"]["duration"]);
assert!((duration - 89.0 / 30.0).abs() < 0.06, "duration {duration}");
assert!((num(&a[0]["duration"]) - 89.0 / 30.0).abs() < 0.06, "audio as long as the picture: {}", a[0]["duration"]);
// picture: frame k of the export is frame k (before the cut) or k + 31 (after) of the source
// as an editor plays it (CFR 30, a dropped frame holding the previous one)
let src_frames = frames(&ffmpeg, &src, Some("fps=30"), "gray");
let got = frames(&ffmpeg, &out, None, "gray");
assert!(src_frames.len() >= 120, "{} source frames", src_frames.len());
assert_eq!(got.len(), 89);
for k in [20, 32, 40, 44, 45, 50, 70, 88] {
let want = if k < 45 { k } else { k + 31 };
assert_shows(&got[k], &src_frames, want, &format!("export frame {k}"));
}
// the flash frames: 30 (with 31 holding it, as the source dropped its frame 31) and 90 - 31
let mean = |f: &Vec<u8>| f.iter().map(|v| f64::from(*v)).sum::<f64>() / f.len() as f64;
let flashes: Vec<usize> = got.iter().enumerate().filter(|(_, f)| mean(f) > 200.0).map(|(i, _)| i).collect();
assert_eq!(flashes, [30, 31, 59], "white frames");
// the title: over frames 0..15 only (its pixels differ from the source there, nowhere after)
let blocks_changed = |k: usize| {
let mut n = 0;
for by in (0..FH).step_by(10) {
for bx in (0..FW).step_by(10) {
let mut d = 0.0;
for y in by..by + 10 {
for x in bx..bx + 10 {
d += f64::from(got[k][y * FW + x].abs_diff(src_frames[k][y * FW + x]));
}
}
n += usize::from(d / 100.0 > 15.0);
}
}
n
};
for k in [2, 8, 14] {
assert!(blocks_changed(k) >= 2, "title missing on frame {k}");
}
for k in [16, 25] {
assert_eq!(blocks_changed(k), 0, "title still on frame {k}");
}
// sound: beeps where the flashes are (A/V sync across the extract), 6 dB down
let src_pcm = pcm(&ffmpeg, &src, 0);
let got_pcm = pcm(&ffmpeg, &out, 0);
let beeps = onsets(&got_pcm);
assert_eq!(beeps.len(), 2, "beeps at {beeps:?}");
for (beep, flash) in beeps.iter().zip([30.0 / 30.0, 59.0 / 30.0]) {
assert!((beep - flash).abs() <= 1.0 / 30.0, "beep at {beep:.3} s, flash at {flash:.3} s");
}
let change = db(rms(&got_pcm, 1.01, 1.09) / rms(&src_pcm, 1.01, 1.09));
assert!((change + 6.0).abs() < 1.0, "audio gain -6 dB measured {change:.2} dB");
let change = db(rms(&got_pcm, 1.98, 2.06) / rms(&src_pcm, 3.01, 3.09));
assert!((change + 6.0).abs() < 1.0, "second beep {change:.2} dB");
}
/// Camera footage: import a long-GOP H.264 Matroska file with two audio tracks through MCP, make
/// a sequence (both tracks land on A1 and A2), ripple-trim 2.6 s off the head (decoding starts at
/// the only keyframe) and 15 frames off the tail, add a red colour matte with a centred cross
/// dissolve, save the project; then a new `filmcraft-cli` process opens it and exports ProRes.
/// The first exported frame is source frame 65, the dissolve mixes the two pictures, both audio
/// tracks are in the mix at their own level and stop where the clip does.
#[test]
fn camera_long_gop_two_audio_tracks_trim_dissolve_to_prores() {
let (ffmpeg, ffprobe) = filmcraft_testkit::require_oracles!();
let Some(src) = camera_long_gop_mkv(&ffmpeg) else {
return filmcraft_testkit::skip("camera workflow", "ffmpeg could not make the long-GOP Matroska fixture (libx264?)");
};
let src = src.to_string_lossy().into_owned();
let dir = Scratch::new("camera");
let data = dir.path("data");
let project = dir.path("camera.fcproj");
{
let mut m = Mcp::start(&data);
let imported = m.tool("media_import", json!({"paths": [src]}));
assert_eq!(imported["errors"], json!([]), "{imported}");
let item = imported["items"][0].as_u64().unwrap();
let made = m.run("file.newSequenceFromClip", json!({"items": [item]}));
let clips: Vec<u64> = made["clips"].as_array().unwrap().iter().filter_map(Value::as_u64).collect();
assert_eq!(clips.len(), 3, "a video clip and one clip per audio track: {made}");
let seq = m.tool("sequence_inspect", json!({}));
assert_eq!(seq["settings"]["frame_rate"], json!({"num": 25, "den": 1}));
assert_eq!(seq["audio"].as_array().map(Vec::len), Some(2), "two audio tracks");
let video = clips_on(&seq, "video", 0)[0]["clip"].as_u64().unwrap();
// 65 frames off the head and 15 off the tail, the linked audio following
m.run("timeline.trim", json!({"clip": video, "edge": "in", "mode": "ripple", "deltaFrames": 65}));
m.run("timeline.trim", json!({"clip": video, "edge": "out", "mode": "ripple", "deltaFrames": -15}));
let seq = m.tool("sequence_inspect", json!({}));
let end = clips_on(&seq, "video", 0)[0]["endFrame"].as_u64().unwrap();
for t in 0..2 {
let a = &clips_on(&seq, "audio", t)[0];
assert_eq!((a["startFrame"].as_u64(), a["endFrame"].as_u64(), a["sourceIn"].as_i64()), (Some(0), Some(end), Some(660_441_600_000)), "A{}", t + 1);
}
let matte = m.run("file.newColorMatte", json!({"color": "#ff0000", "seconds": 2.0}))["item"].as_u64().unwrap();
let placed = m.run("timeline.place", json!({"item": matte, "track": "V1", "frame": end, "insert": false}));
let matte_clip = placed["clips"][0].as_u64().unwrap();
m.run("sequence.applyVideoTransition", json!({"clip": matte_clip, "edge": "in", "frames": 10, "effect": "cross_dissolve"}));
let seq = m.tool("sequence_inspect", json!({}));
let tr = &seq["video"][0]["transitions"][0];
assert_eq!((tr["effect"].as_str(), tr["align"].as_str()), (Some("cross_dissolve"), Some("center")), "{tr}");
m.run("file.saveAs", json!({"path": project}));
}
assert_eq!(end_frame_of(&project), Some(71), "the saved project holds the edit");
// a fresh process opens the saved project and renders it
let out = dir.path("camera_edit.mov");
let o = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(["--data-dir", &data, "--project", &project, "export", &out]).output().unwrap();
assert!(o.status.success(), "export: {}", String::from_utf8_lossy(&o.stderr));
let r: Value = serde_json::from_slice(&o.stdout).unwrap();
let total = 71 + 50;
assert_eq!(r["result"]["frames"], total, "{r}");
let j = probe(&ffprobe, &out);
let (v, a) = (streams(&j, "video"), streams(&j, "audio"));
assert_eq!((v.len(), a.len()), (1, 1), "{j}");
assert_eq!((v[0]["codec_name"].as_str(), v[0]["width"].as_u64(), v[0]["height"].as_u64()), (Some("prores"), Some(320), Some(240)));
assert_eq!(v[0]["r_frame_rate"], "25/1");
assert_eq!(v[0]["nb_read_frames"], total.to_string());
assert_eq!((a[0]["sample_rate"].as_str(), a[0]["channels"].as_u64()), (Some("48000"), Some(2)));
let duration = num(&j["format"]["duration"]);
assert!((duration - f64::from(total) / 25.0).abs() < 0.05, "duration {duration}");
// picture: frame k is source frame 65 + k up to the dissolve (frames 66..76), red after it
let src_frames = frames(&ffmpeg, &src, None, "rgb24");
let got = frames(&ffmpeg, &out, None, "rgb24");
assert_eq!((src_frames.len(), got.len()), (150, total as usize));
for k in [0, 1, 2, 30, 64] {
assert_shows(&got[k], &src_frames, 65 + k, &format!("export frame {k}"));
}
let red = &got[100];
assert!(red.as_chunks::<3>().0.iter().all(|[r, g, b]| *r > 230 && *g < 25 && *b < 25), "the matte is red");
// the dissolve: frame = (1 - a)·camera + a·red, `a` rising through about 0.5 at the cut
let mix = |k: usize| {
let (cam, e) = (&src_frames[65 + k], &got[k]);
let (mut num, mut den) = (0.0, 0.0);
for ((c, r), e) in cam.iter().zip(red).zip(e) {
let (c, r, e) = (f64::from(*c), f64::from(*r), f64::from(*e));
num += (e - c) * (r - c);
den += (r - c) * (r - c);
}
num / den
};
let alphas: Vec<f64> = (64..=78).map(mix).collect();
assert!(alphas[0].abs() < 0.05 && (alphas[14] - 1.0).abs() < 0.05, "outside the dissolve: {alphas:?}");
assert!(alphas.windows(2).all(|w| w[1] >= w[0] - 0.02), "the dissolve progresses: {alphas:?}");
assert!((alphas[7] - 0.5).abs() < 0.12, "half way at the cut (frame 71): {alphas:?}");
// sound: both tracks in the mix at their own level while the clip plays, silence on the matte
let got_pcm = pcm(&ffmpeg, &out, 0);
let (t1, t2) = (pcm(&ffmpeg, &src, 0), pcm(&ffmpeg, &src, 1));
for at in [0.2, 1.5, 2.6] {
let (low, high) = (tone(&got_pcm, at, 440.0), tone(&got_pcm, at, 1320.0));
let (want_low, want_high) = (tone(&t1, at + 2.6, 440.0), tone(&t2, at + 2.6, 1320.0));
assert!(
db(low / want_low).abs() < 1.0 && db(high / want_high).abs() < 1.0,
"at {at} s: 440 Hz {low:.4} (want {want_low:.4}), 1320 Hz {high:.4} (want {want_high:.4})"
);
}
assert!(rms(&got_pcm, 3.0, 4.8) < 1e-3, "silence under the matte");
}
/// `endFrame` of the first clip on V1 of the saved project's active sequence, read back through
/// `filmcraft-cli --project … inspect sequence`.
fn end_frame_of(project: &str) -> Option<u64> {
let o = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(["--project", project, "inspect", "sequence"]).output().ok()?;
let v: Value = serde_json::from_slice(&o.stdout).ok()?;
v["video"][0]["items"][0]["endFrame"].as_u64()
}

5
apps/filmcraft-web/src/import.rs
View File

@ -263,7 +263,10 @@ pub fn install_hooks(app: &mut FilmcraftApp) {
}));
// Saving writes the file in memory and offers it as a download (`fs::WebServices`).
app.hooks.pick_save = Some(Box::new(|name: &str| Some(format!("/projects/{name}"))));
app.hooks.pick_save_as = Some(Box::new(|_filter: &str, _exts: &[&str], name: &str| Some(format!("/exports/{name}"))));
app.hooks.pick_save_as = Some(Box::new(|_filter: &str, _exts: &[&str], name: &str| {
let file = std::path::Path::new(name).file_name().map_or(name.into(), |n| n.to_string_lossy());
Some(format!("/exports/{file}"))
}));
// Link Media ▸ Locate…, Attach Proxies, Reconnect Full Resolution: the picker is async, so the
// hint's command runs once the user has chosen (#111).
app.hooks.pick_file_for_relink = Some(Box::new(|exts: &[&str], hint: Option<RelinkHint>| {

2
apps/filmcraft/Cargo.toml
View File

@ -41,6 +41,8 @@ open = "5"
[dev-dependencies]
glob = "0.3"
# graphics.rs: enumerate the real adapters in a test.
pollster = "0.4"
[target.'cfg(target_os = "macos")'.dependencies]
muda = "0.21"

154
apps/filmcraft/src/args.rs
View File

@ -1,5 +1,8 @@
//! The desktop app's command line (`filmcraft --help`).
use std::ffi::OsString;
use std::path::{Path, PathBuf};
/// `filmcraft --help`.
pub const USAGE: &str = "\
Usage: filmcraft [options] [project.fcproj | media files...]
@ -15,6 +18,9 @@ Options:
--no-recover start without asking about unsaved changes
--data-dir <dir> folder for auto-save, crash recovery and logs
(also: FILMCRAFT_DATA_DIR)
--graphics <mode> start with this graphics mode and keep it: default,
dx12 (Windows), software (Windows, Linux);
`--graphics default` undoes an automatic fallback
--version print the version and exit
-h, --help print this help and exit
";
@ -28,7 +34,7 @@ pub enum Cli {
}
/// Options of a normal start.
#[derive(Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq)]
pub struct Launch {
pub control_port: Option<u16>,
pub control_input_only: bool,
@ -39,6 +45,8 @@ pub struct Launch {
pub startup_flag: bool,
pub recover: Option<bool>,
pub data_dir: Option<std::path::PathBuf>,
/// `--graphics <mode>`: start with this graphics mode and keep it (see `graphics.rs`).
pub graphics: Option<crate::graphics::Mode>,
}
/// Parse the arguments after the program name. `env_control_port` is the value of
@ -48,16 +56,31 @@ pub struct Launch {
/// mistyped flag helps nobody, and starting without the control server that was asked for leaves
/// a script waiting for it. That holds for the whole line, so `--version --bogus` is an error
/// too, and for a port from the environment exactly as for one from `--control`.
pub fn parse(env_control_port: Option<String>, args: impl IntoIterator<Item = String>) -> Result<Cli, String> {
let mut l = Launch { control_port: None, control_input_only: false, files: Vec::new(), demo: true, startup_flag: false, recover: None, data_dir: None };
///
/// The arguments come from `std::env::args_os`: not every argument is Unicode (a Linux file name
/// in Latin-1), and `std::env::args` panics on one. A file name that is not is passed on lossily
/// (its import then reports it missing); `--data-dir` keeps the folder exactly.
pub fn parse(env_control_port: Option<String>, args: impl IntoIterator<Item = OsString>) -> Result<Cli, String> {
let mut l = Launch {
control_port: None,
control_input_only: false,
files: Vec::new(),
demo: true,
startup_flag: false,
recover: None,
data_dir: None,
graphics: None,
};
let (mut help, mut version) = (false, false);
let mut args = args.into_iter();
while let Some(a) = args.next() {
match a.as_str() {
while let Some(raw) = args.next() {
let a = raw.to_string_lossy();
match a.as_ref() {
"-h" | "--help" => help = true,
"--version" => version = true,
"--control" => {
let port = args.next().ok_or("--control needs a port number")?;
let port = port.to_string_lossy();
l.control_port = Some(port.parse().map_err(|_| format!("--control needs a port number, got `{port}`"))?);
}
"--demo" => (l.demo, l.startup_flag) = (true, true),
@ -65,13 +88,17 @@ pub fn parse(env_control_port: Option<String>, args: impl IntoIterator<Item = St
"--empty" => (l.demo, l.startup_flag) = (false, true),
"--recover" => l.recover = Some(true),
"--no-recover" => l.recover = Some(false),
"--data-dir" => l.data_dir = Some(args.next().map(std::path::PathBuf::from).ok_or("--data-dir needs a folder")?),
"--data-dir" => l.data_dir = Some(args.next().map(PathBuf::from).ok_or("--data-dir needs a folder")?),
"--graphics" => {
let name = args.next().ok_or("--graphics needs a mode: default, dx12 or software")?;
l.graphics = Some(graphics_mode(&name.to_string_lossy())?);
}
// everything after `--` is a file, whatever it looks like
"--" => l.files.extend(args.by_ref()),
"--" => l.files.extend(args.by_ref().map(|f| f.to_string_lossy().into_owned())),
// macOS adds a process serial number when it launches an app bundle
_ if a.starts_with("-psn_") => {}
_ if a.len() > 1 && a.starts_with('-') => return Err(format!("unknown option `{a}`")),
_ => l.files.push(a),
_ => l.files.push(a.into_owned()),
}
}
if help {
@ -91,12 +118,45 @@ pub fn parse(env_control_port: Option<String>, args: impl IntoIterator<Item = St
Ok(Cli::Run(l))
}
/// Whether a launch file is a FilmCraft project. The extension's case does not matter: Windows file
/// associations and the Linux MIME glob start the app for `CUT.FCPROJ` too (#850).
pub fn is_project(path: &str) -> bool {
Path::new(path).extension().is_some_and(|e| e.eq_ignore_ascii_case("fcproj"))
}
/// `path` made absolute against the working directory, so a project or media file named relative
/// to a terminal's folder is still found once the app runs from another one (Recent Projects, the
/// media paths saved in the project). Unlike `canonicalize`, `std::path::absolute` keeps symlinks
/// and adds no Windows verbatim `\\?\` prefix. A path it cannot resolve is kept as given.
pub fn absolute(path: &str) -> String {
std::path::absolute(path).ok().and_then(|p| p.into_os_string().into_string().ok()).unwrap_or_else(|| path.to_string())
}
/// The launch files as the project to open (the first one, see [`is_project`]) and the media files
/// to import (every file that is not a project), all made [`absolute`].
pub fn project_and_media(files: &[String]) -> (Option<String>, Vec<String>) {
let project = files.iter().find(|f| is_project(f)).map(|f| absolute(f));
let media = files.iter().filter(|f| !is_project(f)).map(|f| absolute(f)).collect();
(project, media)
}
/// A `--graphics` value: one of this platform's modes.
fn graphics_mode(name: &str) -> Result<crate::graphics::Mode, String> {
use crate::graphics::{LADDER, Mode};
let here = || LADDER.iter().map(|m| m.name()).collect::<Vec<_>>().join(", ");
let mode = Mode::parse(name).ok_or_else(|| format!("--graphics needs one of {}, got `{name}`", here()))?;
if !LADDER.contains(&mode) {
return Err(format!("--graphics {name} is not available on this platform (use one of {})", here()));
}
Ok(mode)
}
#[cfg(test)]
mod tests {
use super::*;
fn parse_env(env: Option<&str>, args: &[&str]) -> Result<Cli, String> {
parse(env.map(str::to_string), args.iter().map(|a| a.to_string()))
parse(env.map(str::to_string), args.iter().map(OsString::from))
}
fn parse_args(args: &[&str]) -> Result<Cli, String> {
@ -166,7 +226,16 @@ mod tests {
let none = launch(&[]);
assert_eq!(
none,
Launch { control_port: None, control_input_only: false, files: vec![], demo: true, startup_flag: false, recover: None, data_dir: None }
Launch {
control_port: None,
control_input_only: false,
files: vec![],
demo: true,
startup_flag: false,
recover: None,
data_dir: None,
graphics: None
}
);
let l = launch(&["--control", "9876", "--empty", "--no-recover", "--data-dir", "/tmp/fc", "cut.fcproj", "a.mp4"]);
assert_eq!(l.control_port, Some(9876));
@ -179,6 +248,52 @@ mod tests {
assert_eq!(launch(&["-", "-psn_0_12345"]).files, ["-"]);
}
/// `std::env::args` panics on an argument that is not Unicode, so such a file name (Latin-1 on
/// Linux) stopped the app before its window opened.
#[cfg(unix)]
#[test]
fn arguments_that_are_not_unicode_still_start_the_app() {
use std::os::unix::ffi::OsStringExt;
let dir = OsString::from_vec(b"/tmp/d\xe9".to_vec());
let file = OsString::from_vec(b"caf\xe9.mov".to_vec());
let Ok(Cli::Run(l)) = parse(None, [OsString::from("--data-dir"), dir.clone(), file.clone(), OsString::from("--"), file]) else {
panic!("should start the app");
};
assert_eq!(l.data_dir, Some(PathBuf::from(dir)), "the data folder is kept exactly");
assert_eq!(l.files, ["caf\u{fffd}.mov", "caf\u{fffd}.mov"]);
}
/// Windows file associations and the Linux MIME glob match `CUT.FCPROJ` too; it used to be
/// imported as a media file instead of opened (#850).
#[test]
fn a_project_is_recognised_whatever_the_case_of_its_extension() {
for p in ["cut.fcproj", "CUT.FCPROJ", "Cut.FcProj", "/edits/a b.fcproj", r"C:\Edits\CUT.FCPROJ"] {
assert!(is_project(p), "{p}");
}
for p in ["cut.mp4", "fcproj", ".fcproj", "a.fcproj.mov", "a.fcprojx", ""] {
assert!(!is_project(p), "{p}");
}
}
/// Relative launch paths were opened and imported as given, so the project's media paths and
/// Recent Projects pointed nowhere once the app ran from another folder (#850).
#[test]
fn launch_files_are_made_absolute_and_split_case_insensitively() {
let cwd = std::env::current_dir().unwrap();
let nested = Path::new("clips").join("a.mp4");
let already = cwd.join("b.wav").to_string_lossy().into_owned();
let files = ["Cut.FCPROJ".to_string(), nested.to_string_lossy().into_owned(), "other.fcproj".to_string(), already.clone()];
let (project, media) = project_and_media(&files);
assert_eq!(project, Some(cwd.join("Cut.FCPROJ").to_string_lossy().into_owned()));
assert_eq!(media, [cwd.join(&nested).to_string_lossy().into_owned(), already]);
for p in media.iter().chain(&project) {
assert!(Path::new(p).is_absolute() && !p.starts_with(r"\\?\"), "{p}");
}
assert_eq!(project_and_media(&[]), (None, vec![]));
// nothing to resolve against: kept as given rather than dropped
assert_eq!(absolute(""), "");
}
#[test]
fn isolated_input_is_explicit_and_requires_a_control_server() {
assert!(!launch(&["--control", "9876"]).control_input_only);
@ -188,4 +303,23 @@ mod tests {
assert_eq!(parse_args(&["--control-input-only", "--help"]), Ok(Cli::Help));
assert!(USAGE.contains("--control-input-only"));
}
#[test]
fn graphics_picks_one_of_this_platforms_modes() {
use crate::graphics::{LADDER, Mode};
assert_eq!(launch(&[]).graphics, None);
assert_eq!(launch(&["--graphics", "default"]).graphics, Some(Mode::Default));
for mode in LADDER {
assert_eq!(launch(&["--graphics", mode.name()]).graphics, Some(*mode));
}
// a mode this platform has not is refused, not ignored
for mode in [Mode::Dx12, Mode::Software].into_iter().filter(|m| !LADDER.contains(m)) {
assert!(parse_args(&["--graphics", mode.name()]).unwrap_err().contains("not available"));
}
assert!(parse_args(&["--graphics"]).is_err(), "missing mode");
assert!(parse_args(&["--graphics", "vulkan"]).unwrap_err().contains("vulkan"));
assert!(parse_args(&["--graphics", "DEFAULT"]).is_err());
assert!(parse_args(&["--graphics", &"x".repeat(1 << 16)]).is_err());
assert!(USAGE.contains("--graphics"));
}
}

347
apps/filmcraft/src/audio.rs
View File

@ -1,18 +1,33 @@
//! cpal audio output: the playback master clock. Settings ▸ Audio Hardware picks the host
//! ("Device Class"), the output device, the I/O buffer size and the sample rate.
//!
//! A stream the device drops while playing (an ALSA xrun the backend does not recover from, which
//! cpal reports as `poll() returned POLLERR`; an unplugged device) is restarted by
//! [`AudioOut::recover`], keeping the played-frame clock and the play-ahead mixer, so sound
//! resumes where it stopped instead of playback going silent (#840).
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, Mutex, PoisonError, TryLockError};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use filmcraft_engine::settings::AudioHardwarePrefs;
use filmcraft_ui_egui::{AudioDevices, AudioOut};
/// The playback mixer callback handed to [`AudioOut::start`].
type Fill = Box<dyn FnMut(&mut [f32], usize) + Send>;
pub struct CpalOut {
stream: Option<cpal::Stream>,
/// The running playback's mixer, shared with the stream so a restarted stream continues it.
fill: Option<Arc<Mutex<Fill>>>,
played: Arc<AtomicU64>,
failed: Arc<AtomicBool>,
health: Arc<Health>,
watchdog: Watchdog,
/// Time base for the watchdog.
epoch: Instant,
rate: u32,
channels: u16,
hw: AudioHardwarePrefs,
@ -128,8 +143,11 @@ impl CpalOut {
};
Self {
stream: None,
fill: None,
played: Arc::new(AtomicU64::new(0)),
failed: Arc::new(AtomicBool::new(false)),
health: Arc::new(Health::default()),
watchdog: Watchdog::default(),
epoch: Instant::now(),
rate: cfg.as_ref().map_or(48_000, |c| c.sample_rate().0),
channels: cfg.as_ref().map_or(2, |c| c.channels()),
hw: AudioHardwarePrefs::default(),
@ -153,19 +171,14 @@ impl CpalOut {
Ok(exact.unwrap_or(default))
}
fn converted_stream<T>(
&self,
dev: &cpal::Device,
config: &cpal::StreamConfig,
mut fill: Box<dyn FnMut(&mut [f32], usize) + Send>,
) -> Result<cpal::Stream, cpal::BuildStreamError>
fn converted_stream<T>(&self, dev: &cpal::Device, config: &cpal::StreamConfig, fill: Arc<Mutex<Fill>>) -> Result<cpal::Stream, cpal::BuildStreamError>
where
T: cpal::SizedSample + cpal::FromSample<f32>,
{
let channels = usize::from(config.channels);
let played = self.played.clone();
let failed = self.failed.clone();
let error_flag = failed.clone();
let health = self.health.clone();
let error_health = health.clone();
let mut scratch = Vec::new();
dev.build_output_stream(
config,
@ -173,39 +186,23 @@ impl CpalOut {
// Reuse the conversion buffer. A pathological driver buffer must not panic the audio thread.
if buf.len() > 1_048_576 || scratch.try_reserve(buf.len().saturating_sub(scratch.len())).is_err() {
buf.fill(T::EQUILIBRIUM);
if !failed.swap(true, Ordering::SeqCst) {
if !health.failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: unable to allocate the audio conversion buffer");
}
return;
}
scratch.resize(buf.len(), 0.0);
fill(&mut scratch, channels);
render(&fill, &mut scratch, channels, &played, &health);
convert_samples(&scratch, buf);
played.fetch_add((buf.len() / channels) as u64, Ordering::SeqCst);
},
move |e| stream_error(&error_flag, e),
move |e| stream_error(&error_health, e),
None,
)
}
}
fn stream_error(failed: &AtomicBool, e: cpal::StreamError) {
if !failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: audio stream error: {e}");
}
}
fn convert_samples<T: cpal::Sample + cpal::FromSample<f32>>(input: &[f32], output: &mut [T]) {
// Integer conversion represents [-1, 1). In particular I24 cannot represent +1 exactly.
let upper = f32::from_bits(1.0_f32.to_bits() - 1);
for (sample, converted) in input.iter().zip(output) {
*converted = T::from_sample(if sample.is_finite() { sample.clamp(-1.0, upper) } else { 0.0 });
}
}
impl AudioOut for CpalOut {
fn start(&mut self, mut fill: Box<dyn FnMut(&mut [f32], usize) + Send>) -> Result<u32, String> {
self.stop();
/// Open the configured output device and play `fill` on it. `expect` is the (sample rate,
/// channels) a restarted stream must keep: the mixer was set up for them.
fn open(&mut self, fill: Arc<Mutex<Fill>>, expect: Option<(u32, u16)>) -> Result<(), String> {
let host = host(&self.hw.device_class);
let DeviceChoice { device: dev, fallback } = output_device(&host, &self.hw.default_output, &self.fallback).ok_or("no output device")?;
self.note = fallback;
@ -214,6 +211,11 @@ impl AudioOut for CpalOut {
if channels == 0 || cfg.sample_rate().0 == 0 {
return Err("the output device returned an invalid channel count or sample rate".into());
}
if let Some((rate, ch)) = expect
&& (rate, ch) != (cfg.sample_rate().0, cfg.channels())
{
return Err(format!("the output device now runs at {} Hz with {} channels", cfg.sample_rate().0, cfg.channels()));
}
let mut config: cpal::StreamConfig = cfg.clone().into();
if let cpal::SupportedBufferSize::Range { min, max } = cfg.buffer_size()
&& (*min..=*max).contains(&self.hw.buffer_size)
@ -222,20 +224,18 @@ impl AudioOut for CpalOut {
}
self.rate = cfg.sample_rate().0;
self.channels = cfg.channels();
self.played.store(0, Ordering::SeqCst);
self.failed.store(false, Ordering::SeqCst);
let played = self.played.clone();
let failed = self.failed.clone();
let stream = match cfg.sample_format() {
cpal::SampleFormat::F32 => dev.build_output_stream(
&config,
move |buf: &mut [f32], _| {
fill(buf, channels);
played.fetch_add((buf.len() / channels) as u64, Ordering::SeqCst);
},
move |e| stream_error(&failed, e),
None,
),
cpal::SampleFormat::F32 => {
let played = self.played.clone();
let health = self.health.clone();
let error_health = health.clone();
dev.build_output_stream(
&config,
move |buf: &mut [f32], _| render(&fill, buf, channels, &played, &health),
move |e| stream_error(&error_health, e),
None,
)
}
cpal::SampleFormat::I8 => self.converted_stream::<i8>(&dev, &config, fill),
cpal::SampleFormat::I16 => self.converted_stream::<i16>(&dev, &config, fill),
cpal::SampleFormat::I24 => self.converted_stream::<cpal::I24>(&dev, &config, fill),
@ -251,10 +251,172 @@ impl AudioOut for CpalOut {
.map_err(|e| e.to_string())?;
stream.play().map_err(|e| e.to_string())?;
self.stream = Some(stream);
Ok(())
}
/// Replace a dropped stream with a new one on the same mixer, clock and format.
fn restart(&mut self) -> Result<(), String> {
let fill = self.fill.clone().ok_or("nothing is playing")?;
let expect = (self.rate, self.channels);
// The old stream (and its audio thread) goes first: only one stream ever runs the mixer.
self.stream = None;
self.health.clear_errors();
self.open(fill, Some(expect))
}
}
/// What the audio thread reports to the UI thread.
#[derive(Default)]
struct Health {
/// The device went away (unplugged): restart the stream at once.
lost: AtomicBool,
/// A backend error since the last check (cpal keeps the stream alive after these): restart
/// only if the stream also stopped consuming samples.
errored: AtomicBool,
/// Unrecoverable: the mixer panicked, a buffer could not be allocated, or restarts ran out.
/// [`AudioOut::played_frames`] reports `None` and playback goes on without sound.
failed: AtomicBool,
/// The first error of a stream has been printed.
reported: AtomicBool,
}
impl Health {
fn clear_errors(&self) {
self.lost.store(false, Ordering::SeqCst);
self.errored.store(false, Ordering::SeqCst);
self.reported.store(false, Ordering::SeqCst);
}
}
/// How long a stream that reported an error may go without consuming samples before it is
/// restarted. Well under the UI's `AUDIO_STALL_S`, which gives up on the device clock.
const STALL_RESTART: Duration = Duration::from_millis(300);
/// At most this many restarts within [`RESTART_WINDOW`]; a device that keeps failing gives up.
const MAX_RESTARTS: usize = 3;
const RESTART_WINDOW: Duration = Duration::from_secs(10);
/// What [`Watchdog::check`] decides about the running stream.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Verdict {
Healthy,
Restart,
GiveUp,
}
/// Decides when a stream that reported errors is restarted (UI thread, every frame).
#[derive(Default)]
struct Watchdog {
/// Played-frame count last seen, and when it last changed.
seen: (u64, Duration),
/// An error was reported and the stream has not consumed samples since.
pending_error: bool,
/// When recent restarts happened (at most [`MAX_RESTARTS`] kept).
restarts: Vec<Duration>,
}
impl Watchdog {
fn reset(&mut self, now: Duration) {
*self = Self { seen: (0, now), ..Self::default() };
}
fn check(&mut self, now: Duration, played: u64, errored: bool, lost: bool) -> Verdict {
if played != self.seen.0 {
self.seen = (played, now);
self.pending_error = false;
}
// An error that arrives with the stream still consuming samples is only a hiccup.
self.pending_error |= errored;
let stalled = now.saturating_sub(self.seen.1) >= STALL_RESTART;
if !lost && !(self.pending_error && stalled) {
return Verdict::Healthy;
}
self.restarts.retain(|t| now.saturating_sub(*t) < RESTART_WINDOW);
if self.restarts.len() >= MAX_RESTARTS {
return Verdict::GiveUp;
}
self.restarts.push(now);
self.seen = (played, now);
self.pending_error = false;
Verdict::Restart
}
}
/// One device callback: run the shared mixer into `buf` and advance the clock. Never panics: a
/// panicking mixer plays silence and fails the stream, and a mixer held by another stream (never
/// expected: a restart drops the old stream first) plays silence without advancing the clock.
fn render(fill: &Mutex<Fill>, buf: &mut [f32], channels: usize, played: &AtomicU64, health: &Health) {
let mut guard = match fill.try_lock() {
Ok(guard) => guard,
Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
Err(TryLockError::WouldBlock) => {
buf.fill(0.0);
return;
}
};
if health.failed.load(Ordering::SeqCst) || catch_unwind(AssertUnwindSafe(|| (*guard)(buf, channels))).is_err() {
buf.fill(0.0);
if !health.failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: the audio mixer failed; playing without sound");
}
return;
}
played.fetch_add((buf.len() / channels.max(1)) as u64, Ordering::SeqCst);
}
fn stream_error(health: &Health, e: cpal::StreamError) {
match e {
cpal::StreamError::DeviceNotAvailable => health.lost.store(true, Ordering::SeqCst),
cpal::StreamError::BackendSpecific { .. } => health.errored.store(true, Ordering::SeqCst),
}
if !health.reported.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: audio stream error: {e}");
}
}
fn convert_samples<T: cpal::Sample + cpal::FromSample<f32>>(input: &[f32], output: &mut [T]) {
// Integer conversion represents [-1, 1). In particular I24 cannot represent +1 exactly.
let upper = f32::from_bits(1.0_f32.to_bits() - 1);
for (sample, converted) in input.iter().zip(output) {
*converted = T::from_sample(if sample.is_finite() { sample.clamp(-1.0, upper) } else { 0.0 });
}
}
impl AudioOut for CpalOut {
fn start(&mut self, fill: Box<dyn FnMut(&mut [f32], usize) + Send>) -> Result<u32, String> {
self.stop();
self.played.store(0, Ordering::SeqCst);
self.health = Arc::new(Health::default());
self.watchdog.reset(self.epoch.elapsed());
let fill = Arc::new(Mutex::new(fill));
self.open(fill.clone(), None)?;
self.fill = Some(fill);
Ok(self.rate)
}
fn stop(&mut self) {
self.stream = None;
self.fill = None;
}
fn recover(&mut self) {
if self.stream.is_none() || self.health.failed.load(Ordering::SeqCst) {
return;
}
let lost = self.health.lost.swap(false, Ordering::SeqCst);
let errored = self.health.errored.swap(false, Ordering::SeqCst);
match self.watchdog.check(self.epoch.elapsed(), self.played.load(Ordering::SeqCst), errored, lost) {
Verdict::Healthy => {}
Verdict::Restart => {
log::warn!("the audio output stream {}; restarting it", if lost { "lost its device" } else { "stopped after an error" });
if let Err(e) = self.restart() {
log::warn!("the audio output stream could not be restarted: {e}");
self.stream = None;
self.health.failed.store(true, Ordering::SeqCst);
}
}
Verdict::GiveUp => {
log::warn!("the audio output stream keeps failing ({MAX_RESTARTS} restarts within {} s)", RESTART_WINDOW.as_secs());
self.health.failed.store(true, Ordering::SeqCst);
}
}
}
fn sample_rate(&self) -> u32 {
self.rate
@ -263,7 +425,7 @@ impl AudioOut for CpalOut {
self.channels as usize
}
fn played_frames(&self) -> Option<u64> {
self.stream.as_ref().filter(|_| !self.failed.load(Ordering::SeqCst)).map(|_| self.played.load(Ordering::SeqCst))
self.stream.as_ref().filter(|_| !self.health.failed.load(Ordering::SeqCst)).map(|_| self.played.load(Ordering::SeqCst))
}
fn devices(&self) -> AudioDevices {
let h = host(&self.hw.device_class);
@ -327,11 +489,92 @@ mod tests {
}
#[test]
fn failed_stream_relinquishes_the_playback_clock() {
let failed = AtomicBool::new(false);
stream_error(&failed, cpal::StreamError::DeviceNotAvailable);
assert!(failed.load(Ordering::SeqCst));
fn stream_errors_ask_for_a_restart_instead_of_failing_playback() {
let health = Health::default();
stream_error(&health, cpal::StreamError::BackendSpecific { err: cpal::BackendSpecificError { description: "`alsa::poll()` returned POLLERR".into() } });
assert!(health.errored.load(Ordering::SeqCst) && !health.lost.load(Ordering::SeqCst));
stream_error(&health, cpal::StreamError::DeviceNotAvailable);
assert!(health.lost.load(Ordering::SeqCst));
assert!(!health.failed.load(Ordering::SeqCst), "a stream error alone must not silence playback");
let output = CpalOut::new();
assert_eq!(output.played_frames(), None);
assert_eq!(output.played_frames(), None, "no stream, no clock");
}
fn ms(n: u64) -> Duration {
Duration::from_millis(n)
}
#[test]
fn an_error_while_samples_keep_flowing_is_only_a_hiccup() {
let mut w = Watchdog::default();
w.reset(ms(0));
assert_eq!(w.check(ms(10), 480, true, false), Verdict::Healthy);
// the stream keeps consuming samples after the error: never restarted
for (i, t) in (20..2_000).step_by(20).enumerate() {
assert_eq!(w.check(ms(t), 480 * (i as u64 + 2), false, false), Verdict::Healthy);
}
}
#[test]
fn a_stream_that_errors_and_stops_consuming_is_restarted() {
let mut w = Watchdog::default();
w.reset(ms(0));
assert_eq!(w.check(ms(10), 480, false, false), Verdict::Healthy);
// POLLERR: the backend reports errors and no longer calls back
assert_eq!(w.check(ms(20), 480, true, false), Verdict::Healthy);
assert_eq!(w.check(ms(200), 480, true, false), Verdict::Healthy);
assert_eq!(w.check(ms(310), 480, false, false), Verdict::Restart);
// the restarted stream gets a fresh grace period
assert_eq!(w.check(ms(400), 480, false, false), Verdict::Healthy);
// a stall without any reported error is left to the UI's own watchdog
assert_eq!(w.check(ms(2_000), 480, false, false), Verdict::Healthy);
}
#[test]
fn a_lost_device_restarts_at_once_and_repeated_failures_give_up() {
let mut w = Watchdog::default();
w.reset(ms(0));
assert_eq!(w.check(ms(5), 0, false, true), Verdict::Restart);
assert_eq!(w.check(ms(10), 0, false, true), Verdict::Restart);
assert_eq!(w.check(ms(15), 0, false, true), Verdict::Restart);
assert_eq!(w.check(ms(20), 0, false, true), Verdict::GiveUp);
// sporadic failures far apart keep being recovered
assert_eq!(w.check(ms(10_006), 0, false, true), Verdict::Restart);
}
#[test]
fn the_device_callback_never_panics_and_counts_played_frames() {
let (played, health) = (AtomicU64::new(0), Health::default());
let fill: Mutex<Fill> = Mutex::new(Box::new(|buf: &mut [f32], _| buf.fill(0.25)));
let mut buf = [1.0_f32; 8];
render(&fill, &mut buf, 2, &played, &health);
assert_eq!((buf, played.load(Ordering::SeqCst)), ([0.25; 8], 4));
// a mixer still held elsewhere plays silence without moving the clock
{
let _held = fill.lock().unwrap();
render(&fill, &mut buf, 2, &played, &health);
}
assert_eq!((buf, played.load(Ordering::SeqCst)), ([0.0; 8], 4));
// zero channels from a broken driver must not divide by zero
render(&fill, &mut buf, 0, &played, &health);
assert_eq!(played.load(Ordering::SeqCst), 12);
}
#[test]
fn a_panicking_mixer_plays_silence_and_fails_the_stream() {
let (played, health) = (AtomicU64::new(0), Health::default());
let fill: Mutex<Fill> = Mutex::new(Box::new(|buf: &mut [f32], _| {
buf.fill(0.5);
panic!("mixer bug");
}));
let mut buf = [1.0_f32; 4];
render(&fill, &mut buf, 2, &played, &health);
assert_eq!(buf, [0.0; 4]);
assert_eq!(played.load(Ordering::SeqCst), 0);
assert!(health.failed.load(Ordering::SeqCst));
// later callbacks stay silent instead of re-entering the broken mixer
buf = [1.0; 4];
render(&fill, &mut buf, 2, &played, &health);
assert_eq!(buf, [0.0; 4]);
}
}

831
apps/filmcraft/src/graphics.rs
View File

@ -1,33 +1,487 @@
//! Windows graphics instance configuration, applied before wgpu can initialize any driver.
//! Graphics start-up: the wgpu backends and adapter the window is created with, and the fallback
//! when a start fails (docs/gaps.md G4, #820).
//!
//! A graphics driver can crash inside wgpu instance or device creation (Intel's Vulkan driver
//! `igvk64.dll`, #512 #413; AMD's OpenGL driver `atio6axx.dll`, #477), where no Rust error
//! handling runs, so every launch died the same way until the user set `WGPU_BACKEND` by hand.
//!
//! # When a start counts as failed
//! The data folder keeps the mode to start with and the failed starts in a row in it
//! ([`STATE_FILE`]). Each start also leaves its own attempt file (`graphics-start/<pid>-<n>.json`,
//! so concurrent launches never write the same file) that is removed once the window has painted
//! two frames, when the process ends normally, or [`DEATH_WINDOW`] after launch, whatever happened
//! by then. An attempt file found by a later launch therefore means a process that died within
//! [`DEATH_WINDOW`] of its start without painting: a driver crash, typically. A start that hangs,
//! or is force-quit or crashes later, is not counted; neither is another instance that is still
//! starting (its attempt is younger than [`DEATH_WINDOW`] and is left alone). After
//! [`FAILURES_TO_MOVE_ON`] such failures in a row in the same mode, the launch moves on to the next
//! [`Mode`] of the platform's [`LADDER`] and keeps it; a start that paints resets the count. A
//! start-up error that wgpu does report (no adapter, device creation failed) moves on at once,
//! within the same process.
//!
//! # Getting back
//! A fallback mode shows a notice with a reset button in the window (`graphics.notice.reset`,
//! [`GraphicsStart::reset_to_default`]); `--graphics default` does the same from the command
//! line, and `--graphics dx12|software` picks a mode by hand. An explicit `WGPU_BACKEND` leaves
//! the saved state alone: the user is in charge.
//!
//! All files are written beside and renamed into place, so a reader never sees half a file; two
//! instances updating [`STATE_FILE`] at the same moment can lose one update (a failure counted
//! once less, a confirmation repeated by the next start), never corrupt it.
use eframe::{NativeOptions, egui_wgpu::WgpuSetup, wgpu::Backends};
use std::io::Read as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// Restrict the wgpu instance to the primary backends (Vulkan, DirectX 12, Metal), i.e. leave
/// OpenGL out, unless `backend_override` (normally `Backends::from_env()`, i.e. `WGPU_BACKEND`)
/// names other backends.
pub fn configure(options: &mut NativeOptions, backend_override: Option<Backends>) {
if let WgpuSetup::CreateNew(create) = &mut options.wgpu_options.wgpu_setup {
// eframe's default includes GL. Creating that backend can crash inside AMD's
// atio6axx.dll before adapter selection or any Rust error handling runs, so the
// window never appears. Choosing an adapter afterwards is too late: only leaving GL
// out of the instance avoids it. Keep Vulkan and DX12 both: forcing DX12 alone sends
// shaders through FXC (which rejects fx.wgsl, silently falling back to the CPU) and
// leaves machines without DX12 with no backend at all.
create.instance_descriptor.backends = backend_override.unwrap_or(Backends::PRIMARY);
use eframe::{
NativeOptions,
egui_wgpu::WgpuSetup,
wgpu::{self, Backends},
};
/// The saved graphics state in the data folder: `{"mode": "<mode>", "failures": <n>}`.
pub const STATE_FILE: &str = "graphics-start.json";
/// The folder in the data folder with one attempt file per start that has not painted yet.
pub const ATTEMPTS_DIR: &str = "graphics-start";
/// How long after launch a start that has not painted may die and still count as failed. A
/// driver crash in instance or device creation happens within a second or two.
pub const DEATH_WINDOW: Duration = Duration::from_secs(15);
/// Failed starts in a row, in the same mode, that move on to the next mode.
pub const FAILURES_TO_MOVE_ON: u32 = 2;
/// Attempt files looked at per launch; anything beyond is left for a later launch.
const MAX_ATTEMPTS: usize = 64;
/// The state and attempt files are a few bytes; anything longer is not ours.
const MAX_FILE: u64 = 4096;
/// How the window's graphics device is created.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Mode {
/// The platform default: Vulkan or DirectX 12 on Windows (no OpenGL, see [`apply`]), Metal on
/// macOS, Vulkan or OpenGL elsewhere.
Default,
/// DirectX 12 only (Windows), for Vulkan drivers that crash at start-up.
Dx12,
/// A software adapter (WARP on Windows, lavapipe / llvmpipe elsewhere); the preview then
/// composites on the CPU (`FilmcraftApp::set_wgpu`).
Software,
}
impl Mode {
/// The name in [`STATE_FILE`] and on the command line (`--graphics <name>`).
pub fn name(self) -> &'static str {
match self {
Mode::Default => "default",
Mode::Dx12 => "dx12",
Mode::Software => "software",
}
}
/// The English name shown in the interface (translated there).
pub fn label(self) -> &'static str {
match self {
Mode::Default => "default graphics",
Mode::Dx12 => "DirectX 12 only",
Mode::Software => "software rendering",
}
}
pub fn parse(name: &str) -> Option<Mode> {
[Mode::Default, Mode::Dx12, Mode::Software].into_iter().find(|m| m.name() == name)
}
}
/// The modes tried in order on this platform. macOS has only Metal, and no software adapter.
pub const LADDER: &[Mode] = if cfg!(target_os = "windows") {
&[Mode::Default, Mode::Dx12, Mode::Software]
} else if cfg!(target_os = "macos") {
&[Mode::Default]
} else {
&[Mode::Default, Mode::Software]
};
/// The mode after `mode` in `ladder`, if any.
fn next_in(ladder: &[Mode], mode: Mode) -> Option<Mode> {
let i = ladder.iter().position(|m| *m == mode)?;
ladder.get(i.checked_add(1)?).copied()
}
/// [`STATE_FILE`]: the mode to start with and the failed starts in a row in it.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Saved {
mode: Mode,
failures: u32,
}
/// An attempt file left by a start: its process id (from the file name), mode and launch time.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Attempt {
pid: u32,
mode: Mode,
/// Seconds since the Unix epoch.
started: u64,
}
/// What a launch makes of the saved state and the attempt files it found.
#[derive(Debug, PartialEq, Eq)]
struct Decision {
/// The state to start with and save.
saved: Saved,
/// Moved on to the next mode because the last starts failed.
moved_on: bool,
/// Per attempt: done with (counted, or not ours to count), delete it.
delete: Vec<bool>,
}
/// This launch's mode from the saved state and the attempts other starts left. `now` is in Unix
/// seconds, `pid` is this process (an attempt with our process id is a dead process whose id was
/// reused), `window` is [`DEATH_WINDOW`] in seconds.
fn decide(saved: Option<Saved>, attempts: &[Attempt], now: u64, pid: u32, window: u64, ladder: &[Mode]) -> Decision {
let first = ladder.first().copied().unwrap_or(Mode::Default);
// Nothing saved, or a mode this platform does not use (a data folder copied from another OS).
let saved = saved.filter(|s| ladder.contains(&s.mode)).unwrap_or(Saved { mode: first, failures: 0 });
let mut failures = saved.failures;
let mut delete = Vec::with_capacity(attempts.len());
for a in attempts {
if a.started > now.saturating_add(window) {
// From the future (a clock that jumped back): unknown, and not worth keeping.
delete.push(true);
continue;
}
let dead = a.pid == pid || now.saturating_sub(a.started) >= window;
// A younger attempt is another instance starting right now: leave it to that instance.
delete.push(dead);
// Only failures of the mode we would start in count; older ones are history.
if dead && a.mode == saved.mode {
failures = failures.saturating_add(1);
}
}
if failures >= FAILURES_TO_MOVE_ON {
return match next_in(ladder, saved.mode) {
Some(next) => Decision { saved: Saved { mode: next, failures: 0 }, moved_on: true, delete },
// Nothing left to try: the last mode is still the best chance.
None => Decision { saved: Saved { mode: saved.mode, failures: 0 }, moved_on: false, delete },
};
}
Decision { saved: Saved { mode: saved.mode, failures }, moved_on: false, delete }
}
fn read_small(path: &Path) -> Option<String> {
let mut text = String::new();
std::fs::File::open(path).ok()?.take(MAX_FILE).read_to_string(&mut text).ok()?;
Some(text)
}
/// [`Saved`] from its text; `None` for anything else. A missing failure count is none.
fn parse_state(text: &str) -> Option<Saved> {
let v: serde_json::Value = serde_json::from_str(text).ok()?;
let mode = Mode::parse(v.get("mode")?.as_str()?)?;
let failures = match v.get("failures") {
None => 0,
Some(n) => u32::try_from(n.as_u64()?).ok()?,
};
Some(Saved { mode, failures })
}
fn state_json(s: Saved) -> String {
serde_json::json!({ "mode": s.mode.name(), "failures": s.failures }).to_string()
}
/// An attempt from its file name (`<pid>-<n>.json`) and text; `None` for anything else.
fn parse_attempt(file_name: &str, text: &str) -> Option<Attempt> {
let pid = file_name.strip_suffix(".json")?.split_once('-')?.0.parse().ok()?;
let v: serde_json::Value = serde_json::from_str(text).ok()?;
Some(Attempt { pid, mode: Mode::parse(v.get("mode")?.as_str()?)?, started: v.get("started")?.as_u64()? })
}
fn attempt_json(mode: Mode, started: u64) -> String {
serde_json::json!({ "mode": mode.name(), "started": started }).to_string()
}
/// Write `text` beside `path` (a name only this process uses) and rename it into place, so a
/// start that dies mid-write, or another instance reading, never sees half a file.
fn write_atomic(path: &Path, text: &str) -> std::io::Result<()> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
let mut tmp = path.as_os_str().to_owned();
tmp.push(format!(".{}.tmp", std::process::id()));
let tmp = PathBuf::from(tmp);
std::fs::write(&tmp, text)?;
std::fs::rename(&tmp, path).inspect_err(|_| {
let _ = std::fs::remove_file(&tmp);
})
}
fn remove_quietly(path: &Path) {
match std::fs::remove_file(path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => log::warn!("can't remove {}: {e}", path.display()),
_ => {}
}
}
/// The attempt files in `dir` (at most [`MAX_ATTEMPTS`]): path and contents, `None` when unreadable.
fn list_attempts(dir: &Path) -> Vec<(PathBuf, Option<Attempt>)> {
let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
entries
.filter_map(Result::ok)
.filter_map(|e| {
let name = e.file_name().into_string().ok()?;
name.ends_with(".json").then(|| {
let path = e.path();
let attempt = read_small(&path).and_then(|text| parse_attempt(&name, &text));
(path, attempt)
})
})
.take(MAX_ATTEMPTS)
.collect()
}
fn unix_now() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_secs())
}
/// Attempt numbers within this process, so each attempt file has its own name.
static NEXT_ATTEMPT: AtomicU32 = AtomicU32::new(0);
/// This launch's graphics mode and its files in the data folder.
#[derive(Clone, Debug)]
pub struct GraphicsStart {
/// [`STATE_FILE`]; `None` without a data folder or with `WGPU_BACKEND` set.
state: Option<PathBuf>,
/// This start's attempt file (with `state`).
attempt: Option<PathBuf>,
mode: Mode,
/// [`Self::reset_to_default`] was used: a later [`Self::confirm`] must not save this mode over it.
reset: Arc<AtomicBool>,
}
impl GraphicsStart {
/// Pick this launch's mode from the data folder and record the attempt. `forced`
/// (`--graphics`) starts in that mode and saves it. `explicit_backend` (`WGPU_BACKEND` set)
/// leaves the data folder alone and starts in `forced`, else [`Mode::Default`].
pub fn begin(data_dir: Option<&Path>, explicit_backend: bool, forced: Option<Mode>) -> Self {
let start = Self::begin_at(data_dir, explicit_backend, forced, unix_now(), std::process::id());
if let Some(path) = start.attempt.clone() {
expire_later(path);
}
start
}
/// [`Self::begin`] at `now` (Unix seconds) as process `pid`, without the expiry timer.
fn begin_at(data_dir: Option<&Path>, explicit_backend: bool, forced: Option<Mode>, now: u64, pid: u32) -> Self {
let first = LADDER.first().copied().unwrap_or(Mode::Default);
let forced = forced.map(|m| if LADDER.contains(&m) { m } else { first });
let reset = Arc::new(AtomicBool::new(false));
let Some(dir) = data_dir.filter(|_| !explicit_backend) else {
return Self { state: None, attempt: None, mode: forced.unwrap_or(Mode::Default), reset };
};
let state = dir.join(STATE_FILE);
let attempts_dir = dir.join(ATTEMPTS_DIR);
let saved = read_small(&state).and_then(|t| parse_state(&t));
let found = list_attempts(&attempts_dir);
let known: Vec<Attempt> = found.iter().filter_map(|(_, a)| *a).collect();
let decision = decide(saved, &known, now, pid, DEATH_WINDOW.as_secs(), LADDER);
let mut to_delete = decision.delete.iter();
for (path, attempt) in &found {
// unreadable files in our folder are debris (a different version, a damaged disk)
if attempt.is_none() || to_delete.next().copied().unwrap_or(false) {
remove_quietly(path);
}
}
let file = state.display();
let next = match forced {
Some(mode) => {
log::info!("graphics mode {} chosen with --graphics", mode.name());
Saved { mode, failures: 0 }
}
None => {
if decision.moved_on {
let failed = saved.map_or(first, |s| s.mode).name();
log::warn!(
"the last {FAILURES_TO_MOVE_ON} starts with graphics mode {failed} died before their window painted; starting with {} (`--graphics default` or the notice in the window goes back)",
decision.saved.mode.name()
);
} else if decision.saved.failures > 0 {
log::warn!("the last start with graphics mode {} died before its window painted", decision.saved.mode.name());
} else if decision.saved.mode != Mode::Default {
log::info!("graphics mode {} after earlier failed starts ({file}; `--graphics default` goes back)", decision.saved.mode.name());
}
decision.saved
}
};
if saved != Some(next)
&& let Err(e) = write_atomic(&state, &state_json(next))
{
log::warn!("can't write {file}: {e}");
}
let start = Self { state: Some(state), attempt: None, mode: next.mode, reset };
start.with_new_attempt(&attempts_dir, now, pid)
}
/// Record a new attempt file for the current mode.
fn with_new_attempt(mut self, dir: &Path, now: u64, pid: u32) -> Self {
let n = NEXT_ATTEMPT.fetch_add(1, Ordering::Relaxed);
let path = dir.join(format!("{pid}-{n}.json"));
match write_atomic(&path, &attempt_json(self.mode, now)) {
Ok(()) => self.attempt = Some(path),
Err(e) => log::warn!("can't write {}: {e}", path.display()),
}
self
}
pub fn mode(&self) -> Mode {
self.mode
}
/// Running in a fallback mode, i.e. not the platform default.
pub fn degraded(&self) -> bool {
self.mode != Mode::Default
}
/// Move on to the next mode after a start-up error wgpu reported; `false` when none is left.
/// The error is certain, so the next mode is saved at once.
pub fn step_down(&mut self) -> bool {
let Some(next) = next_in(LADDER, self.mode) else {
return false;
};
log::warn!("graphics mode {} failed to start; trying {}", self.mode.name(), next.name());
self.mode = next;
if let Some(state) = self.state.clone() {
self.save(&state, Saved { mode: next, failures: 0 });
if let Some(old) = self.attempt.take() {
remove_quietly(&old);
if let Some(dir) = old.parent() {
let path = self.clone().with_new_attempt(dir, unix_now(), std::process::id()).attempt;
if let Some(path) = &path {
expire_later(path.clone());
}
self.attempt = path;
}
}
}
true
}
/// The window is up: keep this mode, and the failed starts in a row are over.
pub fn confirm(&self) {
self.finish();
let Some(state) = &self.state else { return };
if self.reset.load(Ordering::Relaxed) {
return;
}
let ok = Saved { mode: self.mode, failures: 0 };
if read_small(state).and_then(|t| parse_state(&t)) != Some(ok) {
self.save(state, ok);
}
}
/// This start is over, whether it painted or not (the window closed, or every mode failed
/// with an error that was reported): its attempt no longer means a crash.
pub fn finish(&self) {
if let Some(path) = &self.attempt {
remove_quietly(path);
}
}
/// Start with the default mode next time (the notice's reset button). This process keeps
/// its mode: the graphics device can't be swapped under a running window.
pub fn reset_to_default(&self) -> Result<(), String> {
self.reset.store(true, Ordering::Relaxed);
let Some(state) = &self.state else { return Ok(()) };
let first = LADDER.first().copied().unwrap_or(Mode::Default);
write_atomic(state, &state_json(Saved { mode: first, failures: 0 })).map_err(|e| format!("can't write {}: {e}", state.display()))?;
log::info!("graphics mode reset to {} for the next start", first.name());
Ok(())
}
fn save(&self, path: &Path, s: Saved) {
if let Err(e) = write_atomic(path, &state_json(s)) {
log::warn!("can't write {}: {e}", path.display());
}
}
}
/// Remove the attempt file at `path` [`DEATH_WINDOW`] from now: a start still alive then is not
/// counted as a failed one, painted or not.
fn expire_later(path: PathBuf) {
let spawned = std::thread::Builder::new().name("graphics-start".into()).spawn(move || {
let _ = std::panic::catch_unwind(move || {
std::thread::sleep(DEATH_WINDOW);
remove_quietly(&path);
});
});
if let Err(e) = spawned {
log::warn!("can't start the graphics start timer: {e}");
}
}
/// Configure `options` for `mode`, before eframe creates the wgpu instance. `backend_override`
/// is normally `Backends::from_env()`, i.e. `WGPU_BACKEND`.
pub fn apply(options: &mut NativeOptions, mode: Mode, backend_override: Option<Backends>) {
apply_for(options, mode, backend_override, cfg!(target_os = "windows"));
}
fn apply_for(options: &mut NativeOptions, mode: Mode, backend_override: Option<Backends>, windows: bool) {
let WgpuSetup::CreateNew(create) = &mut options.wgpu_options.wgpu_setup else {
return;
};
match mode {
// eframe's default (`WGPU_BACKEND`, else PRIMARY | GL), except on Windows: there it
// includes GL. Creating that backend can crash inside AMD's atio6axx.dll before adapter
// selection or any Rust error handling runs, so the window never appears. Choosing an
// adapter afterwards is too late: only leaving GL out of the instance avoids it. Keep
// Vulkan and DX12 both: forcing DX12 alone sends shaders through FXC (which rejects
// fx.wgsl, silently falling back to the CPU) and leaves machines without DX12 with no
// backend at all.
Mode::Default if windows => create.instance_descriptor.backends = backend_override.unwrap_or(Backends::PRIMARY),
Mode::Default => {}
// Without Vulkan, whose driver is what crashes; GPU effects may fall back to the CPU.
Mode::Dx12 => create.instance_descriptor.backends = Backends::DX12,
Mode::Software => {
create.instance_descriptor.backends = software_backends(windows);
create.native_adapter_selector = Some(Arc::new(pick_software));
}
}
}
/// The backends a software adapter is looked for in: WARP is a DX12 adapter, and GL stays out on
/// Windows for the reason in [`apply_for`].
fn software_backends(windows: bool) -> Backends {
if windows { Backends::DX12 } else { Backends::VULKAN | Backends::GL }
}
/// The first software adapter that can draw to the window's surface.
fn pick_software(adapters: &[wgpu::Adapter], surface: Option<&wgpu::Surface<'_>>) -> Result<wgpu::Adapter, String> {
adapters
.iter()
.find(|a| filmcraft_ui_egui::software_rasterizer(&a.get_info()).is_some() && surface.is_none_or(|s| a.is_surface_supported(s)))
.cloned()
.ok_or_else(|| format!("no software graphics adapter (WARP, lavapipe, llvmpipe) among {} adapters", adapters.len()))
}
/// Whether `err` from `eframe::run_native` is a graphics start-up failure that another mode may
/// avoid (as opposed to a display, event-loop or app error).
pub fn is_graphics_failure(err: &eframe::Error) -> bool {
matches!(err, eframe::Error::Wgpu(_))
}
#[cfg(test)]
mod tests {
use super::*;
fn configured_backends(backend_override: Option<Backends>) -> Backends {
const WINDOWS: &[Mode] = &[Mode::Default, Mode::Dx12, Mode::Software];
const LINUX: &[Mode] = &[Mode::Default, Mode::Software];
const MACOS: &[Mode] = &[Mode::Default];
fn configured(mode: Mode, backend_override: Option<Backends>, windows: bool) -> (Backends, bool) {
let mut options = NativeOptions::default();
configure(&mut options, backend_override);
apply_for(&mut options, mode, backend_override, windows);
let WgpuSetup::CreateNew(create) = options.wgpu_options.wgpu_setup else {
panic!("default native options must create a wgpu instance");
};
create.instance_descriptor.backends
(create.instance_descriptor.backends, create.native_adapter_selector.is_some())
}
#[test]
@ -37,16 +491,349 @@ mod tests {
panic!("default native options must create a wgpu instance");
};
assert!(eframe_default.instance_descriptor.backends.contains(Backends::GL));
let configured = configured_backends(None);
assert_eq!(configured, Backends::PRIMARY);
assert!(!configured.contains(Backends::GL));
assert!(configured.contains(Backends::DX12) && configured.contains(Backends::VULKAN));
let (backends, selector) = configured(Mode::Default, None, true);
assert_eq!(backends, Backends::PRIMARY);
assert!(!backends.contains(Backends::GL));
assert!(backends.contains(Backends::DX12) && backends.contains(Backends::VULKAN));
assert!(!selector);
}
#[test]
fn explicit_backend_override_is_preserved() {
for backends in [Backends::VULKAN, Backends::GL, Backends::DX12, Backends::VULKAN | Backends::DX12, Backends::empty()] {
assert_eq!(configured_backends(Some(backends)), backends);
assert_eq!(configured(Mode::Default, Some(backends), true).0, backends);
}
}
#[test]
fn fallback_modes_leave_out_the_crashing_backends() {
// DX12 only: Intel's and AMD's Vulkan drivers are what crashed (#512, #413).
assert_eq!(configured(Mode::Dx12, None, true), (Backends::DX12, false));
// Software on Windows is WARP (DX12), never GL (atio6axx.dll, #477).
assert_eq!(configured(Mode::Software, None, true), (Backends::DX12, true));
assert_eq!(configured(Mode::Software, None, false), (Backends::VULKAN | Backends::GL, true));
// Off Windows the default is eframe's own.
let WgpuSetup::CreateNew(eframe_default) = NativeOptions::default().wgpu_options.wgpu_setup else {
panic!("default native options must create a wgpu instance");
};
assert_eq!(configured(Mode::Default, None, false), (eframe_default.instance_descriptor.backends, false));
}
const W: u64 = 15;
const NOW: u64 = 1_000_000;
const ME: u32 = 7;
fn saved(mode: Mode, failures: u32) -> Option<Saved> {
Some(Saved { mode, failures })
}
fn dead(mode: Mode) -> Attempt {
Attempt { pid: 1, mode, started: NOW - 100 }
}
fn mode_after(s: Option<Saved>, attempts: &[Attempt], ladder: &[Mode]) -> (Mode, u32, bool) {
let d = decide(s, attempts, NOW, ME, W, ladder);
(d.saved.mode, d.saved.failures, d.moved_on)
}
#[test]
fn two_failed_starts_in_a_row_move_on_to_the_next_mode_and_keep_it() {
// first launch
assert_eq!(mode_after(None, &[], WINDOWS), (Mode::Default, 0, false));
// one start died: counted, but the same mode is tried again
assert_eq!(mode_after(None, &[dead(Mode::Default)], WINDOWS), (Mode::Default, 1, false));
// the second in a row moves on: DX12, then software
assert_eq!(mode_after(saved(Mode::Default, 1), &[dead(Mode::Default)], WINDOWS), (Mode::Dx12, 0, true));
assert_eq!(mode_after(saved(Mode::Dx12, 1), &[dead(Mode::Dx12)], WINDOWS), (Mode::Software, 0, true));
assert_eq!(mode_after(saved(Mode::Default, 1), &[dead(Mode::Default)], LINUX), (Mode::Software, 0, true));
// two dead instances found at once count as two
assert_eq!(mode_after(None, &[dead(Mode::Default), dead(Mode::Default)], WINDOWS), (Mode::Dx12, 0, true));
// a saved mode is kept
assert_eq!(mode_after(saved(Mode::Dx12, 0), &[], WINDOWS), (Mode::Dx12, 0, false));
assert_eq!(mode_after(saved(Mode::Default, 1), &[], WINDOWS), (Mode::Default, 1, false));
// nothing left to try: stay
assert_eq!(mode_after(saved(Mode::Software, 1), &[dead(Mode::Software)], WINDOWS), (Mode::Software, 0, false));
assert_eq!(mode_after(saved(Mode::Default, 5), &[dead(Mode::Default)], MACOS), (Mode::Default, 0, false));
// a mode this platform has not: start over
assert_eq!(mode_after(saved(Mode::Dx12, 1), &[dead(Mode::Dx12)], LINUX), (Mode::Default, 0, false));
assert_eq!(mode_after(saved(Mode::Software, 0), &[], MACOS), (Mode::Default, 0, false));
assert_eq!(mode_after(None, &[], &[]), (Mode::Default, 0, false));
// a hostile count doesn't overflow
assert_eq!(mode_after(saved(Mode::Default, u32::MAX), &[dead(Mode::Default)], WINDOWS), (Mode::Dx12, 0, true));
}
#[test]
fn only_starts_that_died_soon_after_launch_in_the_current_mode_count() {
let starting = Attempt { pid: 2, mode: Mode::Default, started: NOW - 3 };
let just_now = Attempt { pid: 3, mode: Mode::Default, started: NOW };
let future = Attempt { pid: 4, mode: Mode::Default, started: NOW + 3600 };
let older_mode = dead(Mode::Dx12);
let mine = Attempt { pid: ME, mode: Mode::Default, started: NOW - 1 };
let d = decide(saved(Mode::Default, 0), &[starting, just_now, future, older_mode, mine], NOW, ME, W, WINDOWS);
// another instance still starting is left alone; one from the future is dropped; a
// failure in another mode is history; our own process id can only be a dead process
assert_eq!(d.delete, [false, false, true, true, true]);
assert_eq!((d.saved, d.moved_on), (Saved { mode: Mode::Default, failures: 1 }, false));
// the clock a little behind another instance's: still starting
let ahead = Attempt { pid: 5, mode: Mode::Default, started: NOW + 2 };
assert_eq!(decide(None, &[ahead], NOW, ME, W, WINDOWS).delete, [false]);
}
#[test]
fn the_files_round_trip_and_reject_anything_else() {
for mode in WINDOWS {
for failures in [0, 1, 2, u32::MAX] {
assert_eq!(parse_state(&state_json(Saved { mode: *mode, failures })), saved(*mode, failures));
}
assert_eq!(parse_attempt("12-0.json", &attempt_json(*mode, NOW)), Some(Attempt { pid: 12, mode: *mode, started: NOW }));
}
// without a count (an earlier build of this change wrote `pending`): none
assert_eq!(parse_state(r#"{"mode":"dx12","pending":true}"#), saved(Mode::Dx12, 0));
for bad in [
"",
"{",
"null",
"[]",
r#"{"mode":"vulkan","failures":1}"#,
r#"{"failures":1}"#,
r#"{"mode":1,"failures":1}"#,
r#"{"mode":"dx12","failures":"one"}"#,
r#"{"mode":"dx12","failures":-1}"#,
r#"{"mode":"dx12","failures":4294967296}"#,
] {
assert_eq!(parse_state(bad), None, "{bad}");
}
let good = attempt_json(Mode::Default, NOW);
for (name, text) in [
("12-0.json", "{"),
("12-0.json", r#"{"mode":"default"}"#),
("12-0.json", r#"{"mode":"default","started":-5}"#),
("x-0.json", good.as_str()),
("12.json", good.as_str()),
("-0.json", good.as_str()),
("99999999999-0.json", good.as_str()),
("12-0.txt", good.as_str()),
] {
assert_eq!(parse_attempt(name, text), None, "{name} {text}");
}
}
fn temp_dir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("filmcraft-graphics-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
dir
}
fn state_of(dir: &Path) -> Option<Saved> {
read_small(&dir.join(STATE_FILE)).and_then(|t| parse_state(&t))
}
fn attempt_files(dir: &Path) -> usize {
std::fs::read_dir(dir.join(ATTEMPTS_DIR)).map_or(0, |d| d.filter_map(Result::ok).count())
}
/// A launch as another process (`pid`) at `now`.
fn launch(dir: &Path, pid: u32, now: u64) -> GraphicsStart {
GraphicsStart::begin_at(Some(dir), false, None, now, pid)
}
#[test]
fn a_start_that_paints_keeps_its_mode_and_clears_its_attempt() {
let dir = temp_dir("paint");
let start = launch(&dir, 10, NOW);
assert_eq!(start.mode(), Mode::Default);
assert!(!start.degraded());
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert_eq!(attempt_files(&dir), 1);
start.confirm();
assert_eq!(attempt_files(&dir), 0);
// a window closed before it painted is no failure either
let start = launch(&dir, 11, NOW + 60);
start.finish();
assert_eq!(attempt_files(&dir), 0);
assert_eq!(launch(&dir, 12, NOW + 120).mode(), Mode::Default);
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn two_dead_starts_move_on_and_a_painted_one_resets_the_count() {
let dir = temp_dir("dead");
let second = LADDER.get(1).copied().unwrap_or(Mode::Default);
// dies without painting
let _ = launch(&dir, 20, NOW);
// the next launch counts it, tries the same mode again and dies too
let again = launch(&dir, 21, NOW + 60);
assert_eq!(again.mode(), Mode::Default);
assert_eq!(state_of(&dir), saved(Mode::Default, 1));
// two in a row: the next mode, kept
let moved = launch(&dir, 22, NOW + 120);
assert_eq!(moved.mode(), second);
assert_eq!(moved.degraded(), second != Mode::Default);
moved.confirm();
assert_eq!(launch(&dir, 23, NOW + 180).mode(), second);
// one failure, then a start that paints: the count starts over
let _ = std::fs::remove_dir_all(&dir);
let _ = launch(&dir, 30, NOW);
let painted = launch(&dir, 31, NOW + 60);
painted.confirm();
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
let _ = launch(&dir, 32, NOW + 120);
assert_eq!(launch(&dir, 33, NOW + 180).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn concurrent_launches_neither_count_nor_disturb_each_other() {
let dir = temp_dir("race");
// two instances start within the same seconds; neither has painted yet
let a = launch(&dir, 40, NOW);
let b = launch(&dir, 41, NOW + 1);
let c = launch(&dir, 42, NOW + 2);
assert_eq!((a.mode(), b.mode(), c.mode()), (Mode::Default, Mode::Default, Mode::Default));
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert_eq!(attempt_files(&dir), 3, "each instance has its own attempt file");
b.confirm();
a.confirm();
c.confirm();
assert_eq!(attempt_files(&dir), 0);
assert_eq!(launch(&dir, 43, NOW + 60).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn corrupt_huge_or_foreign_files_start_with_the_default() {
let dir = temp_dir("corrupt");
let _ = std::fs::create_dir_all(dir.join(ATTEMPTS_DIR));
let huge = format!(r#"{{"mode":"software","failures":1,"pad":"{}"}}"#, "x".repeat(10_000));
for text in ["garbage", huge.as_str()] {
let _ = std::fs::write(dir.join(STATE_FILE), text);
let _ = std::fs::write(dir.join(ATTEMPTS_DIR).join("50-0.json"), text);
let _ = std::fs::write(dir.join(ATTEMPTS_DIR).join("notes.txt"), text);
let start = launch(&dir, 51, NOW);
assert_eq!(start.mode(), Mode::Default);
// replaced with a well-formed state; the unreadable attempt is gone, other files stay
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert!(!dir.join(ATTEMPTS_DIR).join("50-0.json").exists());
assert!(dir.join(ATTEMPTS_DIR).join("notes.txt").exists());
start.finish();
}
// a data folder that is a file: nothing can be written, the default still starts
let file = dir.join("not-a-folder");
let _ = std::fs::write(&file, "x");
let mut start = GraphicsStart::begin_at(Some(&file), false, None, NOW, 52);
assert_eq!(start.mode(), Mode::Default);
start.confirm();
let _ = start.step_down();
assert!(start.reset_to_default().is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn many_attempt_files_are_bounded_per_launch() {
let dir = temp_dir("many");
let attempts = dir.join(ATTEMPTS_DIR);
let _ = std::fs::create_dir_all(&attempts);
let n = MAX_ATTEMPTS + 10;
for pid in 0..n {
let _ = std::fs::write(attempts.join(format!("{}-0.json", pid + 1000)), attempt_json(Mode::Software, NOW + 1));
}
// all of them belong to instances still starting: none is counted or removed
let start = launch(&dir, 60, NOW);
assert_eq!(start.mode(), Mode::Default);
assert_eq!(attempt_files(&dir), n + 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn graphics_flag_picks_the_mode_and_default_undoes_a_fallback() {
let dir = temp_dir("forced");
let last = LADDER.last().copied().unwrap_or(Mode::Default);
let _ = write_atomic(&dir.join(STATE_FILE), &state_json(Saved { mode: last, failures: 1 }));
launch(&dir, 70, NOW).confirm();
// `--graphics default`
let start = GraphicsStart::begin_at(Some(&dir), false, Some(Mode::Default), NOW + 60, 71);
assert_eq!(start.mode(), Mode::Default);
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
start.confirm();
assert_eq!(launch(&dir, 72, NOW + 120).mode(), Mode::Default);
// `--graphics <the last mode>` is kept for the next launches
GraphicsStart::begin_at(Some(&dir), false, Some(last), NOW + 180, 73).confirm();
assert_eq!(launch(&dir, 74, NOW + 240).mode(), last);
// a mode this platform has not starts with the first one
if !LADDER.contains(&Mode::Dx12) {
assert_eq!(GraphicsStart::begin_at(Some(&dir), false, Some(Mode::Dx12), NOW + 300, 75).mode(), Mode::Default);
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn reset_to_default_wins_over_a_later_confirm() {
let dir = temp_dir("reset");
let last = LADDER.last().copied().unwrap_or(Mode::Default);
let _ = write_atomic(&dir.join(STATE_FILE), &state_json(Saved { mode: last, failures: 0 }));
let start = launch(&dir, 80, NOW);
assert_eq!(start.mode(), last);
// the notice's button, from a clone of the start (the app holds one)
assert_eq!(start.clone().reset_to_default(), Ok(()));
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
start.confirm();
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert_eq!(launch(&dir, 81, NOW + 60).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_explicit_backend_leaves_the_data_folder_alone() {
let dir = temp_dir("explicit");
let _ = write_atomic(&dir.join(STATE_FILE), &state_json(Saved { mode: Mode::Software, failures: 1 }));
let mut start = GraphicsStart::begin_at(Some(&dir), true, None, NOW, 90);
assert_eq!(start.mode(), Mode::Default);
start.confirm();
let _ = start.step_down();
assert_eq!(start.reset_to_default(), Ok(()));
assert_eq!(state_of(&dir), saved(Mode::Software, 1));
assert_eq!(attempt_files(&dir), 0);
// with --graphics too, that mode starts, still without touching the folder
let last = LADDER.last().copied().unwrap_or(Mode::Default);
assert_eq!(GraphicsStart::begin_at(Some(&dir), true, Some(last), NOW, 91).mode(), last);
assert_eq!(state_of(&dir), saved(Mode::Software, 1));
// no data folder: nothing is written, the default still starts
assert_eq!(GraphicsStart::begin_at(None, false, None, NOW, 92).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn step_down_walks_the_ladder_and_saves_each_mode_at_once() {
let dir = temp_dir("step");
let mut start = launch(&dir, ME, NOW);
for expected in LADDER.iter().skip(1) {
assert!(start.step_down());
assert_eq!(start.mode(), *expected);
assert_eq!(state_of(&dir), saved(*expected, 0));
assert_eq!(attempt_files(&dir), 1, "one attempt file per process");
}
assert!(!start.step_down(), "nothing after the last mode");
start.finish();
assert_eq!(attempt_files(&dir), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_software_selector_only_picks_a_software_adapter() {
if wgpu::Instance::enabled_backend_features().is_empty() {
return;
}
// only the backends software mode uses: no driver this test loads is one the app avoids
let backends = software_backends(cfg!(target_os = "windows"));
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { backends, ..wgpu::InstanceDescriptor::new_without_display_handle() });
let adapters = pollster::block_on(instance.enumerate_adapters(backends));
match pick_software(&adapters, None) {
Ok(adapter) => assert!(filmcraft_ui_egui::software_rasterizer(&adapter.get_info()).is_some()),
Err(why) => {
assert!(adapters.iter().all(|a| filmcraft_ui_egui::software_rasterizer(&a.get_info()).is_none()), "{why}");
}
}
assert!(pick_software(&[], None).is_err());
}
}

435
apps/filmcraft/src/main.rs
View File

@ -31,7 +31,6 @@ mod control_server;
#[cfg(target_os = "linux")]
mod dev_icon;
mod file_filters;
#[cfg(any(target_os = "windows", test))]
mod graphics;
mod logging;
#[cfg(target_os = "macos")]
@ -44,6 +43,8 @@ use filmcraft_engine::Session;
use filmcraft_engine::autosave::{AutosaveConfig, default_data_dir};
use filmcraft_ui_egui::FilmcraftApp;
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
/// Reverse-DNS app id: Linux `.desktop` file, hicolor icon name and Wayland app id.
const APP_ID: &str = "ai.storyteller.filmcraft";
@ -80,7 +81,7 @@ fn main() -> eframe::Result {
let logger = logging::install();
// lossy, so a value that is not Unicode is reported like any other bad port instead of ignored
let env_port = std::env::var_os("FILMCRAFT_CONTROL_PORT").map(|p| p.to_string_lossy().into_owned());
let Launch { control_port, control_input_only, files, demo, startup_flag, recover, data_dir } = match args::parse(env_port, std::env::args().skip(1)) {
let launch = match args::parse(env_port, std::env::args_os().skip(1)) {
Ok(Cli::Run(l)) => l,
Ok(Cli::Help) => {
tell(args::USAGE, false);
@ -95,7 +96,8 @@ fn main() -> eframe::Result {
std::process::exit(2);
}
};
let log_dir = data_dir.clone().or_else(default_data_dir).map(|d| d.join(logging::LOG_DIR));
let data_root = launch.data_dir.clone().or_else(default_data_dir);
let log_dir = data_root.as_ref().map(|d| d.join(logging::LOG_DIR));
log::info!("FilmCraft {} ({} {})", env!("CARGO_PKG_VERSION"), std::env::consts::OS, std::env::consts::ARCH);
// The log file lives next to the crash logs; opened after the arguments, so `--help`,
// `--version` and usage errors leave no file behind. Records logged until now go in first.
@ -118,165 +120,38 @@ fn main() -> eframe::Result {
// OS hardware video decoders (VideoToolbox on macOS) in front of our own; Settings ▸ Playback ▸
// Hardware decoding switches them off. Unsupported streams and failures use our decoders.
register_hardware_decoders();
register_gpu_frame_renderer();
let options = eframe::NativeOptions {
viewport: egui::ViewportBuilder::default()
.with_title("FilmCraft")
.with_inner_size([1600.0, 980.0])
.with_min_inner_size([900.0, 560.0])
.with_drag_and_drop(true)
.with_fullsize_content_view(true)
.with_titlebar_shown(false)
.with_title_shown(false)
// Started for an agent (`--control`): open without taking the user's keyboard focus.
.with_active(control_port.is_none())
.with_icon(app_icon())
// Wayland matches the window to `packaging/linux/ai.storyteller.filmcraft.desktop`.
.with_app_id(APP_ID),
event_loop_builder: agent_event_loop(control_port.is_some()),
..Default::default()
// Which graphics backends and adapter the window starts with: `--graphics`, else the saved
// mode, which moves on after starts that died before their window painted (a driver crash),
// see graphics.rs.
let backend_override = eframe::wgpu::Backends::from_env();
let mut graphics = graphics::GraphicsStart::begin(data_root.as_deref(), backend_override.is_some(), launch.graphics);
// The GPU export renderer creates its own wgpu instance with every backend; after a start
// fell back, that would load the driver that failed. Exports then composite on the CPU.
if graphics.mode() == graphics::Mode::Default {
register_gpu_frame_renderer();
}
let started = loop {
let mut options = native_options(launch.control_port.is_some());
// Before eframe creates the wgpu instance (e.g. no OpenGL on Windows, see graphics.rs).
graphics::apply(&mut options, graphics.mode(), backend_override);
let app_created = Arc::new(AtomicBool::new(false));
let result = eframe::run_native("FilmCraft", options, app_creator(launch.clone(), graphics.clone(), Arc::clone(&app_created)));
let app_created = app_created.load(Ordering::Relaxed);
match &result {
// wgpu reported the failure (no adapter, device creation failed) before the app
// existed: try the next mode in this process.
Err(e) if !app_created && graphics::is_graphics_failure(e) && graphics.step_down() => {
log::warn!("graphics start failed: {e}");
continue;
}
// Closed before the second frame: the window still came up.
Ok(()) if app_created => graphics.confirm(),
_ => {}
}
break result;
};
// Before eframe creates the wgpu instance: leave OpenGL out on Windows (see graphics.rs).
#[cfg(target_os = "windows")]
let options = {
let mut options = options;
graphics::configure(&mut options, eframe::wgpu::Backends::from_env());
options
};
let started = eframe::run_native(
"FilmCraft",
options,
Box::new(move |cc| {
let mut session = Session::default();
if let Some(dir) = data_dir.clone().or_else(default_data_dir) {
let mut cfg = AutosaveConfig::new(dir);
cfg.local_offset = local_offset;
if let Err(e) = session.start_autosave(cfg) {
log::warn!("auto-save and crash recovery unavailable: {e}");
}
}
// voice-over recording reads the microphone through cpal
session.voiceover.input = Some(Box::new(audio_in::CpalIn::new(&session.prefs.audio_hardware.device_class)));
let project = files.iter().find(|f| f.ends_with(".fcproj")).cloned();
if let Some(p) = project {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
} else if !startup_flag && session.prefs.general.at_startup == "openMostRecent" {
// Settings ▸ General ▸ At Startup ▸ Open Most Recent
let recent = session.prefs.general.recent_projects.iter().find(|p| std::path::Path::new(p).exists()).cloned();
match recent {
Some(p) => {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
}
None => {
let _ = session.execute("file.openDemoProject", json!({}));
}
}
} else if !startup_flag && session.prefs.general.at_startup == "emptyProject" {
} else if demo {
let _ = session.execute("file.openDemoProject", json!({}));
}
let media: Vec<String> = files.iter().filter(|f| !f.ends_with(".fcproj")).cloned().collect();
if !media.is_empty() {
let _ = session.execute("file.import", json!({"paths": media}));
}
if recover == Some(true) && !session.recovery_candidates().is_empty() {
let id = session.recovery_candidates()[0].id.clone();
match session.execute("file.recover", json!({"id": id})) {
Ok(r) => log::info!("recovered {r}"),
Err(e) => log::error!("recovery failed: {e}"),
}
}
let mut app = FilmcraftApp::new(session);
if recover == Some(false) {
app.dialog = None;
}
app.integrated_titlebar = cfg!(target_os = "macos");
if let Some(rs) = cc.wgpu_render_state.clone()
&& std::env::var_os("FILMCRAFT_CPU_COMPOSITE").is_none()
{
app.set_wgpu(rs);
}
// Keep device selection available even when the default device is unavailable.
// Settings ▸ Audio Hardware is applied on the first frame (`apply_prefs`).
app.audio = Some(Box::new(audio::CpalOut::new()));
app.hooks.pick_files = Some(Box::new(|exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_files()
.unwrap_or_default()
.into_iter()
.map(|p| p.to_string_lossy().to_string())
.collect()
}));
// Link Media ▸ Locate…, Attach Proxies, Reconnect Full Resolution: one path, not imported.
app.hooks.pick_file_for_relink = Some(Box::new(|exts: &[&str], _hint| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save = Some(Box::new(|name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save_as = Some(Box::new(|filter: &str, exts: &[&str], name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_folder = Some(Box::new(|| rfd::FileDialog::new().pick_folder().map(|p| p.to_string_lossy().to_string())));
app.hooks.pick_folder_at =
Some(Box::new(|dir: &str| rfd::FileDialog::new().set_directory(dir).pick_folder().map(|p| p.to_string_lossy().into_owned())));
app.hooks.open_path = Some(Box::new(open_path));
// Help menu, About and Discord links (#642): eframe is built without its `links`
// feature, so `ctx.open_url` does nothing on the desktop. Detached, so a launcher
// that waits for the browser never freezes the UI.
app.hooks.open_url = Some(Box::new(|url: &str| open::that_detached(url).map_err(|e| e.to_string())));
app.hooks.clipboard_media = Some(Box::new(clipboard_media));
// Settings ▸ General ▸ Interface Language ▸ System Language (#218).
app.hooks.cursor_screen_position = Some(Box::new(filmcraft_platform::cursor::cursor_screen_position));
app.hooks.system_languages = Some(Box::new(|| sys_locale::get_locales().collect()));
// Settings ▸ Appearance ▸ Appearance Mode ▸ Sync with system on Linux desktops whose
// compositor reports no theme to winit (no polling: see appearance.rs).
app.hooks.system_theme = appearance::service();
app.hooks.raise_without_focus = Some(Box::new(|| {
window_raise::raise_without_focus();
}));
app.hooks.pick_open_file = Some(Box::new(|filter: &str, exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_open_project = Some(Box::new(|| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
#[cfg(target_os = "macos")]
{
let (rx, update) = native_menu::install(&app, cc.egui_ctx.clone());
app.command_inbox = Some(rx);
app.hooks.shortcuts_changed = Some(update);
app.ui.show_menu_bar = false;
}
if let Some(port) = control_port {
let rx = control_server::start(port, cc.egui_ctx.clone());
app = if control_input_only { app.with_control_input_only(rx) } else { app.with_control(rx) };
}
Ok(Box::new(app))
}),
);
// However it ended, this start no longer counts as one that died.
graphics.finish();
// The window could not be created (no usable GPU / graphics driver, no display): say so
// instead of exiting silently, and keep the reason in the crash log.
if let Err(e) = &started {
@ -288,6 +163,189 @@ fn main() -> eframe::Result {
started
}
/// The main window's options; built for each start attempt (they hold a one-shot hook).
fn native_options(agent: bool) -> eframe::NativeOptions {
eframe::NativeOptions {
viewport: egui::ViewportBuilder::default()
.with_title("FilmCraft")
.with_inner_size([1600.0, 980.0])
.with_min_inner_size([900.0, 560.0])
.with_drag_and_drop(true)
.with_fullsize_content_view(true)
.with_titlebar_shown(false)
.with_title_shown(false)
// Started for an agent (`--control`): open without taking the user's keyboard focus.
.with_active(!agent)
.with_icon(app_icon())
// Wayland matches the window to `packaging/linux/ai.storyteller.filmcraft.desktop`.
.with_app_id(APP_ID),
event_loop_builder: agent_event_loop(agent),
..Default::default()
}
}
/// Creates the app once eframe has its window and graphics device; sets `created` when it runs,
/// and confirms the graphics mode once two frames have painted.
fn app_creator(launch: Launch, graphics: graphics::GraphicsStart, created: Arc<AtomicBool>) -> eframe::AppCreator<'static> {
let Launch { control_port, control_input_only, files, demo, startup_flag, recover, data_dir, graphics: _ } = launch;
Box::new(move |cc| {
created.store(true, Ordering::Relaxed);
// A fallback graphics mode says so in the window, with a button back to the default.
let notice = graphics.degraded().then(|| {
let start = graphics.clone();
filmcraft_ui_egui::graphics_notice::GraphicsNotice::new(graphics.mode().label(), Box::new(move || start.reset_to_default()))
});
let confirmed = AtomicBool::new(false);
cc.egui_ctx.on_end_pass(
"graphics-start",
Arc::new(move |ui| {
if ui.ctx().cumulative_frame_nr() >= 2 && !confirmed.swap(true, Ordering::Relaxed) {
graphics.confirm();
}
}),
);
let mut session = Session::default();
if let Some(dir) = data_dir.clone().or_else(default_data_dir) {
let mut cfg = AutosaveConfig::new(dir);
cfg.local_offset = local_offset;
if let Err(e) = session.start_autosave(cfg) {
log::warn!("auto-save and crash recovery unavailable: {e}");
}
}
// voice-over recording reads the microphone through cpal
session.voiceover.input = Some(Box::new(audio_in::CpalIn::new(&session.prefs.audio_hardware.device_class)));
let (project, media) = args::project_and_media(&files);
if let Some(p) = project {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
} else if !startup_flag && session.prefs.general.at_startup == "openMostRecent" {
// Settings ▸ General ▸ At Startup ▸ Open Most Recent
let recent = session.prefs.general.recent_projects.iter().find(|p| std::path::Path::new(p).exists()).cloned();
match recent {
Some(p) => {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
}
None => {
let _ = session.execute("file.openDemoProject", json!({}));
}
}
} else if !startup_flag && session.prefs.general.at_startup == "emptyProject" {
} else if demo {
let _ = session.execute("file.openDemoProject", json!({}));
}
if !media.is_empty() {
let _ = session.execute("file.import", json!({"paths": media}));
}
if recover == Some(true) && !session.recovery_candidates().is_empty() {
let id = session.recovery_candidates()[0].id.clone();
match session.execute("file.recover", json!({"id": id})) {
Ok(r) => log::info!("recovered {r}"),
Err(e) => log::error!("recovery failed: {e}"),
}
}
let mut app = FilmcraftApp::new(session);
app.graphics_notice = notice;
if recover == Some(false) {
app.dialog = None;
}
app.integrated_titlebar = cfg!(target_os = "macos");
if let Some(rs) = cc.wgpu_render_state.clone()
&& std::env::var_os("FILMCRAFT_CPU_COMPOSITE").is_none()
{
app.set_wgpu(rs);
}
// Keep device selection available even when the default device is unavailable.
// Settings ▸ Audio Hardware is applied on the first frame (`apply_prefs`).
app.audio = Some(Box::new(audio::CpalOut::new()));
app.hooks.pick_files = Some(Box::new(|exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_files()
.unwrap_or_default()
.into_iter()
.map(|p| p.to_string_lossy().to_string())
.collect()
}));
// Link Media ▸ Locate…, Attach Proxies, Reconnect Full Resolution: one path, not imported.
app.hooks.pick_file_for_relink = Some(Box::new(|exts: &[&str], _hint| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save = Some(Box::new(|name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save_as = Some(Box::new(|filter: &str, exts: &[&str], name: &str| {
// a full path (Quick Export, Export mode) opens the dialog in its folder
let path = std::path::Path::new(name);
let mut dialog = rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.set_file_name(path.file_name().map_or(name.into(), |n| n.to_string_lossy()));
if let Some(dir) = path.parent().filter(|d| d.is_dir()) {
dialog = dialog.set_directory(dir);
}
dialog.save_file().map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_folder = Some(Box::new(|| rfd::FileDialog::new().pick_folder().map(|p| p.to_string_lossy().to_string())));
app.hooks.pick_folder_at =
Some(Box::new(|dir: &str| rfd::FileDialog::new().set_directory(dir).pick_folder().map(|p| p.to_string_lossy().into_owned())));
// Linux: the XDG portal call blocks until the user chooses, which would stop the frame
// loop (GNOME offers to force-quit the window after about five seconds, #260). Show the
// dialog on its own thread instead; the app keeps running and acts on the answer.
#[cfg(target_os = "linux")]
{
app.hooks.run_dialog = Some(Box::new(run_dialog_off_thread));
}
app.hooks.open_path = Some(Box::new(open_path));
// Help menu, About and Discord links (#642): eframe is built without its `links`
// feature, so `ctx.open_url` does nothing on the desktop. Detached, so a launcher
// that waits for the browser never freezes the UI.
app.hooks.open_url = Some(Box::new(|url: &str| open::that_detached(url).map_err(|e| e.to_string())));
app.hooks.clipboard_media = Some(Box::new(clipboard_media));
// Settings ▸ General ▸ Interface Language ▸ System Language (#218).
app.hooks.cursor_screen_position = Some(Box::new(filmcraft_platform::cursor::cursor_screen_position));
app.hooks.system_languages = Some(Box::new(|| sys_locale::get_locales().collect()));
// Settings ▸ Appearance ▸ Appearance Mode ▸ Sync with system on Linux desktops whose
// compositor reports no theme to winit (no polling: see appearance.rs).
app.hooks.system_theme = appearance::service();
app.hooks.raise_without_focus = Some(Box::new(|| {
window_raise::raise_without_focus();
}));
app.hooks.pick_open_file = Some(Box::new(|filter: &str, exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_open_project = Some(Box::new(|| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
#[cfg(target_os = "macos")]
{
let (rx, update) = native_menu::install(&app, cc.egui_ctx.clone());
app.command_inbox = Some(rx);
app.hooks.menu_changed = Some(update);
app.ui.show_menu_bar = false;
}
if let Some(port) = control_port {
let rx = control_server::start(port, cc.egui_ctx.clone());
app = if control_input_only { app.with_control_input_only(rx) } else { app.with_control(rx) };
}
Ok(Box::new(app))
})
}
/// Files copied in the file manager, else an image, on the system clipboard (Paste, #611).
fn clipboard_media() -> Option<filmcraft_ui_egui::ClipboardMedia> {
let mut clipboard = arboard::Clipboard::new().ok()?;
@ -319,6 +377,7 @@ fn open_path(path: &str, reveal: bool) -> Result<(), String> {
}
c
} else {
// Linux file managers have no select-file flag: open the parent folder instead.
let mut c = std::process::Command::new("xdg-open");
let p = std::path::Path::new(path);
c.arg(if reveal { p.parent().unwrap_or(p) } else { p });
@ -379,6 +438,60 @@ fn agent_event_loop(agent: bool) -> Option<eframe::EventLoopBuilderHook> {
None
}
/// Show `dialog` with rfd on a worker thread and answer on the returned channel (#260). Filter
/// names are translated here, on the UI thread (the interface language is per thread). A dialog
/// thread that can't start or panics drops the sender: the app takes that as cancelled.
#[cfg(target_os = "linux")]
fn run_dialog_off_thread(dialog: filmcraft_ui_egui::native_dialogs::FileDialog) -> std::sync::mpsc::Receiver<Vec<String>> {
use filmcraft_ui_egui::native_dialogs::FileDialog;
let t = |s: &str| filmcraft_ui_egui::i18n::t(s).to_string();
let media = t("Media");
let project = t("FilmCraft Project");
let dialog = match dialog {
FileDialog::OpenFile { filter, exts } => FileDialog::OpenFile { filter: t(&filter), exts },
FileDialog::SaveAs { filter, exts, name } => FileDialog::SaveAs { filter: t(&filter), exts, name },
other => other,
};
let (tx, rx) = std::sync::mpsc::channel();
let spawned = std::thread::Builder::new().name("file-dialog".into()).spawn(move || {
let shown = std::panic::catch_unwind(move || {
let path = |p: std::path::PathBuf| p.to_string_lossy().into_owned();
match dialog {
FileDialog::OpenFiles { exts } => {
rfd::FileDialog::new().add_filter(&media, &exts).pick_files().unwrap_or_default().into_iter().map(path).collect()
}
FileDialog::OpenFile { filter, exts } => rfd::FileDialog::new().add_filter(&filter, &exts).pick_file().map(path).into_iter().collect(),
FileDialog::Relink { exts, .. } => rfd::FileDialog::new().add_filter(&media, &exts).pick_file().map(path).into_iter().collect(),
FileDialog::OpenProject => rfd::FileDialog::new().add_filter(&project, &["fcproj"]).pick_file().map(path).into_iter().collect(),
FileDialog::SaveProject { name } => {
rfd::FileDialog::new().add_filter(&project, &["fcproj"]).set_file_name(name).save_file().map(path).into_iter().collect()
}
FileDialog::SaveAs { filter, exts, name } => {
rfd::FileDialog::new().add_filter(&filter, &exts).set_file_name(name).save_file().map(path).into_iter().collect()
}
FileDialog::Folder { at } => {
let d = rfd::FileDialog::new();
let d = match at {
Some(dir) if !dir.is_empty() => d.set_directory(dir),
_ => d,
};
d.pick_folder().map(path).into_iter().collect::<Vec<_>>()
}
}
});
match shown {
Ok(paths) => {
let _ = tx.send(paths);
}
Err(_) => log::warn!("the file dialog thread panicked; taken as cancelled"),
}
});
if let Err(e) = spawned {
log::warn!("could not start the file dialog thread: {e}");
}
rx
}
#[cfg(test)]
mod tests {
/// Start-up registers the hardware decoders whether or not anything is logged (no logger is

80
apps/filmcraft/src/native_menu.rs
View File

@ -4,7 +4,7 @@
use std::sync::mpsc::{Receiver, channel};
use filmcraft_ui_egui::FilmcraftApp;
use filmcraft_ui_egui::menus::{MENUS, MenuItem as Item, menu_items};
use filmcraft_ui_egui::menus::{MENUS, MenuItem as Item, NATIVE_APP_MENU, menu_items};
use muda::accelerator::Accelerator;
use muda::{Menu, MenuEvent, MenuItem, PredefinedMenuItem, Submenu};
@ -19,34 +19,53 @@ fn accel(s: &str) -> Option<Accelerator> {
mapped.parse().ok()
}
/// Updates native key equivalents when the active keyboard shortcuts change.
pub type ShortcutUpdater = Box<dyn FnMut(&[Item])>;
/// Updates native item labels, key equivalents and enablement when the menu tree changes.
pub type MenuUpdater = Box<dyn FnMut(&[Item])>;
pub fn install(app: &FilmcraftApp, ctx: egui::Context) -> (Receiver<String>, ShortcutUpdater) {
pub fn install(app: &FilmcraftApp, ctx: egui::Context) -> (Receiver<String>, MenuUpdater) {
let items = menu_items(app);
let mut native: Vec<(String, MenuItem)> = Vec::new();
// each native item with the menu entry it currently shows, so updates touch only what changed
let mut native: Vec<(Item, MenuItem)> = Vec::new();
let mut titles: Vec<(String, Submenu)> = Vec::new();
let bar = Menu::new();
let app_menu = Submenu::new("FilmCraft", true);
let [settings_label, hide_label, hide_others_label, quit_label] = NATIVE_APP_MENU;
// FilmCraft ▸ Settings ▸ <category> (Premiere's app-menu layout; General is Cmd+,)
let settings = Submenu::new(app.ui.language.tr("Settings"), true);
titles.push(("Settings".into(), settings.clone()));
let settings = Submenu::new(app.ui.language.tr(settings_label), true);
titles.push((settings_label.into(), settings.clone()));
for it in items.iter().filter(|i| i.id.starts_with("app.settings.")) {
let label = it.label.trim_end_matches('…').to_string();
let mi = MenuItem::with_id(it.id.clone(), label, true, it.shortcut.as_deref().and_then(accel));
native.push((it.id.clone(), mi.clone()));
let mi = MenuItem::with_id(it.id.clone(), label, it.enabled, it.shortcut.as_deref().and_then(accel));
native.push((it.clone(), mi.clone()));
let _ = settings.append(&mi);
}
// About and Discord repeat Help menu commands: same translated labels, updated with them
let label = |id: &str, english: &str| items.iter().find(|i| i.id == id).map_or_else(|| english.to_string(), |i| i.label.clone());
let about = MenuItem::with_id("app.about", label("app.about", "About FilmCraft"), true, None);
let discord = MenuItem::with_id("help.discord", label("help.discord", "Join the ArtCraft Discord…"), true, None);
for (id, mi) in [("app.about", &about), ("help.discord", &discord)] {
if let Some(it) = items.iter().find(|i| i.id == id) {
// recorded as built (enabled, no shortcut), so the first update applies the real state
native.push((Item { enabled: true, shortcut: None, ..it.clone() }, mi.clone()));
}
}
// the predefined items keep their native actions; our text lets them follow the language
let predefined = [
(hide_label, PredefinedMenuItem::hide(Some(app.ui.language.tr(hide_label)))),
(hide_others_label, PredefinedMenuItem::hide_others(Some(app.ui.language.tr(hide_others_label)))),
(quit_label, PredefinedMenuItem::quit(Some(app.ui.language.tr(quit_label)))),
];
let [(_, hide), (_, hide_others), (_, quit)] = &predefined;
let _ = app_menu.append_items(&[
&MenuItem::with_id("app.about", "About FilmCraft", true, None),
&MenuItem::with_id("help.discord", "Join the ArtCraft Discord…", true, None),
&about,
&discord,
&PredefinedMenuItem::separator(),
&settings,
&PredefinedMenuItem::separator(),
&PredefinedMenuItem::hide(None),
&PredefinedMenuItem::hide_others(None),
hide,
hide_others,
&PredefinedMenuItem::separator(),
&PredefinedMenuItem::quit(None),
quit,
]);
let _ = bar.append(&app_menu);
for top in MENUS {
@ -57,8 +76,8 @@ pub fn install(app: &FilmcraftApp, ctx: egui::Context) -> (Receiver<String>, Sho
// their first item appears
let mut subs: Vec<(Vec<String>, Submenu)> = Vec::new();
for it in mine {
let mi = MenuItem::with_id(it.id.clone(), &it.label, true, it.shortcut.as_deref().and_then(accel));
native.push((it.id.clone(), mi.clone()));
let mi = MenuItem::with_id(it.id.clone(), &it.label, it.enabled, it.shortcut.as_deref().and_then(accel));
native.push(((*it).clone(), mi.clone()));
let mut parent = sub.clone();
for depth in 1..it.path.len() {
let key = it.path[..=depth].to_vec();
@ -91,7 +110,8 @@ pub fn install(app: &FilmcraftApp, ctx: egui::Context) -> (Receiver<String>, Sho
ctx.request_repaint();
}
});
let update: ShortcutUpdater = Box::new(move |items: &[Item]| {
let mut shown_language = app.ui.language;
let update: MenuUpdater = Box::new(move |items: &[Item]| {
let checked = |id: &str| items.iter().any(|it| it.id == id && it.checked == Some(true));
let language = if checked("app.language.japanese") {
filmcraft_ui_egui::i18n::Language::Ja
@ -109,17 +129,33 @@ pub fn install(app: &FilmcraftApp, ctx: egui::Context) -> (Receiver<String>, Sho
filmcraft_ui_egui::i18n::Language::Ru
} else if checked("app.language.french") {
filmcraft_ui_egui::i18n::Language::Fr
} else if checked("app.language.indonesian") {
filmcraft_ui_egui::i18n::Language::Id
} else {
filmcraft_ui_egui::i18n::Language::En
};
for (title, submenu) in &titles {
submenu.set_text(language.tr(title));
if language != shown_language {
shown_language = language;
for (title, submenu) in &titles {
submenu.set_text(language.tr(title));
}
}
for (id, mi) in &native {
if let Some(it) = items.iter().find(|i| &i.id == id) {
mi.set_text(&it.label);
for (label, item) in &predefined {
item.set_text(language.tr(label));
}
for (shown, mi) in &mut native {
let Some(it) = items.iter().find(|i| i.id == shown.id) else { continue };
if it.label != shown.label {
// Settings entries drop the trailing ellipsis, as when the menu was built
mi.set_text(if it.id.starts_with("app.settings.") { it.label.trim_end_matches('…') } else { &it.label });
}
if it.shortcut != shown.shortcut {
let _ = mi.set_accelerator(it.shortcut.as_deref().and_then(accel));
}
if it.enabled != shown.enabled {
mi.set_enabled(it.enabled);
}
*shown = it.clone();
}
});
(rx, update)

3
contributors/people.toml
View File

@ -70,6 +70,9 @@ sync_github_name = false
[people.ossini]
sync_github_name = true
[people.opentechnepal]
sync_github_name = true
[people.vangel76]
real_name = "Valentin Klein"
display_name = "Valentin Klein"

3
crates/aac/README.md
View File

@ -48,6 +48,9 @@ from bitrate/quality), `tns`, `ms_stereo`, `window_shape` (sine/KBD), `adts` (pr
| 7 | 0 + PCE | C L R Ls Rs Cs LFE (PCE in the ASC; repeated in every AU when ADTS-framed) |
| 8 | 7 | C Lc Rc L R Ls Rs LFE |
The decoder also reads configurations 11 (6.1: C L R Ls Rs Cs LFE) and 12 (7.1: C L R Ls Rs Lsr Rsr
LFE) from ISO/IEC 14496-3 Amd 4; ffmpeg 9 writes 7.1 AAC as configuration 12.
`Decoder::layout()` reports the element layout (it follows a PCE for configuration 0).
## Encoder

14
crates/aac/src/config.rs
View File

@ -136,7 +136,8 @@ impl ProgramConfig {
}
}
/// Element layout of `channelConfiguration` 1..=7 (tags as conventionally assigned).
/// Element layout of `channelConfiguration` 1..=7, 11 and 12 (tags as conventionally assigned; 11 and
/// 12 from ISO/IEC 14496-3 Amd 4: 6.1 with a back centre, 7.1 with back surrounds).
pub fn config_layout(channel_config: u8) -> Option<Vec<(ElementType, u8)>> {
use ElementType::*;
Some(match channel_config {
@ -146,7 +147,8 @@ pub fn config_layout(channel_config: u8) -> Option<Vec<(ElementType, u8)>> {
4 => vec![(Sce, 0), (Cpe, 0), (Sce, 1)],
5 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1)],
6 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Lfe, 0)],
7 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)],
7 | 12 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)],
11 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)],
_ => return None,
})
}
@ -388,6 +390,14 @@ mod tests {
assert_eq!(AudioSpecificConfig::parse(&a.to_bytes()).unwrap().channels(), 6);
}
#[test]
fn amd4_channel_configurations() {
// 6.1 and 7.1 back (#922); 22.2 and 7.1 with heights stay unsupported
let channels = |c| AudioSpecificConfig::parse(&AudioSpecificConfig::lc(3, c).to_bytes()).map(|a| a.channels()).ok();
assert_eq!([7, 11, 12].map(channels), [Some(8), Some(7), Some(8)]);
assert_eq!([13, 14].map(config_layout), [None, None]);
}
#[test]
fn asc_with_pce() {
let pce = ProgramConfig {

4
crates/audio-dsp/src/biquad.rs
View File

@ -132,6 +132,10 @@ impl Biquad {
pub fn reset(&mut self) {
*self = Biquad::new();
}
/// Exact zero state, including the sign of zero, for identity-filter fast paths.
pub(crate) fn is_clear(&self) -> bool {
self.s1.to_bits() == 0 && self.s2.to_bits() == 0
}
/// Filter a buffer in place.
pub fn process(&mut self, c: &Coeffs, buf: &mut [f32]) {
for v in buf {

167
crates/audio-dsp/src/effects/eq.rs
View File

@ -110,6 +110,57 @@ impl BandBank {
pub(crate) fn process(&mut self, channels: &mut [&mut [f32]], out_gain: &mut Smoothed) {
let n = block_len(channels);
let nch = channels.len().min(self.state.len());
if n == 0 {
return;
}
if !out_gain.is_smoothing()
&& self.bands.iter().all(|b| !b.dirty && !b.freq.is_smoothing() && !b.gain.is_smoothing() && !b.q.is_smoothing() && !b.mix.is_smoothing())
{
// Keep the exact per-sample arithmetic and recursive state; only the settled
// parameter/smoother bookkeeping is hoisted out of the hot loop.
let g = out_gain.value() as f64;
for (channel, state) in channels.iter_mut().take(nch).zip(&mut self.state) {
for (b, st) in self.bands.iter().zip(state.iter_mut()) {
if b.mix.value() <= 0.0 {
st.reset();
}
}
// A settled unit transfer with zero state is exactly transparent for finite
// f32 input. Do not discard a tail or bypass a non-finite sample. Bounded
// coefficients also ensure intermediate f64 products cannot overflow.
let transparent = self.bands.iter().zip(state.iter()).all(|(b, st)| {
b.mix.value() <= 0.0
|| (st.is_clear()
&& b.coeffs.b0 == 1.0
&& b.coeffs.b1 == b.coeffs.a1
&& b.coeffs.b2 == b.coeffs.a2
&& b.coeffs.b1.abs() <= 4.0
&& b.coeffs.b2.abs() <= 4.0)
});
if transparent && channel.iter().take(n).all(|x| x.is_finite()) {
let active = self.bands.iter().any(|b| b.mix.value() > 0.0);
for sample in channel.iter_mut().take(n) {
let x = *sample as f64;
// An active identity biquad adds +0, which normalizes negative zero.
*sample = ((if active { x + 0.0 } else { x }) * g) as f32;
}
continue;
}
for sample in channel.iter_mut().take(n) {
let mut x = *sample as f64;
for (b, st) in self.bands.iter().zip(state.iter_mut()) {
let m = b.mix.value() as f64;
if m > 0.0 {
let y = st.tick(&b.coeffs, x);
x += m * (y - x);
}
}
*sample = (x * g) as f32;
}
}
self.counter = (self.counter + (n % COEFF_INTERVAL as usize) as u32) % COEFF_INTERVAL;
return;
}
for i in 0..n {
if self.counter == 0 {
self.update_coeffs(false);
@ -376,3 +427,119 @@ mod tests {
assert!((x[4799] - 0.5 * crate::db_to_gain(12.0)).abs() < 1e-4);
}
}
#[cfg(test)]
impl BandBank {
fn process_reference(&mut self, channels: &mut [&mut [f32]], out_gain: &mut Smoothed) {
let n = block_len(channels);
let nch = channels.len().min(self.state.len());
for i in 0..n {
if self.counter == 0 {
self.update_coeffs(false);
}
self.counter = (self.counter + 1) % COEFF_INTERVAL;
for b in &mut self.bands {
b.freq.tick();
b.gain.tick();
b.q.tick();
if b.freq.is_smoothing() || b.gain.is_smoothing() || b.q.is_smoothing() {
b.dirty = true;
}
}
let g = out_gain.tick();
// Tick the band mixes once per sample (shared by all channels).
let mut mixes = [0.0f32; MAX_BANDS];
for (m, b) in mixes.iter_mut().zip(self.bands.iter_mut()) {
*m = b.mix.tick();
}
for ch in 0..nch {
let mut x = channels[ch][i] as f64;
for (k, b) in self.bands.iter().enumerate() {
let m = mixes[k] as f64;
let st = &mut self.state[ch][k];
if m > 0.0 {
let y = st.tick(&b.coeffs, x);
x += m * (y - x);
} else {
st.reset();
}
}
channels[ch][i] = (x * g as f64) as f32;
}
}
}
}
#[cfg(test)]
mod throughput_tests {
use super::*;
#[test]
fn flat_eq_fast_path_preserves_samples_and_filter_tails() {
let mut fast = BandBank::new(48000.0, 2, 1);
fast.set_band(0, true, FilterType::Peaking, 1000.0, 0.0, 0.7, true);
fast.reset();
let mut reference = fast.clone();
let mut gain = Smoothed::new(1.0, 0);
let mut ref_gain = gain;
for input in
[vec![-0.0, 0.0, f32::MIN_POSITIVE, f32::from_bits(1), f32::MAX, -f32::MAX, 0.7], vec![0.0; 100], vec![f32::INFINITY, f32::NEG_INFINITY, f32::NAN]]
{
let mut a = [input.clone(), input];
let mut b = a.clone();
let [al, ar] = &mut a;
let [bl, br] = &mut b;
fast.process(&mut [al, ar], &mut gain);
reference.process_reference(&mut [bl, br], &mut ref_gain);
for (a, b) in a.iter().flatten().zip(b.iter().flatten()) {
assert_eq!(a.to_bits(), b.to_bits());
}
}
// A newly flat filter must drain its old state, not drop the tail.
fast.reset();
fast.set_band(0, true, FilterType::Peaking, 1000.0, 12.0, 0.7, true);
fast.update_coeffs(true);
fast.process(&mut [&mut [0.5], &mut [0.5]], &mut gain);
fast.set_band(0, true, FilterType::Peaking, 1000.0, 0.0, 0.7, true);
fast.update_coeffs(true);
reference = fast.clone();
let mut a = [vec![0.0; 4096], vec![0.0; 4096]];
let mut b = a.clone();
let [al, ar] = &mut a;
let [bl, br] = &mut b;
fast.process(&mut [al, ar], &mut gain);
reference.process_reference(&mut [bl, br], &mut ref_gain);
assert_eq!(a, b);
assert!(a[0].iter().any(|x| *x != 0.0));
}
#[test]
fn settled_and_automated_eq_matches_original_loop() {
let mut fast = BandBank::new(48000.0, 2, 3);
fast.reset();
let mut reference = fast.clone();
let mut gain = Smoothed::with_ms(1.0, 48000.0, 20.0);
let mut ref_gain = gain;
for block in 0..500 {
if matches!(block, 2 | 90 | 180 | 270 | 360) {
let on = block != 180;
let db = if block == 270 { 0.0 } else { 9.0 };
for bank in [&mut fast, &mut reference] {
bank.set_band(0, on, FilterType::Peaking, 1733.0, db, 0.7, false);
}
gain.set(if block == 360 { 1.0 } else { 0.8 });
ref_gain.set(gain.target());
}
let n = [0, 1, 17, 64, 513][block % 5];
let input: Vec<f32> = (0..n).map(|i| if block > 400 { 0.0 } else { ((i + block * 13) as f32 * 0.17).sin() * 0.3 }).collect();
let mut a = [input.clone(), input.clone()];
let mut b = a.clone();
let [al, ar] = &mut a;
let [bl, br] = &mut b;
fast.process(&mut [al, ar], &mut gain);
reference.process_reference(&mut [bl, br], &mut ref_gain);
for (a, b) in a.iter().flatten().zip(b.iter().flatten()) {
assert_eq!(a.to_bits(), b.to_bits(), "block {block}");
}
assert_eq!(fast.counter, reference.counter);
}
}
}

177
crates/audio-dsp/src/effects/time.rs
View File

@ -5,12 +5,26 @@ use crate::{AudioEffect, ParamSpec, ParamValues, Smoothed, Unit, block_len, flus
/// Fractional read from a circular buffer: `delay` samples behind write position `w`.
#[inline]
fn read_frac(buf: &[f32], w: usize, delay: f32) -> f32 {
let len = buf.len();
read_prepared(buf, w, prepare_delay(buf.len(), delay))
}
#[inline]
fn prepare_delay(len: usize, delay: f32) -> (usize, f32) {
let d = delay.clamp(1.0, (len - 2) as f32);
let di = d.floor();
let frac = d - di;
let i0 = (w + len - di as usize) % len;
let i1 = (i0 + len - 1) % len;
// The delay is bounded by this ring's length, so wrapping needs no integer division.
// Keep the fallback for very large rings whose f32 length rounds upward.
let distance = di as usize;
let distance = if distance <= len { distance } else { distance % len };
(distance, frac)
}
#[inline]
fn read_prepared(buf: &[f32], w: usize, (distance, frac): (usize, f32)) -> f32 {
let len = buf.len();
let i0 = if w >= distance { w - distance } else { len - (distance - w) };
let i1 = if i0 == 0 { len - 1 } else { i0 - 1 };
buf[i0] + (buf[i1] - buf[i0]) * frac
}
@ -103,7 +117,7 @@ impl AudioEffect for Delay {
buf[self.w] = flush32(x + fb * yd);
channels[ch][i] = x + mix * (yd - x);
}
self.w = (self.w + 1) % len;
self.w = if self.w + 1 == len { 0 } else { self.w + 1 };
}
}
}
@ -132,6 +146,7 @@ pub struct Reverb {
len: [Smoothed; FDN],
lp: [f32; FDN],
gains: [f32; FDN],
gain_key: ([f32; FDN], f32),
damp: Smoothed,
rt60: Smoothed,
mix: Smoothed,
@ -162,6 +177,7 @@ impl Reverb {
len: [sm(1.0); FDN],
lp: [0.0; FDN],
gains: [0.0; FDN],
gain_key: ([0.0; FDN], 0.0),
damp: sm(0.0),
rt60: sm(1.0),
mix: sm(0.0),
@ -195,6 +211,11 @@ impl Reverb {
fn update_gains(&mut self) {
let rt = self.rt60.value().max(0.01);
let lengths = self.len.map(|l| l.value());
if self.gain_key == (lengths, rt) {
return;
}
self.gain_key = (lengths, rt);
for k in 0..FDN {
// Attenuation per pass so the loop decays 60 dB in rt seconds.
self.gains[k] = 10f32.powf(-3.0 * self.len[k].value() / (rt * self.sr));
@ -222,13 +243,20 @@ impl AudioEffect for Reverb {
let line_len = self.lines[0].len();
let pre_len = self.pre.len();
let hh = 2.0 / FDN as f32;
// During a settled block these positions cannot change. Preserve the original path
// while smoothing, and preserve the 32-sample gain update grid in both cases.
let fixed = self.len.iter().all(|l| !l.is_smoothing());
let delays = self.len.map(|l| prepare_delay(line_len, l.value()));
let predelay = (!self.pre_len.is_smoothing()).then(|| prepare_delay(pre_len, self.pre_len.value()));
for i in 0..n {
if self.counter == 0 {
self.update_gains();
}
self.counter = (self.counter + 1) % 32;
for l in &mut self.len {
l.tick();
if !fixed {
for l in &mut self.len {
l.tick();
}
}
self.rt60.tick();
let damp = self.damp.tick();
@ -241,13 +269,16 @@ impl AudioEffect for Reverb {
}
mono /= nch as f32;
self.pre[self.pre_w] = mono;
let input = read_frac(&self.pre, self.pre_w, pd);
self.pre_w = (self.pre_w + 1) % pre_len;
let input = match predelay {
Some(delay) => read_prepared(&self.pre, self.pre_w, delay),
None => read_frac(&self.pre, self.pre_w, pd),
};
self.pre_w = if self.pre_w + 1 == pre_len { 0 } else { self.pre_w + 1 };
let mut v = [0.0f32; FDN];
let mut sum = 0.0;
for k in 0..FDN {
let d = read_frac(&self.lines[k], self.w, self.len[k].value());
let d = if fixed { read_prepared(&self.lines[k], self.w, delays[k]) } else { read_frac(&self.lines[k], self.w, self.len[k].value()) };
self.lp[k] = flush32(d + (self.lp[k] - d) * damp);
v[k] = self.lp[k] * self.gains[k];
sum += v[k];
@ -259,7 +290,7 @@ impl AudioEffect for Reverb {
wl += SIGN_L[k] * v[k];
wr += SIGN_R[k] * v[k];
}
self.w = (self.w + 1) % line_len;
self.w = if self.w + 1 == line_len { 0 } else { self.w + 1 };
let (wl, wr) = (wl * 0.35, wr * 0.35);
for (c, ch) in channels.iter_mut().enumerate() {
let x = ch[i];
@ -370,3 +401,129 @@ mod tests {
}
}
}
#[cfg(test)]
fn read_frac_reference(buf: &[f32], w: usize, delay: f32) -> f32 {
let len = buf.len();
let d = delay.clamp(1.0, (len - 2) as f32);
let di = d.floor();
let frac = d - di;
let i0 = (w + len - di as usize) % len;
let i1 = (i0 + len - 1) % len;
buf[i0] + (buf[i1] - buf[i0]) * frac
}
#[cfg(test)]
impl Reverb {
fn update_gains_reference(&mut self) {
let rt = self.rt60.value().max(0.01);
for k in 0..FDN {
// Attenuation per pass so the loop decays 60 dB in rt seconds.
self.gains[k] = 10f32.powf(-3.0 * self.len[k].value() / (rt * self.sr));
}
}
fn process_reference(&mut self, channels: &mut [&mut [f32]]) {
let n = block_len(channels);
let nch = channels.len();
if nch == 0 {
return;
}
let line_len = self.lines[0].len();
let pre_len = self.pre.len();
let hh = 2.0 / FDN as f32;
for i in 0..n {
if self.counter == 0 {
self.update_gains_reference();
}
self.counter = (self.counter + 1) % 32;
for l in &mut self.len {
l.tick();
}
self.rt60.tick();
let damp = self.damp.tick();
let mix = self.mix.tick();
let pd = self.pre_len.tick();
let mut mono = 0.0;
for ch in channels.iter() {
mono += ch[i];
}
mono /= nch as f32;
self.pre[self.pre_w] = mono;
let input = read_frac_reference(&self.pre, self.pre_w, pd);
self.pre_w = (self.pre_w + 1) % pre_len;
let mut v = [0.0f32; FDN];
let mut sum = 0.0;
for k in 0..FDN {
let d = read_frac_reference(&self.lines[k], self.w, self.len[k].value());
self.lp[k] = flush32(d + (self.lp[k] - d) * damp);
v[k] = self.lp[k] * self.gains[k];
sum += v[k];
}
let (mut wl, mut wr) = (0.0, 0.0);
for k in 0..FDN {
let fbk = v[k] - hh * sum;
self.lines[k][self.w] = flush32(fbk + input * 0.5);
wl += SIGN_L[k] * v[k];
wr += SIGN_R[k] * v[k];
}
self.w = (self.w + 1) % line_len;
let (wl, wr) = (wl * 0.35, wr * 0.35);
for (c, ch) in channels.iter_mut().enumerate() {
let x = ch[i];
let wet = if nch == 1 {
0.5 * (wl + wr)
} else if c % 2 == 0 {
wl
} else {
wr
};
ch[i] = x + mix * (wet - x);
}
}
}
}
#[cfg(test)]
mod throughput_tests {
use super::*;
#[test]
fn circular_reads_match_modulo_reference() {
for len in [3, 4, 17, 257, 14078] {
let buf: Vec<f32> = (0..len).map(|i| (i as f32 * 0.17).sin()).collect();
for w in 0..len {
for delay in [-10.0, 0.0, 1.0, 1.25, 2.75, len as f32 - 2.0, len as f32, f32::INFINITY] {
assert_eq!(read_frac(&buf, w, delay).to_bits(), read_frac_reference(&buf, w, delay).to_bits());
}
}
}
}
#[test]
fn reverb_automation_and_tail_match_original_loop() {
let mut fast = Reverb::new(48000.0, 2);
let mut reference = Reverb::new(48000.0, 2);
for block in 0..800 {
if matches!(block, 1 | 80 | 200 | 320 | 440) {
for fx in [&mut fast, &mut reference] {
fx.set_param("size", (block % 101) as f32);
fx.set_param("decay", 0.1 + (block % 20) as f32);
fx.set_param("predelay", (block % 200) as f32);
}
}
if block == 600 {
fast.reset();
reference.reset();
}
let n = [0, 1, 17, 64, 513][block % 5];
let input: Vec<f32> = (0..n).map(|i| if block > 450 { 0.0 } else { ((i + block * 13) as f32 * 0.17).sin() * 0.3 }).collect();
let mut a = [input.clone(), input];
let mut b = a.clone();
let [al, ar] = &mut a;
let [bl, br] = &mut b;
fast.process(&mut [al, ar]);
reference.process_reference(&mut [bl, br]);
for (a, b) in a.iter().flatten().zip(b.iter().flatten()) {
assert_eq!(a.to_bits(), b.to_bits(), "block {block}");
}
}
}
}

30
crates/automation/src/bridge.rs
View File

@ -47,14 +47,14 @@ impl BridgeClient {
}
let mut start = self.call("engine.execute", json!({"command": id, "params": params})).await?;
let Some(job) = start.get("job").and_then(Value::as_u64) else { return Ok(start) };
let result = loop {
let (result, offline) = loop {
tokio::time::sleep(JOB_POLL).await;
let jobs = self.call("engine.execute", json!({"command": "jobs.list", "params": {}})).await?;
let Some(j) = jobs.as_array().and_then(|a| a.iter().find(|j| j["id"].as_u64() == Some(job))).cloned() else {
break Value::Null;
break (Value::Null, None);
};
if j["finished"].as_bool() == Some(true) {
break j["result"].clone();
break (j["result"].clone(), j.get("offlineMedia").cloned());
}
};
if let Some(e) = result.get("error").and_then(Value::as_str) {
@ -62,6 +62,9 @@ impl BridgeClient {
}
if let Some(o) = start.as_object_mut() {
o.insert("result".into(), result);
if let Some(off) = offline {
o.insert("offlineMedia".into(), off);
}
}
Ok(start)
}
@ -121,6 +124,11 @@ mod tests {
/// A stand-in for the app's control server: the export starts job 7, which `jobs.list` reports
/// running twice and then finished with `result`. Every request is recorded.
async fn fake_app(result: Value) -> (String, Arc<Mutex<Vec<Value>>>) {
fake_app_with(result, None).await
}
/// [`fake_app`] whose finished job also lists `offline` as its `offlineMedia`.
async fn fake_app_with(result: Value, offline: Option<Value>) -> (String, Arc<Mutex<Vec<Value>>>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap().to_string();
let seen = Arc::new(Mutex::new(Vec::new()));
@ -138,7 +146,11 @@ mod tests {
Some("jobs.list") => {
polls += 1;
let finished = polls >= 3;
json!([{"id": 7, "finished": finished, "result": if finished { result.clone() } else { Value::Null }}])
let mut j = json!({"id": 7, "finished": finished, "result": if finished { result.clone() } else { Value::Null }});
if let Some(o) = offline.clone().filter(|_| finished) {
j["offlineMedia"] = o;
}
json!([j])
}
_ => json!(null),
};
@ -165,6 +177,16 @@ mod tests {
assert_eq!(seen.iter().filter(|q| q["params"]["command"] == "jobs.list").count(), 3);
}
/// The offline media a polled export drew comes back as in a blocking export's reply (#1054).
#[tokio::test]
async fn a_polled_export_reports_its_offline_media() {
let off = json!([{"item": 8, "status": "missing"}]);
let (addr, _) = fake_app_with(json!({"frames": 3}), Some(off.clone())).await;
let b = BridgeClient::new(addr).unwrap();
let r = b.execute("file.exportMedia", json!({"path": "/tmp/out.mov", "wait": true})).await.unwrap();
assert_eq!(r["offlineMedia"], off, "{r}");
}
#[tokio::test]
async fn a_failed_blocking_export_is_an_error() {
let (addr, _) = fake_app(json!({"error": "encode: disk full"})).await;

4
crates/automation/src/long_job.rs
View File

@ -72,6 +72,7 @@ impl FilmcraftMcp {
rep.report(done, total, j["status"].as_str().unwrap_or_default()).await;
}
if finished {
let offline = state.as_ref().and_then(|j| j.get("offlineMedia").cloned());
let result = state.map(|j| j["result"].clone()).unwrap_or(Value::Null);
if let Some(e) = result.get("error").and_then(Value::as_str) {
return CallToolResult::error(vec![Content::text(format!("export failed: {e}"))]);
@ -79,6 +80,9 @@ impl FilmcraftMcp {
let mut out = start;
if let Some(o) = out.as_object_mut() {
o.insert("result".into(), result);
if let Some(off) = offline {
o.insert("offlineMedia".into(), off);
}
}
return CallToolResult::success(vec![Content::text(serde_json::to_string_pretty(&out).unwrap_or_default())]);
}

9
crates/av1/README.md
View File

@ -70,12 +70,15 @@ with libdav1d: **bit-exact** on all 6 fixtures.
wasm32, or without the default `threads` feature, everything runs on the calling thread). Headers are parsed in order; each frame is
then decoded by a frame worker once the reference frames it reads are finished, so independent
frames (hierarchical GOP layers) decode concurrently; tiles of a frame decode in parallel into
private buffers that are merged afterwards. Results never depend on scheduling (the oracle tests
compare 1 thread, many threads and libdav1d). With frame threads, pictures may be returned by a
private buffers that are merged afterwards; the in-loop filters (deblocking, CDEF, loop
restoration) each run in parallel bands of 64 rows (`postfilter.rs` explains why the bands are
independent). Results never depend on scheduling (the oracle tests compare 1 thread, many threads
and libdav1d). With frame threads, pictures may be returned by a
later `decode_pts` call or by `flush`; each carries the `pts` of the temporal unit that showed it.
`tests/perf.rs` (ignored) prints single-threaded fps (thread CPU time), per-stage ms/frame
(`Decoder::stats()`) and all-core fps for 1080p fixtures. The hot loops (sub-pixel filters,
(`Decoder::stats()`) and all-core fps with its per-stage ms/frame for 1080p fixtures (key frames,
GOPs, tiles, low delay) and a single-tile 2160p GOP; `AV1_PERF_ONLY=a,b` picks fixtures. The hot loops (sub-pixel filters,
CDEF, inverse transforms, deblocking, entropy decoding) are written for auto-vectorisation.
Measured 2026-10-01 on a 14-core Apple Silicon Mac (release build, SVT-AV1 preset 8 1080p

53
crates/av1/src/cdef.rs
View File

@ -1,35 +1,45 @@
//! CDEF (7.15).
use crate::frame::{FrameBuf, Plane};
use crate::frame::{FrameBuf, RowBand};
use crate::par::Pool;
use crate::postfilter::bands;
use crate::spec_tables::*;
use crate::state::FrameState;
/// Run CDEF on `fs.cur`; returns the CdefFrame, or None when no block is filtered (CdefFrame
/// equals CurrFrame).
pub(crate) fn apply(fs: &FrameState) -> Option<FrameBuf> {
/// equals CurrFrame). Bands of 64 luma rows (whole 8x8 blocks) are filtered in parallel: every
/// block reads only CurrFrame and writes only its own samples.
pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option<FrameBuf> {
let fh = &fs.fh;
if fh.coded_lossless || fh.allow_intrabc || !fs.seq.enable_cdef {
return None;
}
let (mi_rows, mi_cols) = (fh.mi_rows as usize, fh.mi_cols as usize);
let mut out = fs.cur.clone();
let mut any = false;
let mut r = 0;
while r < mi_rows {
let mut c = 0;
while c < mi_cols {
let idx = fs.cdef_idx(r & !15, c & !15);
if idx >= 0 && cdef_block(fs, &mut out, r, c, idx as usize) {
any = true;
let ranges = bands(pool, mi_rows * 4);
let mut planes = out.row_bands(&ranges);
let any = pool.map_mut(&mut planes, |b, band| {
let (start, end) = ranges[b];
let mut any = false;
let mut r = start / 4;
while r < mi_rows.min(end / 4) {
let mut c = 0;
while c < mi_cols {
let idx = fs.cdef_idx(r & !15, c & !15);
if idx >= 0 && cdef_block(fs, band, r, c, idx as usize) {
any = true;
}
c += 2;
}
c += 2;
r += 2;
}
r += 2;
}
any.then_some(out)
any
});
drop(planes);
any.contains(&true).then_some(out)
}
fn cdef_block(fs: &FrameState, out: &mut FrameBuf, r: usize, c: usize, idx: usize) -> bool {
fn cdef_block(fs: &FrameState, out: &mut [RowBand; 3], r: usize, c: usize, idx: usize) -> bool {
let mi = &fs.mi;
let sk = |rr: usize, cc: usize| -> bool { if rr < mi.rows && cc < mi.cols { mi.skip[mi.idx(rr, cc)] } else { true } };
let skip = sk(r, c) && sk(r + 1, c) && sk(r, c + 1) && sk(r + 1, c + 1);
@ -125,7 +135,7 @@ fn constrain(diff: i32, threshold: i32, adj: u32) -> i32 {
}
#[allow(clippy::too_many_arguments)]
fn filter(fs: &FrameState, out: &mut FrameBuf, plane: usize, r: usize, c: usize, pri: i32, sec: i32, damping: i32, dir: usize) {
fn filter(fs: &FrameState, out: &mut [RowBand; 3], plane: usize, r: usize, c: usize, pri: i32, sec: i32, damping: i32, dir: usize) {
if pri == 0 && sec == 0 {
// Every tap contributes 0 and the clamp to [min, max] keeps x: CdefFrame = CurrFrame.
return;
@ -169,7 +179,7 @@ fn filter(fs: &FrameState, out: &mut FrameBuf, plane: usize, r: usize, c: usize,
let sec_taps = [CDEF_SEC_TAPS[pri_tap][0] as i32, CDEF_SEC_TAPS[pri_tap][1] as i32];
let adj = |t: i32| if t == 0 { 0 } else { (damping - (31 - (t as u32).leading_zeros()) as i32).max(0) as u32 };
let (pri_adj, sec_adj) = (adj(pri), adj(sec));
let dst = &mut out.planes[plane];
let dst = &mut out[plane];
if all_avail {
// Every tap is available: whole rows at a time (constrain() with threshold 0 is 0).
let mut taps = [(0isize, 0i32, 0i32, 0u32); 12];
@ -230,7 +240,12 @@ fn filter(fs: &FrameState, out: &mut FrameBuf, plane: usize, r: usize, c: usize,
/// CDEF of a block whose taps are all available, `W` samples per row. Samples (at most 12
/// bits), differences and the weighted tap sum all fit in i16, which doubles the SIMD width.
#[inline(always)]
fn filter_rows<const W: usize>(buf: &[i32; PAD_STRIDE * PAD_STRIDE], taps: &[(isize, i32, i32, u32); 12], dst: &mut Plane, (y0, x0, h): (usize, usize, usize)) {
fn filter_rows<const W: usize>(
buf: &[i32; PAD_STRIDE * PAD_STRIDE],
taps: &[(isize, i32, i32, u32); 12],
dst: &mut RowBand,
(y0, x0, h): (usize, usize, usize),
) {
let mut b16 = [0i16; PAD_STRIDE * PAD_STRIDE];
for (d, &s) in b16.iter_mut().zip(buf.iter()) {
*d = s as i16;

89
crates/av1/src/frame.rs
View File

@ -106,6 +106,48 @@ impl Plane {
dst.row_from_mut(self.oy + y, self.ox)[..self.stride].copy_from_slice(s);
}
}
/// Disjoint mutable bands of whole rows, for filtering in parallel: one per `[start, end)`
/// range of rows (relative to `oy`; ascending and not overlapping, else the later band is
/// cut to what is left). Ranges are clipped to the stored rows.
pub fn row_bands(&mut self, ranges: &[(usize, usize)]) -> Vec<RowBand<'_>> {
let (stride, oy) = (self.stride.max(1), self.oy);
let mut out = Vec::with_capacity(ranges.len());
let mut rest = self.data.as_mut_slice();
// First row of `rest`.
let mut y = 0usize;
for &(start, end) in ranges {
let skip = start.saturating_sub(y).saturating_mul(stride).min(rest.len());
rest = std::mem::take(&mut rest).split_at_mut(skip).1;
y = y.saturating_add(skip / stride);
let len = end.saturating_sub(y).saturating_mul(stride).min(rest.len());
let (band, tail) = std::mem::take(&mut rest).split_at_mut(len);
out.push(RowBand { data: band, stride, y0: oy.saturating_add(y) });
y = y.saturating_add(len / stride);
rest = tail;
}
out
}
}
/// Mutable whole rows [y0, y0 + data.len() / stride) of a plane, addressed with the plane's
/// coordinates (see [`Plane::row_bands`]).
pub struct RowBand<'a> {
pub data: &'a mut [u16],
pub stride: usize,
pub y0: usize,
}
impl RowBand<'_> {
#[inline(always)]
pub fn set(&mut self, x: usize, y: usize, v: u16) {
self.data[(y - self.y0) * self.stride + x] = v;
}
#[inline(always)]
pub fn row_mut(&mut self, y: usize) -> &mut [u16] {
let o = (y - self.y0) * self.stride;
&mut self.data[o..o + self.stride]
}
}
/// A frame's three planes, sized to the macroblock-aligned coded area plus margins.
@ -158,6 +200,16 @@ impl FrameBuf {
}
}
/// [`Plane::row_bands`] of all three planes for ranges of luma rows (chroma rows scaled
/// down): element `i` holds the bands of range `i`.
pub fn row_bands(&mut self, luma: &[(usize, usize)]) -> Vec<[RowBand<'_>; 3]> {
let ssy = self.subsampling_y;
let chroma: Vec<(usize, usize)> = luma.iter().map(|&(s, e)| (s >> ssy, e >> ssy)).collect();
let [p0, p1, p2] = &mut self.planes;
let (b1, b2) = (p1.row_bands(&chroma), p2.row_bands(&chroma));
p0.row_bands(luma).into_iter().zip(b1).zip(b2).map(|((y, u), v)| [y, u, v]).collect()
}
pub fn plane_width(&self, plane: usize) -> usize {
if plane == 0 { self.width } else { (self.width + self.subsampling_x) >> self.subsampling_x }
}
@ -285,3 +337,40 @@ impl MiInfo {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rows(b: &RowBand) -> (usize, usize) {
(b.y0, b.data.len() / b.stride)
}
#[test]
fn row_bands_split_whole_rows() {
let mut p = Plane::new(8, 20);
for (i, v) in p.data.iter_mut().enumerate() {
*v = (i / 8) as u16;
}
let bands = p.row_bands(&[(0, 8), (8, 16), (16, usize::MAX)]);
assert_eq!(bands.iter().map(rows).collect::<Vec<_>>(), [(0, 8), (8, 8), (16, 4)]);
// Each band's row y holds the samples of plane row y.
for mut b in bands {
let y = b.y0 + 1;
assert!(b.row_mut(y).iter().all(|&v| v as usize == y));
}
// Gaps between bands (the loop filter's even / odd turns).
let bands = p.row_bands(&[(0, 4), (10, 14), (18, 30)]);
assert_eq!(bands.iter().map(rows).collect::<Vec<_>>(), [(0, 4), (10, 4), (18, 2)]);
}
#[test]
fn row_bands_never_overlap_or_panic() {
let mut p = Plane::new(4, 10);
// Overlapping, descending and out-of-range ranges are cut to what is left.
let bands = p.row_bands(&[(2, 6), (4, 8), (1, 3), (9, 2), (50, 60), (usize::MAX, usize::MAX)]);
assert_eq!(bands.iter().map(rows).collect::<Vec<_>>(), [(2, 4), (6, 2), (8, 0), (9, 0), (10, 0), (10, 0)]);
let mut empty = Plane::default();
assert!(empty.row_bands(&[(0, 64), (64, usize::MAX)]).iter().all(|b| b.data.is_empty()));
}
}

165
crates/av1/src/postfilter.rs
View File

@ -1,20 +1,33 @@
//! In-loop post filters run by decode_frame_wrapup: loop filter (7.14), CDEF (7.15),
//! super-resolution upscaling (7.16) and loop restoration (7.17).
use crate::frame::RowBand;
use crate::spec_tables::*;
use crate::state::FrameState;
use crate::stats::{DecodeStats, Stage, Timer};
/// The post-filters run on the frame worker; `_pool` is for row-parallel filtering (not yet used).
/// `draft`: only super-resolution (which sets the output size) runs.
pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, _pool: &crate::par::Pool, draft: bool) {
/// Luma rows per band of the row-parallel filters: whole 64x64 CDEF filter blocks, and no
/// transform block crosses a band edge.
const BAND_ROWS: usize = 64;
/// The [start, end) luma rows of the bands covering `rows`: [`BAND_ROWS`] each (the last one
/// runs to the end of the plane), or one band for everything when `pool` is sequential.
pub(crate) fn bands(pool: &crate::par::Pool, rows: usize) -> Vec<(usize, usize)> {
let n = if pool.is_parallel() { rows.div_ceil(BAND_ROWS).max(1) } else { 1 };
(0..n).map(|b| (b * BAND_ROWS, if b + 1 == n { usize::MAX } else { (b + 1) * BAND_ROWS })).collect()
}
/// The post-filters run on the frame worker, each in row bands on `pool` (the result does not
/// depend on how the bands are scheduled). `draft`: only super-resolution (which sets the
/// output size) runs.
pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate::par::Pool, draft: bool) {
let mut t = Timer::start();
let lvl = fs.fh.lf.level;
if (lvl[0] != 0 || lvl[1] != 0) && !draft {
loop_filter(fs);
loop_filter(fs, pool);
}
stats.add(Stage::LoopFilter, t.lap());
let cdef = if draft { None } else { crate::cdef::apply(fs) };
let cdef = if draft { None } else { crate::cdef::apply(fs, pool) };
stats.add(Stage::Cdef, t.lap());
let (up_cur, up_cdef) = if fs.fh.use_superres {
let c = crate::restoration::upscale(fs, &fs.cur);
@ -25,7 +38,7 @@ pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, _pool: &crate:
};
stats.add(Stage::Superres, t.lap());
fs.cur = if fs.fh.lr.uses_lr && !draft {
crate::restoration::loop_restoration(fs, &up_cur, up_cdef.as_ref().unwrap_or(&up_cur))
crate::restoration::loop_restoration(fs, &up_cur, up_cdef.as_ref().unwrap_or(&up_cur), pool)
} else {
up_cdef.unwrap_or(up_cur)
};
@ -35,28 +48,102 @@ pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, _pool: &crate:
// ---------------------------------------------------------------------------------------------
// Loop filter (7.14)
fn loop_filter(fs: &mut FrameState) {
let (mi_rows, mi_cols) = (fs.fh.mi_rows as usize, fs.fh.mi_cols as usize);
for plane in 0..fs.num_planes {
if plane == 0 || fs.fh.lf.level[1 + plane] != 0 {
for pass in 0..2 {
let row_step = if plane == 0 { 1 } else { 1 << fs.ssy };
let col_step = if plane == 0 { 1 } else { 1 << fs.ssx };
let mut row = 0;
while row < mi_rows {
let mut col = 0;
while col < mi_cols {
edge(fs, plane, pass, row, col);
col += col_step;
}
row += row_step;
/// Luma rows above a band that its horizontal edges read or write (p6 .. p0 of the widest
/// filter, rounded up to whole 4x4 rows); chroma: `LF_MARGIN >> subsampling_y`.
const LF_MARGIN: usize = 8;
/// The loop filter in row bands, all of a frame's in one parallel pass (each join costs a
/// worker wake-up, which is slow on a busy machine).
///
/// Vertical edges (pass 0) only touch the 4 rows of their own edge. A horizontal edge (pass 1)
/// reads and writes at most `filter_size / 2` samples on each side, never more than the
/// transform blocks on either side are high, so no two edges touch the same sample and
/// any order gives the spec's raster-order result. Transform blocks never cross a band edge, so
/// the only edges reaching outside a band are those on its top row, whose p side lies in the
/// band above. Each band therefore also filters the vertical edges of a copy of the
/// `LF_MARGIN` rows above it (the same work as the band above does on them), runs its top
/// horizontal edges against that copy, and afterwards the samples those edges changed are
/// written back; the band above never touches them.
fn loop_filter(fs: &mut FrameState, pool: &crate::par::Pool) {
let luma = bands(pool, fs.fh.mi_rows as usize * 4);
let planes: Vec<usize> = (0..fs.num_planes).filter(|&p| p == 0 || fs.fh.lf.level[1 + p] != 0).collect();
let mut cur = std::mem::take(&mut fs.cur);
let f: &FrameState = fs;
// Unfiltered copies of the rows above each band but the first.
let margins: Vec<[Vec<u16>; 3]> = luma
.iter()
.map(|&(start, _)| {
std::array::from_fn(|p| {
let sub_y = if p == 0 { 0 } else { f.ssy };
let pl = &cur.planes[p];
let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y);
let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride);
if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() }
})
})
.collect();
let mut bands: Vec<([RowBand; 3], [Vec<u16>; 3])> = cur.row_bands(&luma).into_iter().zip(margins).collect();
let changed = pool.map_mut(&mut bands, |b, (band, margin)| {
let mut changed: [Vec<(usize, u16)>; 3] = Default::default();
for &p in &planes {
changed[p] = filter_band(f, p, luma[b], &mut band[p], &mut margin[p]);
}
changed
});
drop(bands);
for (&(start, _), ch) in luma.iter().zip(&changed) {
for (p, list) in ch.iter().enumerate() {
let sub_y = if p == 0 { 0 } else { f.ssy };
let pl = &mut cur.planes[p];
let first = (start.saturating_sub(LF_MARGIN) >> sub_y) * pl.stride;
for &(i, v) in list {
if let Some(d) = pl.data.get_mut(first + i) {
*d = v;
}
}
}
}
fs.cur = cur;
}
fn edge(fs: &mut FrameState, plane: usize, pass: usize, row: usize, col: usize) {
/// Both loop filter passes over the band of `plane` holding luma rows [start, end); `margin` is
/// an unfiltered copy of the rows above the band (empty for the first band). Returns the samples
/// of `margin` that the band's top horizontal edges changed, as (index, value).
fn filter_band(fs: &FrameState, plane: usize, (start, end): (usize, usize), band: &mut RowBand, margin: &mut [u16]) -> Vec<(usize, u16)> {
let mi_rows = fs.fh.mi_rows as usize;
let (r0, r1) = (start / 4, mi_rows.min(end / 4));
let sub_y = if plane == 0 { 0 } else { fs.ssy };
let edges = |band: &mut RowBand, mut above: Option<&mut RowBand>, pass: usize, rows: std::ops::Range<usize>| {
let row_step = if plane == 0 { 1 } else { 1 << fs.ssy };
let col_step = if plane == 0 { 1 } else { 1 << fs.ssx };
let mi_cols = fs.fh.mi_cols as usize;
let mut row = rows.start;
while row < rows.end {
let mut col = 0;
while col < mi_cols {
edge(fs, band, above.as_deref_mut(), plane, pass, row, col);
col += col_step;
}
row += row_step;
}
};
edges(band, None, 0, r0..r1);
if margin.is_empty() {
edges(band, None, 1, r0..r1);
return Vec::new();
}
let stride = band.stride;
let y0 = start.saturating_sub(LF_MARGIN) >> sub_y;
let mut above = RowBand { data: margin, stride, y0 };
edges(&mut above, None, 0, r0.saturating_sub(LF_MARGIN / 4)..r0);
let before = above.data.to_vec();
edges(band, Some(&mut above), 1, r0..r1);
above.data.iter().zip(&before).enumerate().filter(|(_, (a, b))| a != b).map(|(i, (&a, _))| (i, a)).collect()
}
/// One loop filter edge (7.14.2) at mode info position (row, col): 4 lines of samples across
/// it. `above` holds the rows above `band` for horizontal edges on the band's top row.
fn edge(fs: &FrameState, band: &mut RowBand, above: Option<&mut RowBand>, plane: usize, pass: usize, row: usize, col: usize) {
let (sub_x, sub_y) = if plane == 0 { (0, 0) } else { (fs.ssx, fs.ssy) };
let (dx, dy) = if pass == 0 { (1i32, 0i32) } else { (0, 1) };
let x = col * 4;
@ -102,16 +189,40 @@ fn edge(fs: &mut FrameState, plane: usize, pass: usize, row: usize, col: usize)
return;
}
let t = LfThresh::new(limit, blimit, thresh, fs.bit_depth);
let pl = &mut fs.cur.planes[plane];
let stride = pl.stride;
let (step, along) = if pass == 0 { (1, stride) } else { (stride, 1) };
let base = yp * stride + xp;
let chroma = plane > 0;
let stride = band.stride;
if let Some(above) = above.filter(|_| pass == 1 && yp == band.y0) {
// The p side is in the rows above the band: filter each line in a copy that spans both.
let (n_above, n_band) = (above.data.len() / stride.max(1), band.data.len() / stride.max(1));
for k in 0..4 {
let x = xp + k;
let mut line = [0u16; 2 * LF_LINE];
for j in 1..=n_above.min(LF_LINE) {
line[LF_LINE - j] = above.data[(n_above - j) * stride + x];
}
for j in 0..n_band.min(LF_LINE) {
line[LF_LINE + j] = band.data[j * stride + x];
}
sample_filter(&mut line, LF_LINE, 1, chroma, filter_size, t);
for j in 1..=n_above.min(LF_LINE) {
above.data[(n_above - j) * stride + x] = line[LF_LINE - j];
}
for j in 0..n_band.min(LF_LINE) {
band.data[j * stride + x] = line[LF_LINE + j];
}
}
return;
}
let (step, along) = if pass == 0 { (1, stride) } else { (stride, 1) };
let base = (yp - band.y0) * stride + xp;
for k in 0..4 {
sample_filter(&mut pl.data, base + k * along, step, chroma, filter_size, t);
sample_filter(band.data, base + k * along, step, chroma, filter_size, t);
}
}
/// Samples on each side of an edge that the sample filter can read (at most p6 .. q6).
const LF_LINE: usize = 8;
/// Adaptive filter strength (7.14.4): (lvl, limit, blimit, thresh).
fn strength(fs: &FrameState, row: usize, col: usize, plane: usize, pass: usize) -> (i32, i32, i32, i32) {
let mi = &fs.mi;

94
crates/av1/src/restoration.rs
View File

@ -1,6 +1,8 @@
//! Super-resolution upscaling (7.16) and loop restoration (7.17).
use crate::frame::{FrameBuf, Plane};
use crate::frame::{FrameBuf, Plane, RowBand};
use crate::par::Pool;
use crate::postfilter::bands;
use crate::spec_tables::*;
use crate::state::FrameState;
@ -70,56 +72,66 @@ impl LrCtx<'_> {
}
/// Loop restoration (7.17): returns LrFrame given UpscaledCurrFrame and UpscaledCdefFrame.
pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf) -> FrameBuf {
/// Bands of 64 luma rows are restored in parallel: every unit reads only the two input frames
/// and writes only its own samples.
pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, pool: &Pool) -> FrameBuf {
let fh = &fs.fh;
let mut lr = cdef.clone();
let up_w = fh.upscaled_width as i32;
let fr_h = fh.frame_height as i32;
let bd = fs.bit_depth;
for plane in 0..fs.num_planes {
let rtype_frame = fh.lr.frame_restoration_type[plane];
if rtype_frame == RESTORE_NONE as u8 {
continue;
}
let (sub_x, sub_y) = if plane > 0 { (fs.ssx as i32, fs.ssy as i32) } else { (0, 0) };
let unit_size = fh.lr.loop_restoration_size[plane] as i32;
let unit_rows = fs.lr_unit_rows[plane] as i32;
let unit_cols = fs.lr_unit_cols[plane] as i32;
let plane_end_x = ((up_w + sub_x) >> sub_x) - 1;
let plane_end_y = ((fr_h + sub_y) >> sub_y) - 1;
let mut row = 0;
while row * 4 < fr_h {
let luma_y = row * 4;
let stripe_num = (luma_y + 8) / 64;
let stripe_start_y = (-8 + stripe_num * 64) >> sub_y;
let stripe_end_y = stripe_start_y + (64 >> sub_y) - 1;
let ctx = LrCtx { cur: &cur.planes[plane], cdef: &cdef.planes[plane], plane_end_x, plane_end_y, stripe_start_y, stripe_end_y };
let unit_row = (unit_rows - 1).min(((row * 4 + 8) >> sub_y) / unit_size);
let mut col = 0;
while col * 4 < up_w {
let unit_col = (unit_cols - 1).min(((col * 4) >> sub_x) / unit_size);
let x = (col * 4) >> sub_x;
let y = (row * 4) >> sub_y;
let w = (4 >> sub_x).min(plane_end_x - x + 1);
let h = (4 >> sub_y).min(plane_end_y - y + 1);
if w > 0 && h > 0 {
let ui = (unit_row * unit_cols + unit_col) as usize;
match fs.lr.lr_type[plane][ui] {
t if t == RESTORE_WIENER as u8 => wiener(fs, &ctx, &mut lr.planes[plane], plane, ui, x, y, w, h, bd),
t if t == RESTORE_SGRPROJ as u8 => self_guided(fs, &ctx, &mut lr.planes[plane], plane, ui, x, y, w, h, bd),
_ => {}
}
}
col += 1;
let ranges = bands(pool, fh.frame_height as usize);
let mut planes = lr.row_bands(&ranges);
pool.map_mut(&mut planes, |b, band| {
// Rows of 4 luma samples in this band.
let (start, end) = ranges[b];
let (row0, row1) = ((start / 4) as i32, (end / 4).min(fh.frame_height as usize) as i32);
for (plane, out) in band.iter_mut().enumerate().take(fs.num_planes) {
let rtype_frame = fh.lr.frame_restoration_type[plane];
if rtype_frame == RESTORE_NONE as u8 {
continue;
}
let (sub_x, sub_y) = if plane > 0 { (fs.ssx as i32, fs.ssy as i32) } else { (0, 0) };
let unit_size = fh.lr.loop_restoration_size[plane] as i32;
let unit_rows = fs.lr_unit_rows[plane] as i32;
let unit_cols = fs.lr_unit_cols[plane] as i32;
let plane_end_x = ((up_w + sub_x) >> sub_x) - 1;
let plane_end_y = ((fr_h + sub_y) >> sub_y) - 1;
let mut row = row0;
while row < row1 && row * 4 < fr_h {
let luma_y = row * 4;
let stripe_num = (luma_y + 8) / 64;
let stripe_start_y = (-8 + stripe_num * 64) >> sub_y;
let stripe_end_y = stripe_start_y + (64 >> sub_y) - 1;
let ctx = LrCtx { cur: &cur.planes[plane], cdef: &cdef.planes[plane], plane_end_x, plane_end_y, stripe_start_y, stripe_end_y };
let unit_row = (unit_rows - 1).min(((row * 4 + 8) >> sub_y) / unit_size);
let mut col = 0;
while col * 4 < up_w {
let unit_col = (unit_cols - 1).min(((col * 4) >> sub_x) / unit_size);
let x = (col * 4) >> sub_x;
let y = (row * 4) >> sub_y;
let w = (4 >> sub_x).min(plane_end_x - x + 1);
let h = (4 >> sub_y).min(plane_end_y - y + 1);
if w > 0 && h > 0 {
let ui = (unit_row * unit_cols + unit_col) as usize;
match fs.lr.lr_type[plane][ui] {
t if t == RESTORE_WIENER as u8 => wiener(fs, &ctx, out, plane, ui, x, y, w, h, bd),
t if t == RESTORE_SGRPROJ as u8 => self_guided(fs, &ctx, out, plane, ui, x, y, w, h, bd),
_ => {}
}
}
col += 1;
}
row += 1;
}
row += 1;
}
}
});
drop(planes);
lr
}
#[allow(clippy::too_many_arguments)]
fn wiener(fs: &FrameState, ctx: &LrCtx, out: &mut Plane, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
fn wiener(fs: &FrameState, ctx: &LrCtx, out: &mut RowBand, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
let round0: u32 = if bd == 12 { 5 } else { 3 };
let round1: u32 = if bd == 12 { 9 } else { 11 };
let coef = &fs.lr.lr_wiener[plane][ui];
@ -163,7 +175,7 @@ fn wiener(fs: &FrameState, ctx: &LrCtx, out: &mut Plane, plane: usize, ui: usize
}
#[allow(clippy::too_many_arguments)]
fn self_guided(fs: &FrameState, ctx: &LrCtx, out: &mut Plane, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
fn self_guided(fs: &FrameState, ctx: &LrCtx, out: &mut RowBand, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
let set = fs.lr.lr_sgr_set[plane][ui] as usize;
let mut flt0 = [[0i32; 4]; 4];
let mut flt1 = [[0i32; 4]; 4];

18
crates/av1/tests/common/mod.rs
View File

@ -16,6 +16,11 @@ pub fn ffmpeg_dav1d() -> Option<PathBuf> {
ffmpeg_with(&[("-decoders", "libdav1d")])
}
/// ffmpeg with libaom (4:2:2 and 4:4:4 fixtures, which SVT-AV1 cannot make) and libdav1d.
pub fn ffmpeg_libaom() -> Option<PathBuf> {
ffmpeg_with(&[("-encoders", "libaom-av1"), ("-decoders", "libdav1d")])
}
/// Builds such as the Windows "essentials" ones lack these libraries; skip rather than fail.
fn ffmpeg_with(needs: &[(&str, &str)]) -> Option<PathBuf> {
let ff = filmcraft_testkit::ffmpeg_or_skip("av1 oracle")?;
@ -38,9 +43,11 @@ pub fn run(ff: &Path, args: &[&str]) {
assert!(out.status.success(), "ffmpeg {:?} failed: {}", args, String::from_utf8_lossy(&out.stderr));
}
/// An AV1 fixture: a lavfi source encoded by ffmpeg's libsvtav1 into IVF.
/// An AV1 fixture: a lavfi source encoded by ffmpeg (libsvtav1 unless [`Spec::encoder`] says
/// otherwise) into IVF.
pub struct Spec {
pub name: &'static str,
pub encoder: &'static str,
pub lavfi: String,
pub frames: u32,
pub pix_fmt: &'static str,
@ -50,7 +57,12 @@ pub struct Spec {
impl Spec {
pub fn new(name: &'static str, lavfi: impl Into<String>, frames: u32, pix_fmt: &'static str, enc: &[&str]) -> Spec {
Spec { name, lavfi: lavfi.into(), frames, pix_fmt, enc: enc.iter().map(|s| s.to_string()).collect() }
Spec { name, encoder: "libsvtav1", lavfi: lavfi.into(), frames, pix_fmt, enc: enc.iter().map(|s| s.to_string()).collect() }
}
pub fn encoder(mut self, encoder: &'static str) -> Spec {
self.encoder = encoder;
self
}
}
@ -60,7 +72,7 @@ pub fn make(ff: &Path, spec: &Spec) -> PathBuf {
let tmp = filmcraft_testkit::temp_path(&path);
let frames = spec.frames.to_string();
let t = tmp.to_str().unwrap().to_string();
let mut args: Vec<&str> = vec!["-f", "lavfi", "-i", spec.lavfi.as_str(), "-frames:v", frames.as_str(), "-pix_fmt", spec.pix_fmt, "-c:v", "libsvtav1"];
let mut args: Vec<&str> = vec!["-f", "lavfi", "-i", spec.lavfi.as_str(), "-frames:v", frames.as_str(), "-pix_fmt", spec.pix_fmt, "-c:v", spec.encoder];
args.extend(spec.enc.iter().map(String::as_str));
args.extend(["-f", "ivf", t.as_str()]);
run(ff, &args);

16
crates/av1/tests/oracle_inter.rs
View File

@ -101,3 +101,19 @@ fn inter_1080p() {
check_bit_exact(&ff, &s);
}
}
/// 4:2:2 and 4:4:4 (libaom; SVT-AV1 only makes 4:2:0): the post-filters run in row bands whose
/// chroma rows depend on the vertical subsampling, so these check the band edges there too.
#[test]
fn inter_chroma_444_422() {
let Some(ff) = ffmpeg_libaom() else { return };
let p = ["-cpu-used", "8", "-crf", "40", "-enable-restoration", "1"];
let src = "testsrc2=s=416x400:r=25,noise=alls=8:allf=t";
for s in [
Spec::new("inter_aom_444", src, 6, "yuv444p", &p).encoder("libaom-av1"),
Spec::new("inter_aom_422", src, 6, "yuv422p", &p).encoder("libaom-av1"),
Spec::new("inter_aom_444_10bit", src, 6, "yuv444p10le", &p).encoder("libaom-av1"),
] {
check_bit_exact(&ff, &s);
}
}

47
crates/av1/tests/perf.rs
View File

@ -51,8 +51,8 @@ fn run(frames: &[Vec<u8>], threads: usize) -> Run {
}
fn bench(spec: &Spec) {
// AV1_PERF_ONLY=<substring> limits the run to matching fixtures.
if std::env::var("AV1_PERF_ONLY").is_ok_and(|f| !spec.name.contains(f.as_str())) {
// AV1_PERF_ONLY=<substring>[,<substring>...] limits the run to matching fixtures.
if std::env::var("AV1_PERF_ONLY").is_ok_and(|f| !f.split(',').any(|f| spec.name.contains(f))) {
return;
}
let Some(ff) = ffmpeg() else { return };
@ -68,7 +68,6 @@ fn bench(spec: &Spec) {
}
}
let b = best.unwrap();
let n = b.stats.frames.max(1) as f64;
let cpu_fps = b.cpu.map(|c| format!("{:6.1}", b.shown as f64 / c)).unwrap_or_else(|| " ?".into());
println!(
"{:<26} {:>3} shown {:>3} decoded {:>3} tiles/frame | 1 thread: {} fps (thread CPU) {:6.1} fps (wall)",
@ -79,24 +78,35 @@ fn bench(spec: &Spec) {
cpu_fps,
b.shown as f64 / b.wall
);
let total = b.stats.total_secs();
let mut line = String::from(" ms/frame:");
println!("{}", stage_line(" ms/frame:", &b.stats));
// All cores: best of 3 by wall time.
let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
let mut best: Option<Run> = None;
for _ in 0..3 {
let r = run(&frames, threads);
assert_eq!(r.shown, b.shown);
if best.as_ref().is_none_or(|b| r.wall < b.wall) {
best = Some(r);
}
}
let m = best.unwrap();
println!(" {threads} threads: {:6.1} fps (wall, load average {})", b.shown as f64 / m.wall, load_average());
// Per-stage wall time on the frame workers (stages of different frames overlap).
println!("{}", stage_line(&format!(" {threads} threads, ms/frame:"), &m.stats));
}
/// Per-stage milliseconds per frame and share of the total.
fn stage_line(label: &str, stats: &DecodeStats) -> String {
let n = stats.frames.max(1) as f64;
let total = stats.total_secs();
let mut line = String::from(label);
for s in Stage::ALL {
let v = b.stats.secs(s);
let v = stats.secs(s);
if v > 0.0005 {
line += &format!(" {} {:.2} ({:.0}%)", s.name(), 1e3 * v / n, 100.0 * v / total);
}
}
println!("{line}");
// All cores: best of 3 by wall time.
let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
let mut best_wall = f64::INFINITY;
for _ in 0..3 {
let r = run(&frames, threads);
assert_eq!(r.shown, b.shown);
best_wall = best_wall.min(r.wall);
}
println!(" {threads} threads: {:6.1} fps (wall, load average {})", b.shown as f64 / best_wall, load_average());
line
}
#[test]
@ -110,4 +120,9 @@ fn decode_throughput() {
// Higher quality (more coefficients, finer partitions) and a multi-tile stream (4x2 tiles).
bench(&Spec::new("perf_1080p_gop_hq", src, 60, "yuv420p", &p("18", "keyint=60")));
bench(&Spec::new("perf_1080p_gop_tiles", src, 60, "yuv420p", &p("30", "keyint=60:tile-columns=2:tile-rows=1")));
// Low delay (each frame references the previous one, as screen and camera recorders encode):
// frames cannot overlap, so a frame's own parallelism decides the speed.
bench(&Spec::new("perf_1080p_lowdelay", src, 60, "yuv420p", &p("30", "keyint=60:pred-struct=1")));
// One tile at 2160p: the post-filters' row bands are most of the parallelism there.
bench(&Spec::new("perf_2160p_gop", "testsrc2=s=3840x2160:r=25,noise=alls=6:allf=t", 30, "yuv420p", &p("30", "keyint=30")));
}

15
crates/avi/Cargo.toml Normal file
View File

@ -0,0 +1,15 @@
[package]
name = "filmcraft-avi"
description = "Clean-room AVI (RIFF) and OpenDML demuxer"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[dependencies]
[dev-dependencies]
filmcraft-testkit = { workspace = true }

43
crates/avi/README.md Normal file
View File

@ -0,0 +1,43 @@
# filmcraft-avi
A clean-room AVI demuxer, written from Microsoft's published AVI RIFF file reference and the
OpenDML AVI File Format Extensions 1.02. Layer L0: no dependencies beyond `std`, no `unsafe`, builds
for `wasm32-unknown-unknown`. No FFmpeg, libavformat or other AVI implementation was read; ffmpeg
and ffprobe are used only as external **test oracles**.
## API
```rust
use filmcraft_avi::{open, StreamKind};
let file = open(&bytes)?; // any ByteSource: a slice, a file, a web Blob…
for (i, s) in file.streams.iter().enumerate() {
match s.kind {
StreamKind::Video => println!("{:?} {} fps", s.video.as_ref().map(|b| b.compression), s.rate as f64 / s.scale as f64),
StreamKind::Audio => println!("format 0x{:04x}", s.audio.as_ref().map_or(0, |w| w.format_tag)),
_ => {}
}
let first = file.read_chunk(&bytes, i, 0)?; // frame data, on demand
}
```
## What it reads
| Part | |
|---|---|
| Headers | `avih`; per stream `strh`, `strf` (`BITMAPINFOHEADER`, `WAVEFORMATEX` / `WAVEFORMATEXTENSIBLE`), `strn` |
| Index | OpenDML `indx` super index → `ix##` standard indexes (field indexes too), else `idx1` (offsets relative to `movi` or absolute), else a scan of the `movi` lists |
| Large files | `RIFF AVIX` extensions: through the OpenDML index, or scanned after `idx1`, which only covers the first RIFF |
| Key frames | from the index; a scan marks every chunk and says so (`keyframes_known`) |
| Damage | sizes clamped to the file (a cut-off recording opens with what is there), `rec ` lists nested at most 3 deep, tables bounded by what the file could hold |
`filmcraft-codecs` maps the streams onto decoders (`AviSource`): Motion JPEG, H.264, HEVC and
uncompressed RGB / YUV video; PCM, MP3, MP2 and AC-3 audio.
## Tests
`cargo test -p filmcraft-avi`: hand-built files for every index path (`idx1` relative and absolute,
`rec ` lists, OpenDML standard indexes, a scan), truncation, and every single-byte mutation and
truncation of them (never a panic). `tests/ffmpeg_oracle.rs` checks ffmpeg-made files (Motion JPEG,
H.264 with B-frames, BGR24, YUY2; PCM and MP3) against ffprobe; the ignored `opendml_file_past_1_gb`
checks a 1.16 GB OpenDML file with `AVIX` extensions.

542
crates/avi/src/lib.rs Normal file
View File

@ -0,0 +1,542 @@
//! Clean-room AVI demuxer, written from Microsoft's published AVI RIFF file reference and the
//! OpenDML AVI File Format Extensions 1.02.
//!
//! [`open`] reads the `hdrl` header list (`avih`, and per stream `strh` / `strf` / `strn` /
//! `indx`) of the first `RIFF AVI ` and builds a per-stream table of [`Chunk`]s from the best index
//! the file has: the OpenDML super index (`indx` → `ix##` standard indexes, which also cover the
//! `RIFF AVIX` extensions of files past 1 GB), else the AVI 1.0 `idx1` (plus a scan of any `AVIX`
//! extension it cannot see), else a scan of the `movi` lists. Frame data is read on demand with
//! [`AviFile::read_chunk`]. Truncated files (a recording cut off) open with what is there.
//!
//! Every size and count comes from the file and is treated as hostile: reads are bounded by the
//! file length, tables by the number of chunks the file could hold, and nesting by a fixed depth.
//!
//! Layer L0: no dependencies beyond `std`, no `unsafe`, builds for `wasm32-unknown-unknown`.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
mod source;
#[cfg(test)]
mod tests;
pub use source::ByteSource;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Error {
/// Not an AVI file (no `RIFF` … `AVI ` header).
NotAvi,
/// A read failed.
Io(String),
/// Structurally invalid data.
Invalid(String),
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Error::NotAvi => f.write_str("not an AVI file"),
Error::Io(e) => write!(f, "I/O error: {e}"),
Error::Invalid(e) => write!(f, "invalid AVI: {e}"),
}
}
}
impl std::error::Error for Error {}
pub type Result<T> = std::result::Result<T, Error>;
/// Whether `head` starts like an AVI file: `RIFF` <size> `AVI `.
pub fn sniff(head: &[u8]) -> bool {
head.get(..4) == Some(b"RIFF") && head.get(8..12) == Some(b"AVI ")
}
/// `avih`: the main header (frame period, flags, frame count, size).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct MainHeader {
pub micro_sec_per_frame: u32,
pub flags: u32,
pub total_frames: u32,
pub streams: u32,
pub width: u32,
pub height: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamKind {
Video,
Audio,
/// Text, MIDI or anything else (`fccType`).
Other([u8; 4]),
}
/// `strf` of a video stream: a `BITMAPINFOHEADER`.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct BitmapInfo {
pub width: i32,
/// Positive: rows stored bottom-up (uncompressed RGB); negative: top-down.
pub height: i32,
pub bit_count: u16,
/// `biCompression` as four bytes: a FourCC (`MJPG`, `H264`, `YUY2`…) or 0 / 3 (`BI_RGB`,
/// `BI_BITFIELDS`) in little-endian.
pub compression: [u8; 4],
pub size_image: u32,
/// Bytes after the 40-byte header (codec setup, palette).
pub extra: Vec<u8>,
}
impl BitmapInfo {
/// Uncompressed RGB (`BI_RGB` / `BI_BITFIELDS`).
pub fn is_rgb(&self) -> bool {
matches!(u32::from_le_bytes(self.compression), 0 | 3)
}
}
/// `strf` of an audio stream: a `WAVEFORMATEX` (`WAVEFORMATEXTENSIBLE` is resolved to its sub-format).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct WaveFormat {
/// 1 PCM, 3 IEEE float, 0x50 MPEG audio, 0x55 MP3, 0x2000 AC-3, 0xFF AAC…
pub format_tag: u16,
pub channels: u16,
pub sample_rate: u32,
pub avg_bytes_per_sec: u32,
pub block_align: u16,
pub bits_per_sample: u16,
/// `cbSize` bytes after the 18-byte header.
pub extra: Vec<u8>,
}
/// One stored frame or audio chunk.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Chunk {
/// Offset of the data (after the 8-byte chunk header).
pub offset: u64,
pub size: u32,
/// A key frame (always true for audio and when the index does not say).
pub key: bool,
}
/// Where the chunk tables came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum IndexSource {
/// OpenDML `indx` super index and `ix##` standard indexes.
OpenDml,
/// The AVI 1.0 `idx1` (plus a scan of the `AVIX` extensions).
Idx1,
/// No usable index: the `movi` lists were scanned (key frames unknown).
Scan,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Stream {
pub kind: StreamKind,
/// `fccHandler`.
pub handler: [u8; 4],
/// Time base: `rate / scale` units per second (frames per second for video).
pub scale: u32,
pub rate: u32,
/// Start time in `scale / rate` units.
pub start: u32,
/// Length in `scale / rate` units.
pub length: u32,
/// Bytes per sample for fixed-size samples (PCM block align), 0 when samples vary.
pub sample_size: u32,
pub video: Option<BitmapInfo>,
pub audio: Option<WaveFormat>,
/// `strn`.
pub name: Option<String>,
pub chunks: Vec<Chunk>,
/// Whether [`Chunk::key`] comes from an index (a scan cannot tell).
pub keyframes_known: bool,
/// The OpenDML super index entries (offsets of the `ix##` chunks).
super_index: Vec<u64>,
}
impl Stream {
/// Stream index of a `movi` chunk id (`00dc`, `01wb`…): its first two characters.
fn number(id: &[u8; 4]) -> Option<usize> {
let d = |b: u8| (b as char).to_digit(16).map(|v| v as usize);
Some(d(id[0])? * 16 + d(id[1])?)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AviFile {
pub header: MainHeader,
pub streams: Vec<Stream>,
pub index: IndexSource,
}
/// Options for [`open_with`] (tests reach the fallback indexes with them).
#[derive(Clone, Copy, Debug, Default)]
pub struct OpenOptions {
pub ignore_odml: bool,
pub ignore_idx1: bool,
}
/// Headers larger than this are refused (a real `hdrl` is a few kilobytes).
const MAX_HEADER: u64 = 16 << 20;
/// Streams beyond this are ignored.
const MAX_STREAMS: usize = 64;
/// `LIST rec ` inside `movi` nests once; deeper is refused.
const MAX_DEPTH: u32 = 3;
fn io(e: std::io::Error) -> Error {
Error::Io(e.to_string())
}
fn read<S: ByteSource + ?Sized>(src: &S, at: u64, n: u64) -> Result<Vec<u8>> {
let end = at.checked_add(n).filter(|e| *e <= src.len()).ok_or_else(|| Error::Invalid(format!("{n} bytes at {at} run past the end")))?;
let len = usize::try_from(end - at).map_err(|_| Error::Invalid("chunk too large".into()))?;
let mut buf = vec![0; len];
src.read_at(at, &mut buf).map_err(io)?;
Ok(buf)
}
fn fourcc(b: &[u8], at: usize) -> [u8; 4] {
let mut f = [0; 4];
if let Some(s) = b.get(at..at + 4) {
f.copy_from_slice(s);
}
f
}
fn le16(b: &[u8], at: usize) -> u16 {
b.get(at..at + 2).map_or(0, |s| u16::from_le_bytes([s[0], s[1]]))
}
fn le32(b: &[u8], at: usize) -> u32 {
b.get(at..at + 4).map_or(0, |s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]))
}
fn le64(b: &[u8], at: usize) -> u64 {
u64::from(le32(b, at)) | (u64::from(le32(b, at + 4)) << 32)
}
/// A chunk header at `at`: (id, size, data offset). The size is clamped to the file (a truncated
/// recording keeps what is there).
fn header_at<S: ByteSource + ?Sized>(src: &S, at: u64) -> Result<([u8; 4], u64, u64)> {
let h = read(src, at, 8)?;
let data = at + 8;
let size = u64::from(le32(&h, 4)).min(src.len().saturating_sub(data));
Ok((fourcc(&h, 0), size, data))
}
/// Iterate the chunks of `[from, to)`: (id, size, data offset). Stops at the first unreadable
/// header; every step moves forward, so it ends.
fn children<S: ByteSource + ?Sized>(src: &S, from: u64, to: u64) -> Vec<([u8; 4], u64, u64)> {
let mut out = Vec::new();
let mut at = from;
while at.checked_add(8).is_some_and(|e| e <= to) {
let Ok((id, size, data)) = header_at(src, at) else { break };
out.push((id, size, data));
// chunks are padded to an even size
let Some(next) = data.checked_add(size).and_then(|e| e.checked_add(size & 1)) else { break };
if next <= at {
break;
}
at = next;
}
out
}
/// Open an AVI file.
pub fn open<S: ByteSource + ?Sized>(src: &S) -> Result<AviFile> {
open_with(src, OpenOptions::default())
}
pub fn open_with<S: ByteSource + ?Sized>(src: &S, opts: OpenOptions) -> Result<AviFile> {
let head = read(src, 0, 12.min(src.len())).map_err(|_| Error::NotAvi)?;
if !sniff(&head) {
return Err(Error::NotAvi);
}
// the RIFF forms: the first `AVI `, then any `AVIX` extensions (OpenDML)
let mut hdrl = None;
let mut movis: Vec<(u64, u64, u64)> = Vec::new(); // (position of the `movi` id, data start, data end)
let mut idx1 = None;
for (id, size, data) in children(src, 0, src.len()) {
if id != *b"RIFF" || size < 4 {
continue;
}
let end = data + size;
let form = fourcc(&read(src, data, 4)?, 0);
if form != *b"AVI " && form != *b"AVIX" {
continue;
}
for (cid, csize, cdata) in children(src, data + 4, end) {
match &cid {
b"LIST" if csize >= 4 => {
let ltype = fourcc(&read(src, cdata, 4)?, 0);
if ltype == *b"hdrl" && hdrl.is_none() && form == *b"AVI " {
hdrl = Some((cdata + 4, cdata + csize));
} else if ltype == *b"movi" {
movis.push((cdata, cdata + 4, cdata + csize));
}
}
b"idx1" if form == *b"AVI " => idx1 = Some((cdata, csize)),
_ => {}
}
}
}
let (hfrom, hto) = hdrl.ok_or_else(|| Error::Invalid("no hdrl header list".into()))?;
if hto - hfrom > MAX_HEADER {
return Err(Error::Invalid("header list too large".into()));
}
let (header, mut streams) = parse_hdrl(src, hfrom, hto)?;
if streams.is_empty() {
return Err(Error::Invalid("no streams".into()));
}
// chunk tables from the best index
let odml = !opts.ignore_odml && streams.iter().any(|s| !s.super_index.is_empty());
let index = if odml && read_odml(src, &mut streams) {
IndexSource::OpenDml
} else if let (false, Some((at, size)), Some(first)) = (opts.ignore_idx1, idx1, movis.first()) {
clear(&mut streams);
read_idx1(src, &mut streams, at, size, first.0);
// idx1 only covers the first RIFF: the AVIX extensions are scanned
for &(_, from, to) in movis.iter().skip(1) {
scan(src, &mut streams, from, to, 0);
}
if streams.iter().all(|s| s.chunks.is_empty()) { scan_all(src, &mut streams, &movis) } else { IndexSource::Idx1 }
} else {
scan_all(src, &mut streams, &movis)
};
Ok(AviFile { header, streams, index })
}
fn clear(streams: &mut [Stream]) {
for s in streams.iter_mut() {
s.chunks.clear();
s.keyframes_known = true;
}
}
fn scan_all<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream], movis: &[(u64, u64, u64)]) -> IndexSource {
clear(streams);
for &(_, from, to) in movis {
scan(src, streams, from, to, 0);
}
for s in streams.iter_mut() {
s.keyframes_known = false;
}
IndexSource::Scan
}
fn parse_hdrl<S: ByteSource + ?Sized>(src: &S, from: u64, to: u64) -> Result<(MainHeader, Vec<Stream>)> {
let mut header = MainHeader::default();
let mut streams = Vec::new();
for (id, size, data) in children(src, from, to) {
match &id {
b"avih" => {
let b = read(src, data, size.min(64))?;
header = MainHeader {
micro_sec_per_frame: le32(&b, 0),
flags: le32(&b, 12),
total_frames: le32(&b, 16),
streams: le32(&b, 24),
width: le32(&b, 32),
height: le32(&b, 36),
};
}
b"LIST" if size >= 4 && streams.len() < MAX_STREAMS && fourcc(&read(src, data, 4)?, 0) == *b"strl" => {
streams.push(parse_strl(src, data + 4, data + size)?);
}
_ => {}
}
}
Ok((header, streams))
}
fn parse_strl<S: ByteSource + ?Sized>(src: &S, from: u64, to: u64) -> Result<Stream> {
let mut s = Stream {
kind: StreamKind::Other([0; 4]),
handler: [0; 4],
scale: 1,
rate: 1,
start: 0,
length: 0,
sample_size: 0,
video: None,
audio: None,
name: None,
chunks: Vec::new(),
keyframes_known: true,
super_index: Vec::new(),
};
let mut strf = None;
for (id, size, data) in children(src, from, to) {
match &id {
b"strh" => {
let b = read(src, data, size.min(64))?;
s.kind = match &fourcc(&b, 0) {
b"vids" => StreamKind::Video,
b"auds" => StreamKind::Audio,
other => StreamKind::Other(*other),
};
s.handler = fourcc(&b, 4);
s.scale = le32(&b, 20);
s.rate = le32(&b, 24);
s.start = le32(&b, 28);
s.length = le32(&b, 32);
s.sample_size = le32(&b, 44);
}
b"strf" => strf = Some(read(src, data, size.min(1 << 20))?),
b"strn" => {
let b = read(src, data, size.min(4096))?;
let name = String::from_utf8_lossy(b.split(|&c| c == 0).next().unwrap_or_default()).trim().to_string();
s.name = (!name.is_empty()).then_some(name);
}
b"indx" => s.super_index = super_index(&read(src, data, size.min(MAX_HEADER))?),
_ => {}
}
}
if let Some(b) = strf {
match s.kind {
StreamKind::Video => {
s.video = Some(BitmapInfo {
width: le32(&b, 4) as i32,
height: le32(&b, 8) as i32,
bit_count: le16(&b, 14),
compression: fourcc(&b, 16),
size_image: le32(&b, 20),
extra: b.get(40.min(b.len())..).unwrap_or_default().to_vec(),
});
}
StreamKind::Audio => {
let cb = usize::from(le16(&b, 16));
let extra = b.get(18.min(b.len())..(18 + cb).min(b.len())).unwrap_or_default().to_vec();
let mut tag = le16(&b, 0);
// WAVE_FORMAT_EXTENSIBLE: the sub-format GUID starts with the real tag
if tag == 0xFFFE && extra.len() >= 22 {
tag = le16(&extra, 6);
}
s.audio = Some(WaveFormat {
format_tag: tag,
channels: le16(&b, 2),
sample_rate: le32(&b, 4),
avg_bytes_per_sec: le32(&b, 8),
block_align: le16(&b, 12),
bits_per_sample: le16(&b, 14),
extra,
});
}
StreamKind::Other(_) => {}
}
}
if s.scale == 0 || s.rate == 0 {
(s.scale, s.rate) = (1, 1);
}
Ok(s)
}
/// The `ix##` chunk offsets of an `indx` super index (`AVI_INDEX_OF_INDEXES`, 4 longs per entry).
fn super_index(b: &[u8]) -> Vec<u64> {
let longs = usize::from(le16(b, 0));
let (subtype, itype, n) = (b.get(2).copied().unwrap_or(0), b.get(3).copied().unwrap_or(0xFF), le32(b, 4) as usize);
if itype != 0 || subtype != 0 || longs != 4 {
return Vec::new();
}
// header 24 bytes, entries of 16: qwOffset, dwSize, dwDuration
let n = n.min(b.len().saturating_sub(24) / 16);
(0..n).map(|k| le64(b, 24 + k * 16)).filter(|&o| o > 0).collect()
}
/// Fill the chunk tables from the OpenDML standard indexes; false (tables cleared) when a stream
/// that has data lacks one or none could be read.
fn read_odml<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream]) -> bool {
let limit = src.len() / 8;
let mut total = 0u64;
for s in streams.iter_mut() {
s.chunks.clear();
s.keyframes_known = true;
for &at in &s.super_index {
let Ok((id, size, data)) = header_at(src, at) else { continue };
if &id[..2] != b"ix" {
continue;
}
let Ok(b) = read(src, data, size.min(MAX_HEADER * 8)) else { continue };
let (longs, subtype, itype) = (usize::from(le16(&b, 0)), b.get(2).copied().unwrap_or(0), b.get(3).copied().unwrap_or(0));
// AVI_INDEX_OF_CHUNKS; 2 longs per entry, or 3 for the field index (AVI_INDEX_2FIELD)
if itype != 1 || !(longs == 2 && subtype == 0 || longs == 3 && subtype == 1) {
continue;
}
let base = le64(&b, 12);
let step = longs * 4;
let n = (le32(&b, 4) as usize).min(b.len().saturating_sub(24) / step);
for k in 0..n {
let e = 24 + k * step;
let (off, raw) = (u64::from(le32(&b, e)), le32(&b, e + 4));
let size = raw & 0x7FFF_FFFF;
let Some(offset) = base.checked_add(off).filter(|o| o.checked_add(u64::from(size)).is_some_and(|end| end <= src.len())) else { continue };
total += 1;
if total > limit {
return false;
}
s.chunks.push(Chunk { offset, size, key: raw & 0x8000_0000 == 0 });
}
}
}
let ok = streams.iter().any(|s| !s.chunks.is_empty()) && streams.iter().all(|s| !s.super_index.is_empty() || matches!(s.kind, StreamKind::Other(_)));
if !ok {
clear(streams);
}
ok
}
/// Fill the chunk tables from `idx1` (16-byte entries: id, flags, offset, size). Offsets are
/// relative to the `movi` list's type id, or (some writers) absolute: the first entry decides.
fn read_idx1<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream], at: u64, size: u64, movi: u64) {
let Ok(b) = read(src, at, size.min(src.len() / 2)) else { return };
let entries: Vec<&[u8]> = b.as_chunks::<16>().0.iter().map(|e| e.as_slice()).collect();
let id_at = |pos: u64| read(src, pos, 4).ok().map(|v| fourcc(&v, 0));
let first = entries.iter().find(|e| le32(e, 4) & 1 == 0);
let relative = match first {
Some(e) => {
let (id, off) = (fourcc(e, 0), u64::from(le32(e, 8)));
id_at(movi + off) == Some(id) || id_at(off) != Some(id)
}
None => true,
};
let base = if relative { movi } else { 0 };
for e in entries {
let (id, flags, off, len) = (fourcc(e, 0), le32(e, 4), u64::from(le32(e, 8)), le32(e, 12));
// AVIIF_LIST entries (`rec ` lists) and palette changes carry no samples
if flags & 1 != 0 || &id[2..] == b"pc" {
continue;
}
let Some(s) = Stream::number(&id).and_then(|n| streams.get_mut(n)) else { continue };
let Some(offset) = base.checked_add(off).and_then(|o| o.checked_add(8)).filter(|o| o.checked_add(u64::from(len)).is_some_and(|end| end <= src.len()))
else {
continue;
};
s.chunks.push(Chunk { offset, size: len, key: flags & 0x10 != 0 || s.kind != StreamKind::Video });
}
}
/// Append the data chunks of a `movi` list (`[from, to)`) to the streams' tables, in file order.
fn scan<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream], from: u64, to: u64, depth: u32) {
if depth >= MAX_DEPTH {
return;
}
for (id, size, data) in children(src, from, to) {
if id == *b"LIST" {
if size >= 4 {
scan(src, streams, data + 4, data + size, depth + 1);
}
continue;
}
if &id[..2] == b"ix" || &id[2..] == b"pc" {
continue;
}
if let Some(s) = Stream::number(&id).and_then(|n| streams.get_mut(n)) {
s.chunks.push(Chunk { offset: data, size: u32::try_from(size).unwrap_or(u32::MAX), key: true });
}
}
}
impl AviFile {
/// The bytes of chunk `i` of stream `stream`.
pub fn read_chunk<S: ByteSource + ?Sized>(&self, src: &S, stream: usize, i: usize) -> Result<Vec<u8>> {
let c = self.streams.get(stream).and_then(|s| s.chunks.get(i)).ok_or_else(|| Error::Invalid(format!("no chunk {i} in stream {stream}")))?;
read(src, c.offset, u64::from(c.size))
}
}

59
crates/avi/src/source.rs Normal file
View File

@ -0,0 +1,59 @@
//! Random-access byte sources for the demuxer (same shape as the other container crates', so one
//! adapter serves them all).
use std::io;
/// Random-access, read-only byte source (file, in-memory buffer, web Blob…).
pub trait ByteSource {
/// Total length in bytes.
fn len(&self) -> u64;
/// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end.
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>;
/// True if the source holds no bytes.
fn is_empty(&self) -> bool {
self.len() == 0
}
}
fn eof() -> io::Error {
io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
}
impl ByteSource for [u8] {
fn len(&self) -> u64 {
<[u8]>::len(self) as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
let start = usize::try_from(offset).map_err(|_| eof())?;
let end = start.checked_add(buf.len()).ok_or_else(eof)?;
buf.copy_from_slice(self.get(start..end).ok_or_else(eof)?);
Ok(())
}
}
impl ByteSource for Vec<u8> {
fn len(&self) -> u64 {
self.as_slice().len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.as_slice().read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for &T {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for std::sync::Arc<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}

269
crates/avi/src/tests.rs Normal file
View File

@ -0,0 +1,269 @@
use super::*;
fn chunk(id: &[u8; 4], data: &[u8]) -> Vec<u8> {
let mut v = id.to_vec();
v.extend_from_slice(&(data.len() as u32).to_le_bytes());
v.extend_from_slice(data);
if data.len() % 2 == 1 {
v.push(0);
}
v
}
fn list(ltype: &[u8; 4], body: &[u8]) -> Vec<u8> {
let mut d = ltype.to_vec();
d.extend_from_slice(body);
chunk(b"LIST", &d)
}
fn strl(video: bool) -> Vec<u8> {
let mut strh = vec![0u8; 56];
strh[..4].copy_from_slice(if video { b"vids" } else { b"auds" });
strh[4..8].copy_from_slice(if video { b"MJPG" } else { &[0; 4] });
let (scale, rate, ss) = if video { (1u32, 25u32, 0u32) } else { (4, 192_000, 4) };
strh[20..24].copy_from_slice(&scale.to_le_bytes());
strh[24..28].copy_from_slice(&rate.to_le_bytes());
strh[44..48].copy_from_slice(&ss.to_le_bytes());
let strf = if video {
let mut b = vec![0u8; 40];
b[..4].copy_from_slice(&40u32.to_le_bytes());
b[4..8].copy_from_slice(&64i32.to_le_bytes());
b[8..12].copy_from_slice(&48i32.to_le_bytes());
b[14..16].copy_from_slice(&24u16.to_le_bytes());
b[16..20].copy_from_slice(b"MJPG");
b
} else {
let mut b = vec![0u8; 18];
b[..2].copy_from_slice(&1u16.to_le_bytes());
b[2..4].copy_from_slice(&2u16.to_le_bytes());
b[4..8].copy_from_slice(&48_000u32.to_le_bytes());
b[8..12].copy_from_slice(&192_000u32.to_le_bytes());
b[12..14].copy_from_slice(&4u16.to_le_bytes());
b[14..16].copy_from_slice(&16u16.to_le_bytes());
b
};
let mut body = chunk(b"strh", &strh);
body.extend(chunk(b"strf", &strf));
body.extend(chunk(b"strn", if video { b"Camera\0" } else { b"Mic\0" }));
list(b"strl", &body)
}
/// A small AVI: 3 video frames (the second a delta, the third odd-sized) and 2 audio chunks,
/// optionally inside a `rec ` list, with an `idx1` whose offsets are relative to `movi` or absolute.
fn avi(idx1: bool, absolute: bool, rec: bool) -> Vec<u8> {
let mut avih = vec![0u8; 56];
avih[0..4].copy_from_slice(&40_000u32.to_le_bytes());
avih[12..16].copy_from_slice(&0x10u32.to_le_bytes());
avih[16..20].copy_from_slice(&3u32.to_le_bytes());
avih[24..28].copy_from_slice(&2u32.to_le_bytes());
avih[32..36].copy_from_slice(&64u32.to_le_bytes());
avih[36..40].copy_from_slice(&48u32.to_le_bytes());
let mut hdrl = chunk(b"avih", &avih);
hdrl.extend(strl(true));
hdrl.extend(strl(false));
let hdrl = list(b"hdrl", &hdrl);
let samples: [(&[u8; 4], Vec<u8>, bool); 5] =
[(b"00dc", vec![1; 10], true), (b"01wb", vec![2; 16], true), (b"00dc", vec![3; 6], false), (b"01wb", vec![4; 8], true), (b"00dc", vec![5; 7], true)];
let mut body = Vec::new();
let mut entries = Vec::new();
for (id, data, key) in &samples {
entries.push((**id, body.len() as u32 + 4, data.len() as u32, *key));
body.extend(chunk(id, data));
}
let movi_body = if rec {
let r = list(b"rec ", &body);
// the samples moved 12 bytes in (the rec list's header and type)
for e in &mut entries {
e.1 += 12;
}
r
} else {
body
};
let movi = list(b"movi", &movi_body);
let mut riff = b"AVI ".to_vec();
riff.extend(hdrl);
// position of the `movi` type id: after `RIFF` <size>, what `riff` holds so far, `LIST` <size>
let movi_at = (8 + riff.len() + 8) as u32;
riff.extend(movi);
if idx1 {
let mut ix = Vec::new();
for (id, off, len, key) in entries {
ix.extend_from_slice(&id);
ix.extend_from_slice(&(if key { 0x10u32 } else { 0 }).to_le_bytes());
ix.extend_from_slice(&(if absolute { off + movi_at } else { off }).to_le_bytes());
ix.extend_from_slice(&len.to_le_bytes());
}
riff.extend(chunk(b"idx1", &ix));
}
chunk(b"RIFF", &riff)
}
fn table(f: &AviFile, s: usize) -> Vec<(u32, bool)> {
f.streams[s].chunks.iter().map(|c| (c.size, c.key)).collect()
}
#[test]
fn headers_and_every_index_path_agree() {
for (idx1, absolute, rec, want) in [
(true, false, false, IndexSource::Idx1),
(true, true, false, IndexSource::Idx1),
(true, false, true, IndexSource::Idx1),
(false, false, false, IndexSource::Scan),
(false, false, true, IndexSource::Scan),
] {
let file = avi(idx1, absolute, rec);
let f = open(&file).unwrap();
let what = format!("idx1 {idx1} absolute {absolute} rec {rec}");
assert_eq!(f.index, want, "{what}");
assert_eq!((f.header.width, f.header.height, f.header.total_frames, f.header.micro_sec_per_frame), (64, 48, 3, 40_000));
let (v, a) = (&f.streams[0], &f.streams[1]);
assert_eq!((v.kind, v.handler, v.scale, v.rate, v.name.as_deref()), (StreamKind::Video, *b"MJPG", 1, 25, Some("Camera")));
assert_eq!(v.video.as_ref().map(|b| (b.width, b.height, b.bit_count, b.compression)), Some((64, 48, 24, *b"MJPG")));
let w = a.audio.as_ref().unwrap();
assert_eq!((w.format_tag, w.channels, w.sample_rate, w.block_align, w.bits_per_sample, a.sample_size), (1, 2, 48_000, 4, 16, 4));
// the index knows the delta frame; a scan cannot
let keys = if idx1 { [true, false, true] } else { [true; 3] };
assert_eq!(table(&f, 0), vec![(10, keys[0]), (6, keys[1]), (7, keys[2])], "{what}");
assert_eq!(f.streams[0].keyframes_known, idx1);
assert_eq!(table(&f, 1), vec![(16, true), (8, true)], "{what}");
assert_eq!(f.read_chunk(&file, 0, 1).unwrap(), vec![3; 6], "{what}");
assert_eq!(f.read_chunk(&file, 0, 2).unwrap(), vec![5; 7], "{what}");
assert_eq!(f.read_chunk(&file, 1, 1).unwrap(), vec![4; 8], "{what}");
assert!(f.read_chunk(&file, 0, 3).is_err() && f.read_chunk(&file, 5, 0).is_err());
}
// the idx1 path can be skipped
let f = open_with(&avi(true, false, false), OpenOptions { ignore_idx1: true, ..Default::default() }).unwrap();
assert_eq!(f.index, IndexSource::Scan);
}
#[test]
fn not_avi_and_truncated_recordings() {
assert_eq!(open(&b"RIFF\0\0\0\0WAVEfmt ".to_vec()), Err(Error::NotAvi));
assert_eq!(open(&Vec::new()), Err(Error::NotAvi));
assert!(sniff(&avi(true, false, false)) && !sniff(b"RIFF"));
// a recording cut off in its last frame: the headers and the whole frames are there
let full = avi(false, false, false);
let cut = full[..full.len() - 4].to_vec();
let f = open(&cut).unwrap();
assert_eq!(f.streams[0].chunks.len(), 3);
assert!(f.read_chunk(&cut, 0, 0).is_ok());
}
/// Every single-byte change and every truncation of a valid file opens or fails, never panics, and
/// whatever opens reads its chunks without panicking.
#[test]
fn mutations_never_panic() {
for base in [avi(true, false, true), avi(false, false, false), avi(true, true, false)] {
for cut in 0..base.len() {
let b = base[..cut].to_vec();
if let Ok(f) = open(&b) {
for (s, st) in f.streams.iter().enumerate() {
for i in 0..st.chunks.len() {
let _ = f.read_chunk(&b, s, i);
}
}
}
}
for at in 0..base.len() {
for v in [0x00u8, 0xFF, 0x7F, 0x80] {
let mut b = base.clone();
b[at] = v;
if let Ok(f) = open(&b) {
for (s, st) in f.streams.iter().enumerate() {
for i in 0..st.chunks.len() {
let _ = f.read_chunk(&b, s, i);
}
}
}
}
}
}
}
/// An OpenDML file: each stream's `indx` super index points at an `ix##` standard index (base
/// offset + offsets to the data, bit 31 of the size marking delta frames). Built twice: the second
/// pass knows where the `ix##` chunks landed.
fn avi_odml(ix_at: [u64; 2]) -> (Vec<u8>, [u64; 2]) {
let indx = |stream: u8, at: u64| {
let mut b = Vec::new();
b.extend_from_slice(&4u16.to_le_bytes()); // wLongsPerEntry
b.extend_from_slice(&[0, 0]); // subtype, AVI_INDEX_OF_INDEXES
b.extend_from_slice(&1u32.to_le_bytes());
b.extend_from_slice(&[b'0', b'0' + stream, if stream == 0 { b'd' } else { b'w' }, if stream == 0 { b'c' } else { b'b' }]);
b.extend_from_slice(&[0; 12]);
b.extend_from_slice(&at.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
chunk(b"indx", &b)
};
let mut hdrl = chunk(b"avih", &[0; 56]);
for (s, at) in ix_at.iter().enumerate() {
let l = strl(s == 0);
// strl = LIST <size> strl …: append indx inside it
let mut body = l[12..].to_vec();
body.extend(indx(s as u8, *at));
hdrl.extend(list(b"strl", &body));
}
let mut riff = b"AVI ".to_vec();
riff.extend(list(b"hdrl", &hdrl));
// movi: video 10 (key) 6 (delta) 7 (key), audio 16 8; then ix00, ix01
let movi_data_at = 8 + riff.len() as u64 + 12; // RIFF hdr, so far, LIST hdr + `movi`
let mut body = Vec::new();
let mut at: [Vec<(u64, u32, bool)>; 2] = [Vec::new(), Vec::new()];
for (s, data, key) in [(0usize, vec![1u8; 10], true), (1, vec![2; 16], true), (0, vec![3; 6], false), (1, vec![4; 8], true), (0, vec![5; 7], true)] {
let id = if s == 0 { b"00dc" } else { b"01wb" };
at[s].push((movi_data_at + body.len() as u64 + 8, data.len() as u32, key));
body.extend(chunk(id, &data));
}
let mut where_ix = [0u64; 2];
for (s, entries) in at.iter().enumerate() {
let base = 100u64; // offsets are relative to qwBaseOffset
let mut b = Vec::new();
b.extend_from_slice(&2u16.to_le_bytes());
b.extend_from_slice(&[0, 1]); // subtype 0, AVI_INDEX_OF_CHUNKS
b.extend_from_slice(&(entries.len() as u32).to_le_bytes());
b.extend_from_slice(if s == 0 { b"00dc" } else { b"01wb" });
b.extend_from_slice(&base.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
for &(off, size, key) in entries {
b.extend_from_slice(&((off - base) as u32).to_le_bytes());
b.extend_from_slice(&(size | if key { 0 } else { 0x8000_0000 }).to_le_bytes());
}
where_ix[s] = movi_data_at + body.len() as u64;
body.extend(chunk(if s == 0 { b"ix00" } else { b"ix01" }, &b));
}
riff.extend(list(b"movi", &body));
(chunk(b"RIFF", &riff), where_ix)
}
#[test]
fn opendml_standard_indexes() {
let (_, at) = avi_odml([0, 0]);
let (file, again) = avi_odml(at);
assert_eq!(at, again, "the layout does not depend on the offsets");
let f = open(&file).unwrap();
assert_eq!(f.index, IndexSource::OpenDml);
assert_eq!(table(&f, 0), vec![(10, true), (6, false), (7, true)]);
assert_eq!(table(&f, 1), vec![(16, true), (8, true)]);
assert_eq!(f.read_chunk(&file, 0, 2).unwrap(), vec![5; 7]);
// ignoring the OpenDML index, the scan finds the same chunks (and skips the ix## chunks)
let s = open_with(&file, OpenOptions { ignore_odml: true, ..Default::default() }).unwrap();
assert_eq!(s.index, IndexSource::Scan);
assert_eq!(s.streams[0].chunks.iter().map(|c| c.offset).collect::<Vec<_>>(), f.streams[0].chunks.iter().map(|c| c.offset).collect::<Vec<_>>());
// a broken super index (pointing nowhere) falls back to a scan
let (bad, _) = avi_odml([3, 5]);
assert_eq!(open(&bad).unwrap().index, IndexSource::Scan);
// and mutations of the OpenDML file never panic
for i in 0..file.len() {
let mut b = file.clone();
b[i] ^= 0xFF;
if let Ok(f) = open(&b) {
for (s, st) in f.streams.iter().enumerate() {
for k in 0..st.chunks.len() {
let _ = f.read_chunk(&b, s, k);
}
}
}
}
}

151
crates/avi/tests/ffmpeg_oracle.rs Normal file
View File

@ -0,0 +1,151 @@
//! ffmpeg-made AVI files (skipped without ffmpeg): the chunk tables match ffprobe's packet counts,
//! and every index the file has (`idx1`; OpenDML once a file passes 1 GB) and a `movi` scan describe
//! the same chunks.
use std::path::{Path, PathBuf};
use std::process::Command;
use filmcraft_avi::{IndexSource, OpenOptions, StreamKind, open, open_with};
fn fixture(name: &str, args: &[&str]) -> Option<PathBuf> {
let ffmpeg = filmcraft_testkit::ffmpeg_or_skip("avi oracle")?;
let out = filmcraft_testkit::fixtures_dir("avi").join(name);
filmcraft_testkit::fixtures::generate(&out, |tmp: &Path| {
let mut c = Command::new(&ffmpeg);
c.args(["-v", "error", "-y"]).args(args).args(["-f", "avi"]).arg(tmp);
c.status().is_ok_and(|s| s.success())
})
}
fn packets(path: &Path, stream: usize) -> usize {
let ffprobe = filmcraft_testkit::ffprobe().expect("ffprobe next to ffmpeg");
let out = Command::new(ffprobe)
.args(["-v", "error", "-select_streams", &stream.to_string(), "-count_packets", "-show_entries", "stream=nb_read_packets", "-of", "csv=p=0"])
.arg(path)
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).trim().parse().unwrap_or(0)
}
fn packet_bytes(path: &Path, stream: usize) -> u64 {
let ffprobe = filmcraft_testkit::ffprobe().expect("ffprobe next to ffmpeg");
let out = Command::new(ffprobe)
.args(["-v", "error", "-select_streams", &stream.to_string(), "-show_entries", "packet=size", "-of", "csv=p=0"])
.arg(path)
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).lines().filter_map(|l| l.trim().parse::<u64>().ok()).sum()
}
const TONE: [&str; 4] = ["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=2"];
fn cases() -> Vec<(&'static str, Vec<&'static str>, [u8; 4])> {
let pic = vec!["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2"];
let with = |v: &[&'static str], a: &[&'static str]| -> Vec<&'static str> {
let mut x = pic.clone();
x.extend_from_slice(&TONE);
x.extend_from_slice(v);
x.extend_from_slice(a);
x
};
vec![
("mjpeg_pcm.avi", with(&["-c:v", "mjpeg", "-q:v", "4", "-pix_fmt", "yuvj420p"], &["-c:a", "pcm_s16le"]), *b"MJPG"),
("h264_mp3.avi", with(&["-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p"], &["-c:a", "libmp3lame", "-b:a", "128k"]), *b"H264"),
("bgr24_pcm.avi", with(&["-c:v", "rawvideo", "-pix_fmt", "bgr24"], &["-c:a", "pcm_s16le"]), [0; 4]),
("yuyv_pcm.avi", with(&["-c:v", "rawvideo", "-pix_fmt", "yuyv422"], &["-c:a", "pcm_s16le"]), *b"YUY2"),
]
}
#[test]
fn ffmpeg_files_match_ffprobe_on_every_index_path() {
for (name, args, fourcc) in cases() {
let Some(path) = fixture(name, &args) else { return };
let bytes = std::fs::read(&path).unwrap();
let f = open(&bytes).unwrap();
// ffmpeg reserves room for the OpenDML index but writes only idx1 under 1 GB
assert_eq!(f.index, IndexSource::Idx1, "{name}");
assert_eq!(f.streams.len(), 2, "{name}");
let (v, a) = (&f.streams[0], &f.streams[1]);
assert_eq!((v.kind, a.kind), (StreamKind::Video, StreamKind::Audio), "{name}");
let bmp = v.video.as_ref().unwrap();
assert_eq!((bmp.width, bmp.height.abs(), bmp.compression), (160, 120, fourcc), "{name}");
assert_eq!((v.rate as f64 / v.scale as f64).round(), 25.0, "{name}");
assert_eq!(a.audio.as_ref().map(|w| (w.sample_rate, w.channels)), Some((48_000, 1)), "{name}");
// the tables hold what ffprobe reads
assert_eq!(v.chunks.len(), packets(&path, 0), "{name}: video");
// PCM chunks are packets; ffprobe re-splits MP3 into its frames, which AVI chunks need not
// follow, so compressed audio compares by bytes
if a.audio.as_ref().is_some_and(|w| w.format_tag == 1) {
assert_eq!(a.chunks.len(), packets(&path, 1), "{name}: audio");
} else {
let ours: u64 = a.chunks.iter().map(|c| u64::from(c.size)).sum();
assert_eq!(ours, packet_bytes(&path, 1), "{name}: audio bytes");
}
assert!(v.chunks[0].key, "{name}: the first frame is a key frame");
if fourcc == *b"H264" {
assert!(v.chunks.iter().any(|c| !c.key), "{name}: the index marks the delta frames");
}
// a scan finds the same chunks (it cannot know key frames)
let scanned = open_with(&bytes, OpenOptions { ignore_odml: true, ignore_idx1: true }).unwrap();
assert_eq!(scanned.index, IndexSource::Scan, "{name}");
for s in 0..2 {
assert_eq!(sizes(&scanned, s), sizes(&f, s), "{name}: scan stream {s}");
}
}
}
fn sizes(f: &filmcraft_avi::AviFile, s: usize) -> Vec<(u64, u32)> {
f.streams[s].chunks.iter().map(|c| (c.offset, c.size)).collect()
}
/// A file past 1 GB, as long recordings are: ffmpeg writes `RIFF AVIX` extensions and the
/// OpenDML index. Needs ffmpeg and 1.2 GB of disk: `cargo test -p filmcraft-avi --test
/// ffmpeg_oracle -- --ignored`.
#[test]
#[ignore]
fn opendml_file_past_1_gb() {
let args = [
"-f",
"lavfi",
"-i",
"testsrc2=s=640x480:r=25:d=50",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=48000:d=50",
"-c:v",
"rawvideo",
"-pix_fmt",
"bgr24",
"-c:a",
"pcm_s16le",
];
let Some(path) = fixture("large_opendml.avi", &args) else { return };
let file = std::fs::File::open(&path).unwrap();
let src = FileSource(file);
let f = open(&src).unwrap();
assert_eq!(f.index, IndexSource::OpenDml);
assert_eq!(f.streams[0].chunks.len(), packets(&path, 0));
assert_eq!(f.streams[1].chunks.len(), packets(&path, 1));
assert_eq!(f.streams[0].chunks.len(), 1250);
assert!(f.streams[0].chunks.last().unwrap().offset > 1 << 30, "the last frames are in an AVIX extension");
// idx1 covers the first RIFF only; the AVIX extensions are scanned: the same chunks
let idx1 = open_with(&src, OpenOptions { ignore_odml: true, ..Default::default() }).unwrap();
assert_eq!(idx1.index, IndexSource::Idx1);
for s in 0..2 {
assert_eq!(sizes(&idx1, s), sizes(&f, s), "stream {s}");
}
assert_eq!(f.read_chunk(&src, 0, 1249).unwrap().len(), 640 * 480 * 3);
}
/// A file on disk as a byte source, read where asked.
struct FileSource(std::fs::File);
impl filmcraft_avi::ByteSource for FileSource {
fn len(&self) -> u64 {
self.0.metadata().map(|m| m.len()).unwrap_or(0)
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> {
std::os::unix::fs::FileExt::read_exact_at(&self.0, buf, offset)
}
}

2
crates/captions/README.md
View File

@ -16,7 +16,7 @@ and writes caption files, converts them to and from caption tracks, and draws ca
| Scenarist SCC `.scc` | yes | yes | CEA-608 at 29.97 fps, drop-frame or not. Reads pop-on, roll-up and paint-on (channel 1); writes pop-on. |
| MacCaption MCC `.mcc` | yes | yes | One SMPTE ST 334-2 caption distribution packet (CDP) per frame in MacCaption's hex text with letter abbreviations. Writes 29.97 fps (`30DF` / `30`) with CEA-608 field 1 (the SCC schedule) **and** CEA-708 service 1 (two-window pop-on emulation). Reads CEA-608 field 1 when present, else CEA-708 service 1; time code rates 30DF, 30, 60DF, 60, 25, 24, 50. |
| EBU STL `.stl` | yes | yes | EBU Tech 3264-E: GSI block + 128-byte TTI blocks, `STL25.01` (25 fps) or `STL30.01` (read/written as 29.97 fps, non-drop labels), ISO/IEC 6937 Latin text (accents as diacritic + letter), italics / underline codes as `<i>` / `<u>`, extension blocks for long subtitles, start-of-programme (TCP) subtracted. |
| TTML `.ttml` (IMSC1) | yes | yes | Writes the IMSC1 Text profile (TTML2 namespaces): `<p>` per cue, `<br/>`, styled `<span>`s for `<i>` / `<b>` / `<u>`, `ttm:agent` speakers, exact frame times (`123f`) at the sequence rate. Reads any TTML (TTML1, TTML2, the 2006 `ttaf1` drafts): clock / frame / offset / tick times, `dur`, nested time containers, timed spans, referenced styles. |
| TTML `.ttml` (IMSC1) | yes | yes | Writes the IMSC1 Text profile (TTML2 namespaces): `<p>` per cue, `<br/>`, styled `<span>`s for `<i>` / `<b>` / `<u>`, `ttm:agent` speakers, exact frame times (`123f`) at the sequence rate. Reads any TTML (TTML1, TTML2, the 2006 `ttaf1` drafts): clock / frame / offset / tick times, `dur`, nested time containers (`body` / `div` begins offset and ends clip their contents; `seq` containers are read as `par` with a warning), timed spans, referenced styles. |
| DFXP `.dfxp` | yes | yes | TTML1 with the DFXP presentation profile; clock times in milliseconds (frame-exact after `snap_to_frames`). |
All cue times are exact `Tick`s. Milliseconds convert exactly (1 ms = 254 016 000 ticks); SCC

89
crates/captions/src/ttml.rs
View File

@ -10,8 +10,8 @@
//!
//! **Reading** is namespace-agnostic (TTML1 2010, the 2006 `ttaf1` drafts and TTML2 all read):
//! clock times (`HH:MM:SS`, `.fraction`, `:FF` frames and `.subframes`), offset times (`h`, `m`,
//! `s`, `ms`, `f`, `t` with `ttp:tickRate`), `begin` / `end` / `dur`, nested time containers
//! (`body` / `div` begins add up), timed `<span>`s inside an untimed `<p>`, `<br/>`, whitespace
//! `s`, `ms`, `f`, `t` with `ttp:tickRate`), `begin` / `end` / `dur`, nested `par` time
//! containers (`body` / `div` begins add up and their ends clip what is inside), timed `<span>`s inside an untimed `<p>`, `<br/>`, whitespace
//! collapsing, styles referenced by id (italic / bold / underline) and `ttm:agent` / `xml:id`.
use filmcraft_time::{FrameRate, TICKS_PER_SECOND, Tick};
@ -423,44 +423,84 @@ pub fn parse(text: &str) -> Result<Document> {
let Some(body) = root.children().find(|n| n.is_element() && n.tag_name().name() == "body") else {
return Ok(doc);
};
walk(body, Tick::ZERO, &tm, &styles, Fmt::default(), &mut doc);
if root.descendants().any(|n| attr(n, "timeContainer").is_some_and(|v| v.trim() == "seq")) {
doc.warnings.push("timeContainer=\"seq\" is not supported; its children are timed as in a par container".into());
}
// the body is a time container like a div: its begin offsets, and its end clips, everything inside
let t = times(body, Scope { begin: Tick::ZERO, end: None }, tm, &mut doc);
walk(body, Scope { begin: t.begin.unwrap_or(Tick::ZERO), end: t.end }, &tm, &styles, Fmt::default(), &mut doc);
doc.cues.sort_by_key(|c| (c.start, c.end));
Ok(doc)
}
/// Begin and end of a timed element relative to `parent` (None when it has no timing).
fn times(n: roxmltree::Node, parent: Tick, tm: &Timing, doc: &mut Document) -> (Option<Tick>, Option<Tick>) {
let p = |k: &str, doc: &mut Document| -> Option<Tick> {
/// The active interval of the enclosing time containers (`par`): child times are offsets from
/// `begin`, and nothing inside outlasts `end` (None when no ancestor sets an end).
#[derive(Clone, Copy)]
struct Scope {
begin: Tick,
end: Option<Tick>,
}
/// Whether `n` has timing attributes of its own.
fn has_timing(n: roxmltree::Node) -> bool {
["begin", "end", "dur"].into_iter().any(|k| attr(n, k).is_some())
}
/// Timing of an element in its parent container.
struct Times {
/// The begin, when the element sets one.
begin: Option<Tick>,
/// Its own end (`end`, or `begin` + `dur`, the earlier of the two) clipped to the parent's
/// end, else the parent's end.
end: Option<Tick>,
/// A timing attribute did not parse (and was warned about).
bad: bool,
}
fn times(n: roxmltree::Node, parent: Scope, tm: Timing, doc: &mut Document) -> Times {
let mut bad = false;
let mut p = |k: &str, doc: &mut Document| -> Option<Tick> {
let v = attr(n, k)?;
let t = parse_time(v, *tm);
let t = parse_time(v, tm);
if t.is_none() {
bad = true;
doc.warnings.push(format!("bad time expression \"{v}\""));
}
t
};
let begin = p("begin", doc).map(|b| parent + b);
let end = match (p("end", doc), p("dur", doc)) {
(Some(e), _) => Some(parent + e),
(None, Some(d)) => Some(begin.unwrap_or(parent) + d),
_ => None,
let begin = p("begin", doc).map(|b| parent.begin + b);
let own_end = match (p("end", doc).map(|e| parent.begin + e), p("dur", doc).map(|d| begin.unwrap_or(parent.begin) + d)) {
(Some(e), Some(d)) => Some(e.min(d)),
(e, d) => e.or(d),
};
(begin, end)
let end = match (own_end, parent.end) {
(Some(e), Some(pe)) => Some(e.min(pe)),
(e, pe) => e.or(pe),
};
Times { begin, end, bad }
}
fn walk(n: roxmltree::Node, parent: Tick, tm: &Timing, styles: &std::collections::HashMap<String, Fmt>, fmt: Fmt, doc: &mut Document) {
fn walk(n: roxmltree::Node, parent: Scope, tm: &Timing, styles: &std::collections::HashMap<String, Fmt>, fmt: Fmt, doc: &mut Document) {
for c in n.children().filter(|c| c.is_element()) {
let f = style_of(c, styles, fmt);
match c.tag_name().name() {
"div" => {
let (b, _) = times(c, parent, tm, doc);
walk(c, b.unwrap_or(parent), tm, styles, f, doc);
let t = times(c, parent, *tm, doc);
walk(c, Scope { begin: t.begin.unwrap_or(parent.begin), end: t.end }, tm, styles, f, doc);
}
"p" => {
let (b, e) = times(c, parent, tm, doc);
let t = times(c, parent, *tm, doc);
if t.bad {
continue;
}
let (b, e) = (t.begin.unwrap_or(parent.begin), t.end);
let speaker = attr(c, "agent").map(|a| a.trim().trim_start_matches('_').replace('_', " "));
let id = c.attributes().find(|a| a.name() == "id").map(|a| a.value().to_string());
match (b, e) {
(Some(b), Some(e)) => {
let spans: Vec<roxmltree::Node> = c.children().filter(|x| x.is_element() && x.tag_name().name() == "span" && has_timing(*x)).collect();
// a timed paragraph, or one without timed spans in a container with an end, is one cue
let whole = if has_timing(c) || spans.is_empty() { e } else { None };
match whole {
Some(e) => {
let mut o = Out { s: String::new(), pending: false };
let tags: Vec<&str> = [(f.italic, "i"), (f.bold, "b"), (f.underline, "u")].into_iter().filter(|x| x.0).map(|x| x.1).collect();
for t in &tags {
@ -475,12 +515,13 @@ fn walk(n: roxmltree::Node, parent: Tick, tm: &Timing, styles: &std::collections
doc.cues.push(Cue { start: b, end: e, text, speaker, id, ..Default::default() });
}
}
_ => {
None => {
// untimed paragraph: its timed spans are the cues
let base = b.unwrap_or(parent);
for sp in c.children().filter(|x| x.is_element() && x.tag_name().name() == "span") {
let (sb, se) = times(sp, base, tm, doc);
if let (Some(sb), Some(se)) = (sb, se) {
let scope = Scope { begin: b, end: e };
for sp in spans {
let st = times(sp, scope, *tm, doc);
let sb = st.begin.unwrap_or(scope.begin);
if let (false, Some(se)) = (st.bad, st.end) {
let mut o = Out { s: String::new(), pending: false };
collect(sp, styles, style_of(sp, styles, f), &mut o);
let text = clean(&o.s);

92
crates/captions/tests/ttml_body_timing.rs Normal file
View File

@ -0,0 +1,92 @@
//! TTML / DFXP inherited timing: `begin`, `end` and `dur` on `body` and `div` time containers
//! offset and clip the paragraphs inside them (#895).
use filmcraft_captions::{Format, WriteOptions, parse, write};
/// (start, end, text) of every cue, in seconds; panics on a parse error.
fn cues(xml: &str, format: Format) -> Vec<(f64, f64, String)> {
let doc = parse(xml.as_bytes(), format).unwrap();
assert!(doc.warnings.is_empty(), "{:?}", doc.warnings);
doc.cues.iter().map(|c| (c.start.seconds(), c.end.seconds(), c.text.clone())).collect()
}
fn both(xml: &str, want: &[(f64, f64, &str)]) {
let want: Vec<(f64, f64, String)> = want.iter().map(|(s, e, t)| (*s, *e, t.to_string())).collect();
for format in [Format::Ttml, Format::Dfxp] {
assert_eq!(cues(xml, format), want, "{format:?}: {xml}");
}
}
#[test]
fn body_begin_is_inherited() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div><p begin="1s" end="2s">x</p></div></body></tt>"#, &[(6.0, 7.0, "x")]);
}
#[test]
fn div_begin_is_inherited() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div begin="5s"><p begin="1s" end="2s">x</p></div></body></tt>"#, &[(6.0, 7.0, "x")]);
}
#[test]
fn untimed_containers_leave_times_unchanged() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div><p begin="1s" end="2s">x</p></div></body></tt>"#, &[(1.0, 2.0, "x")]);
}
#[test]
fn nested_begins_add_up() {
both(
r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div begin="00:00:10.000"><div begin="1s"><p begin="1s" end="2s">deep</p></div><p begin="0s" dur="1s">mid</p></div><p begin="1s" end="2s">top</p></body></tt>"#,
&[(6.0, 7.0, "top"), (15.0, 16.0, "mid"), (17.0, 18.0, "deep")],
);
}
#[test]
fn body_begin_offsets_timed_spans() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div><p><span begin="1s" end="2s">x</span></p></div></body></tt>"#, &[(6.0, 7.0, "x")]);
}
#[test]
fn container_ends_clip_descendants() {
// div active 11 s to 13 s: a later paragraph is dropped, a longer one is cut at 13 s
both(
r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="1s"><div begin="10s" end="12s"><p begin="1s" end="5s">cut</p><p begin="3s" end="4s">gone</p></div></body></tt>"#,
&[(12.0, 13.0, "cut")],
);
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body dur="3s"><div><p begin="1s" end="5s">x</p></div></body></tt>"#, &[(1.0, 3.0, "x")]);
}
#[test]
fn paragraphs_without_an_end_last_until_their_container_ends() {
both(
r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div begin="2s" end="6s"><p begin="1s">open</p></div><div begin="10s" dur="2s"><p>whole</p></div></body></tt>"#,
&[(3.0, 6.0, "open"), (10.0, 12.0, "whole")],
);
}
#[test]
fn end_and_dur_take_the_earlier() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div><p begin="1s" end="5s" dur="2s">x</p></div></body></tt>"#, &[(1.0, 3.0, "x")]);
}
#[test]
fn seq_containers_warn() {
let xml = r#"<tt xmlns="http://www.w3.org/ns/ttml" xmlns:ttp="http://www.w3.org/ns/ttml#parameter"><body><div timeContainer="seq"><p begin="1s" end="2s">x</p></div></body></tt>"#;
for format in [Format::Ttml, Format::Dfxp] {
let doc = parse(xml.as_bytes(), format).unwrap();
assert_eq!(doc.cues.len(), 1);
assert!(doc.warnings.iter().any(|w| w.contains("seq")), "{:?}", doc.warnings);
}
}
#[test]
fn inherited_times_survive_export() {
// the writer uses untimed body / div and absolute paragraph times, so the resolved times stay
let xml = r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div begin="1s" end="4s"><p begin="1s" end="9s">x</p></div></body></tt>"#;
for format in [Format::Ttml, Format::Dfxp] {
let doc = parse(xml.as_bytes(), format).unwrap();
let back = parse(&write(&doc, format, WriteOptions::default()), format).unwrap();
let times = |d: &filmcraft_captions::Document| d.cues.iter().map(|c| (c.start, c.end, c.text.clone())).collect::<Vec<_>>();
assert_eq!(times(&back), times(&doc), "{format:?}");
assert_eq!((back.cues[0].start.seconds(), back.cues[0].end.seconds()), (7.0, 9.0), "{format:?}");
}
}

4
crates/codecs/Cargo.toml
View File

@ -14,16 +14,19 @@ filmcraft-time = { workspace = true }
filmcraft-color = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-media = { workspace = true }
filmcraft-avi = { workspace = true }
filmcraft-isobmff = { workspace = true }
filmcraft-matroska = { workspace = true }
filmcraft-mxf = { workspace = true }
filmcraft-mpegts = { workspace = true }
filmcraft-mpeg2v = { workspace = true }
filmcraft-ac3 = { workspace = true }
filmcraft-mpa = { workspace = true }
filmcraft-ogg = { workspace = true }
filmcraft-bitstream = { workspace = true }
filmcraft-h264 = { workspace = true }
filmcraft-hevc = { workspace = true }
filmcraft-vp8 = { workspace = true }
filmcraft-vp9 = { workspace = true }
filmcraft-prores = { workspace = true }
filmcraft-dnx = { workspace = true }
@ -31,6 +34,7 @@ filmcraft-av1 = { workspace = true }
filmcraft-apv = { workspace = true }
filmcraft-aac = { workspace = true }
filmcraft-opus = { workspace = true }
rayon = { workspace = true }
thiserror = { workspace = true }
log = { workspace = true }
# Decode timing counters (`gop_stats`); std's Instant panics on wasm32.

214
crates/codecs/src/audio.rs
View File

@ -21,7 +21,7 @@ enum Inner {
/// AAC whose configuration arrives with the first frame (ADTS / LATM).
AacPending,
/// MPEG audio at this rate whose layer arrives with the first frame (MP4 / QuickTime name the
/// codec, not the layer, and symphonia's decoder reads only the layer it was made for).
/// codec, not the layer; layers I / II go to our decoder, layer III to symphonia's).
MpegPending(u32),
/// Our own Opus decoder (always 48 kHz output; pre-skip is left to container timestamps).
/// `order` maps output channel → decoded channel (Vorbis → WAV/SMPTE order for surround).
@ -30,6 +30,8 @@ enum Inner {
Symphonia(Box<dyn Decoder>),
/// Our AC-3 and E-AC-3 decoder (ATSC A/52, Annex E).
Ac3(Box<filmcraft_ac3::Decoder>),
/// Our MPEG-1/2 audio Layer I and II decoder (ISO/IEC 11172-3, 13818-3).
Mpa(Box<filmcraft_mpa::Decoder>),
}
/// Opus always decodes at 48 kHz (RFC 7845 §5.1: the input rate in the header is informational).
@ -59,13 +61,16 @@ fn vorbis_to_wav_order(channels: usize) -> Option<&'static [usize]> {
/// Channel order for AAC surround (ISO/IEC 14496-3 Table 1.19, our decoder's element order: C, L R,
/// surrounds, LFE): output channel → decoded channel, giving the WAV/SMPTE order (L R C LFE Ls Rs)
/// used elsewhere, as the AC-3 decoder does and the AAC encoder's caller undoes on export. Only the
/// unambiguous channel configurations 3–6; 1, 2, 7 and PCE layouts keep the decoded order.
/// unambiguous channel configurations 3–6, 11 (6.1: … LFE, back centre, sides) and 12 (7.1: … LFE,
/// backs, sides); 1, 2, 7 and PCE layouts keep the decoded order.
fn aac_to_wav_order(channel_config: u8) -> Option<&'static [usize]> {
Some(match channel_config {
3 => &[1, 2, 0],
4 => &[1, 2, 0, 3],
5 => &[1, 2, 0, 3, 4],
6 => &[1, 2, 0, 5, 3, 4],
11 => &[1, 2, 0, 6, 5, 3, 4],
12 => &[1, 2, 0, 7, 5, 6, 3, 4],
_ => return None,
})
}
@ -127,11 +132,12 @@ impl PacketDecoder {
let order = if dec.head().mapping_family == 1 { vorbis_to_wav_order(channels) } else { None };
Ok(Self { inner: Inner::Opus { dec: Box::new(dec), order }, channels })
}
/// MPEG-1/2 audio layer I, II or III (symphonia).
/// MPEG-1/2 audio layer I, II (our decoder: symphonia rejects the bit rate / mode
/// combinations ISO/IEC 11172-3 does not allow, such as ffmpeg's default 384 kbit/s mono MP2)
/// or III (symphonia).
pub fn mpeg_audio(layer: u8, sample_rate: u32) -> Result<Self> {
let codec = match layer {
1 => symphonia::core::codecs::CODEC_TYPE_MP1,
2 => symphonia::core::codecs::CODEC_TYPE_MP2,
1 | 2 => return Ok(Self { inner: Inner::Mpa(Box::default()), channels: 0 }),
_ => CODEC_TYPE_MP3,
};
Self::new(codec, sample_rate, None)
@ -172,7 +178,8 @@ impl PacketDecoder {
C::Mp3 => Ok(Self::lazy_mpeg_audio(rate)),
C::Ac3 { .. } | C::Eac3 { .. } => Self::ac3(),
C::Alac { cookie } => Self::new(CODEC_TYPE_ALAC, rate, Some(cookie.clone())),
C::Flac(_) => Self::new(CODEC_TYPE_FLAC, rate, None),
// the decoder needs the STREAMINFO body: `dfLa` holds it after the 4-byte block header
C::Flac(f) => Self::new(CODEC_TYPE_FLAC, rate, f.metadata_blocks.get(4..38).map(<[u8]>::to_vec)),
C::Opus(o) => Self::opus(filmcraft_opus::OpusHead::from_dops(&o.to_bytes()).map_err(|e| CodecError::Unsupported(format!("Opus: {e}")))?),
other => Err(CodecError::Unsupported(format!("{} audio", other.name()))),
}
@ -228,6 +235,31 @@ impl PacketDecoder {
self.channels = out.len();
return Ok(out);
}
Inner::Mpa(d) => {
// a packet holds one frame or several; a damaged first frame is an error, damage
// after a good frame ends the packet
let mut out: Vec<Vec<f32>> = Vec::new();
let mut p = 0;
while let Some(rest) = data.get(p..).filter(|r| !r.is_empty()) {
let f = match d.decode(rest) {
Ok(f) => f,
Err(e) if out.is_empty() => return Err(CodecError::Decode(e.to_string())),
Err(_) => break,
};
p += f.header.frame_bytes;
if out.is_empty() {
out = f.channels;
} else if out.len() == f.channels.len() {
for (o, c) in out.iter_mut().zip(f.channels) {
o.extend(c);
}
} else {
break;
}
}
self.channels = out.len();
return Ok(out);
}
};
let pkt = Packet::new_from_slice(0, ts, 0, data);
let buf = dec.decode(&pkt).map_err(|e| CodecError::Decode(e.to_string()))?;
@ -256,6 +288,7 @@ impl PacketDecoder {
up.reset();
}
}
Inner::Mpa(d) => d.reset(),
Inner::Opus { dec, .. } => dec.reset(),
Inner::Symphonia(d) => d.reset(),
Inner::Ac3(d) => d.reset(),
@ -765,12 +798,30 @@ pub struct AudioFileSource {
impl AudioFileSource {
pub fn decode(name: &str, bytes: Arc<[u8]>) -> Result<Self> {
let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
if matches!(ext.as_str(), "mp2" | "mp3")
&& let Some((layer, rate, samples)) = decode_mpeg_layer12(&bytes)
{
let frames = samples.first().map_or(0, Vec::len);
let info = MediaInfo {
name: name.to_string(),
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: samples.len() as u32, codec: format!("MP{layer}"), bits_per_sample: None }],
container: ext.to_uppercase(),
start_timecode: None,
file_size: Some(bytes.len() as u64),
};
return Ok(Self { info, rate, samples });
}
let mss = MediaSourceStream::new(Box::new(std::io::Cursor::new(bytes.clone())), Default::default());
let mut hint = Hint::new();
hint.with_extension(&ext);
let probed = symphonia::default::get_probe()
.format(&hint, mss, &FormatOptions::default(), &MetadataOptions::default())
.map_err(|e| CodecError::Unsupported(e.to_string()))?;
// gapless: trim the encoder delay/padding that MP3 (LAME/Xing tag) files declare, so audio lands on the timeline where other editors put it.
// MP3 only: the other formats' timing (AAC priming in MP4 edit lists, …) is handled elsewhere and checked against ffmpeg.
let fmt_opts = FormatOptions { enable_gapless: ext == "mp3", ..FormatOptions::default() };
let probed =
symphonia::default::get_probe().format(&hint, mss, &fmt_opts, &MetadataOptions::default()).map_err(|e| CodecError::Unsupported(e.to_string()))?;
let mut format = probed.format;
let track = format.default_track().ok_or_else(|| CodecError::Unsupported("no audio track".into()))?.clone();
let rate = track.codec_params.sample_rate.unwrap_or(48_000);
@ -820,6 +871,49 @@ impl AudioFileSource {
}
}
/// A raw MPEG audio Layer I / II stream (`.mp2`) decoded with our decoder: (layer, sample rate,
/// planar samples). Frames follow each other from the first frame header (after an ID3v2 tag);
/// damaged data is skipped up to the next frame header of the same layer and rate, and a frame
/// with another channel count keeps its duration as silence. `None` when the stream does not start
/// with a Layer I / II frame followed by another frame (or the end), e.g. MP3.
fn decode_mpeg_layer12(bytes: &[u8]) -> Option<(u8, u32, Vec<Vec<f32>>)> {
let mut p = match bytes {
// ID3v2: 10-byte header, a syncsafe size, and a 10-byte footer when flagged
[b'I', b'D', b'3', _, _, flags, s0, s1, s2, s3, ..] => {
let size = [*s0, *s1, *s2, *s3].iter().fold(0usize, |a, &b| (a << 7) | (b & 0x7F) as usize);
size.saturating_add(if flags & 0x10 != 0 { 20 } else { 10 })
}
_ => 0,
};
let frame_at = |p: usize| bytes.get(p..).and_then(|b| filmcraft_mpa::parse_header(b).ok()).filter(|h| h.frame_bytes > 0);
let first = frame_at(p)?;
let next = p.saturating_add(first.frame_bytes);
if first.layer == 3 || (next < bytes.len() && frame_at(next).is_none_or(|h| h.layer != first.layer || h.sample_rate != first.sample_rate)) {
return None;
}
let mut dec = filmcraft_mpa::Decoder::new();
let mut out: Vec<Vec<f32>> = vec![Vec::new(); first.channels()];
while let Some(rest) = bytes.get(p..).filter(|r| r.len() >= 4) {
match dec.decode(rest) {
Ok(f) if f.header.layer == first.layer && f.header.sample_rate == first.sample_rate => {
let n = f.header.samples();
if f.channels.len() == out.len() {
for (o, c) in out.iter_mut().zip(f.channels) {
o.extend(c);
}
} else {
for o in out.iter_mut() {
o.resize(o.len() + n, 0.0);
}
}
p += f.header.frame_bytes;
}
_ => p += 1,
}
}
out.first().is_some_and(|c| !c.is_empty()).then_some((first.layer, first.sample_rate, out))
}
impl MediaSource for AudioFileSource {
fn info(&self) -> &MediaInfo {
&self.info
@ -840,6 +934,57 @@ pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<std::result::Result<Shared
Some(AudioFileSource::decode(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
#[cfg(test)]
mod mpeg_layer12_tests {
use super::*;
/// A silent 48 kHz MPEG-1 Layer II frame (all-zero allocations) of `bitrate_index` and `mode`.
fn silent_frame(bitrate_index: u8, mode: u8) -> Vec<u8> {
let h = [0xFF, 0xFD, bitrate_index << 4 | 0x04, mode << 6];
let mut f = h.to_vec();
f.resize(filmcraft_mpa::parse_header(&h).unwrap().frame_bytes, 0);
f
}
/// #809: mono MP2 at 384 kbit/s (ffmpeg's default for mono) is outside ISO/IEC 11172-3's
/// allowed combinations, and symphonia rejected every frame, so the track played as silence.
#[test]
fn mono_mp2_at_384k_decodes() {
let mut d = PacketDecoder::mpeg_audio(2, 48_000).unwrap();
let out = d.decode(&silent_frame(14, 3), 0).unwrap();
assert_eq!((out.len(), out[0].len(), d.channels), (1, 1152, 1));
// a packet of two stereo frames decodes to both
let two = [silent_frame(10, 0), silent_frame(10, 0)].concat();
let out = d.decode(&two, 1152).unwrap();
assert_eq!((out.len(), out[0].len()), (2, 2304));
// damage: an error for the first frame, the good frames before it otherwise
assert!(d.decode(&[0u8; 8], 0).is_err());
assert!(d.decode(&silent_frame(10, 0)[..100], 0).is_err());
let partial = [silent_frame(10, 0), vec![0xFF, 0xFD, 0xA4]].concat();
assert_eq!(d.decode(&partial, 0).unwrap()[0].len(), 1152);
d.reset();
}
/// Raw `.mp2` files: after an ID3v2 tag, frame after frame, skipping damage.
#[test]
fn raw_streams_resync_after_damage() {
let mono = silent_frame(14, 3);
let file = [&b"ID3\x04\0\0\0\0\0\x05abcde"[..], &mono, &mono, &[0xFF, 0xFD, 0x13, 0x37, 0x42], &mono].concat();
let (layer, rate, samples) = decode_mpeg_layer12(&file).unwrap();
assert_eq!((layer, rate, samples.len(), samples[0].len()), (2, 48_000, 1, 3 * 1152));
let src = AudioFileSource::decode("x.mp2", file.clone().into()).unwrap();
assert_eq!(src.info().audio().map(|a| (a.channels, a.codec.as_str())), Some((1, "MP2")));
// MP3, junk and truncated data are not ours (symphonia gets them)
assert!(decode_mpeg_layer12(&[0xFF, 0xFB, 0x90, 0x44, 0, 0]).is_none());
for n in 0..40 {
assert!(decode_mpeg_layer12(&file[..n]).is_none(), "{n}");
}
assert!(decode_mpeg_layer12(&[&mono[..], &[0x12; 50]].concat()).is_none());
// an ID3v2 tag claiming the largest size (with a footer) points past the end
assert!(decode_mpeg_layer12(&[&b"ID3\x04\0\x10\x7F\x7F\x7F\x7F"[..], &mono].concat()).is_none());
}
}
#[cfg(test)]
mod he_aac_tests {
use super::*;
@ -1070,3 +1215,54 @@ mod mpeg_audio_tests {
}
}
}
#[cfg(test)]
mod gapless_tests {
use super::*;
const FRAME_LEN: usize = 417;
const AUDIO_FRAMES: usize = 10;
/// CRC-16 as used by the LAME tag (polynomial 0x8005, reflected, start 0).
fn crc16(data: &[u8]) -> u16 {
let mut crc = 0u16;
for &b in data {
crc ^= b as u16;
for _ in 0..8 {
crc = if crc & 1 != 0 { (crc >> 1) ^ 0xA001 } else { crc >> 1 };
}
}
crc
}
/// Silent mono 44.1 kHz 128 kbps MPEG-1 layer III file: an Info frame carrying a LAME tag with
/// an encoder delay of 576 samples and no padding, then ten frames of silence.
fn tagged_mp3() -> Vec<u8> {
let header = [0xFF, 0xFB, 0x90, 0xC0];
let mut info = vec![0u8; FRAME_LEN];
info[..4].copy_from_slice(&header);
info[21..25].copy_from_slice(b"Info");
info[25..29].copy_from_slice(&0x0Fu32.to_be_bytes());
info[29..33].copy_from_slice(&(AUDIO_FRAMES as u32).to_be_bytes());
info[33..37].copy_from_slice(&((FRAME_LEN * (AUDIO_FRAMES + 1)) as u32).to_be_bytes());
info[141..150].copy_from_slice(b"LAME3.100");
// encoder delay 576 (12 bits), padding 0 (12 bits)
info[162..165].copy_from_slice(&[0x24, 0x00, 0x00]);
let crc = crc16(&info[..175]);
info[175..177].copy_from_slice(&crc.to_be_bytes());
let mut file = info;
for _ in 0..AUDIO_FRAMES {
let mut f = vec![0u8; FRAME_LEN];
f[..4].copy_from_slice(&header);
file.extend_from_slice(&f);
}
file
}
/// #1137: the LAME tag's encoder delay (576) plus the decoder delay (529) is cut from the start.
#[test]
fn mp3_encoder_delay_is_trimmed() {
let src = AudioFileSource::decode("tagged.mp3", Arc::from(tagged_mp3())).unwrap();
assert_eq!(src.samples[0].len(), AUDIO_FRAMES * 1152 - (576 + 529));
}
}

29
crates/codecs/src/audio_timing_tests.rs
View File

@ -176,3 +176,32 @@ fn playing_at_another_rate_leaves_no_holes() {
let tail: Vec<Vec<f32>> = got.iter().map(|c| c[2048..].to_vec()).collect();
assert_eq!(holes(&tail), 0);
}
/// #790: the encoder priming stored as `CodecDelay` (1024 samples: 21.333 ms at 48 kHz, 23.220 ms
/// at 44.1 kHz) is trimmed exactly. The demuxer takes it off the millisecond timestamps rounded to
/// whole milliseconds; starting the first packet there played the track 16 (48 kHz) or 10 (44.1 kHz)
/// samples late.
#[test]
fn matroska_codec_delay_is_trimmed_to_the_sample() {
for rate in [48_000u32, 44_100] {
let (asc, units) = aac(rate);
let want = continuous(&asc, rate, &units);
let delay_ns = (1024 * 1_000_000_000 + rate as i64 / 2) / rate as i64;
let mut spec = TrackSpec::new(TrackKind::Audio, "A_AAC");
spec.codec_private = asc.clone();
spec.audio = Some((rate as f64, 2, None));
spec.codec_delay_ns = delay_ns as u64;
let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![spec], MuxOptions::default()).unwrap();
for (k, u) in units.iter().enumerate() {
let start_ns = (k as i64 * 1024 * 1_000_000_000 + rate as i64 / 2) / rate as i64;
w.write_frame(0, start_ns - delay_ns, true, u, None).unwrap();
}
let bytes: Arc<[u8]> = w.finish().unwrap().into_inner().into();
let src = MkvSource::open("primed.mkv", bytes).unwrap();
let frames = units.len().saturating_sub(2) * 1024;
let got = played(&src, rate, frames);
let trimmed: Vec<Vec<f32>> = want.iter().map(|c| c[1024..].to_vec()).collect();
let (n, first) = mismatches(&got, &trimmed, 0);
assert_eq!(n, 0, "{rate} Hz: {n} samples differ from the decode with the priming trimmed, first (channel, sample) {first:?}");
}
}

585
crates/codecs/src/avi.rs Normal file
View File

@ -0,0 +1,585 @@
//! AVI media source (`filmcraft-avi` demux, GOP-aware video via [`crate::gop`]).
//!
//! Video: Motion JPEG (frames without their own Huffman tables get the standard ones), H.264 and
//! HEVC (Annex B byte streams, our decoders), and uncompressed RGB / YUV ([`RawFormat`]). A frame
//! slot with an empty chunk (a "drop frame") repeats the picture before it. MPEG-4 Part 2
//! (DivX / Xvid) and DV have no decoder yet: their streams are listed and report the codec.
//!
//! Audio: PCM (integer and float) straight from the chunks; MPEG audio (MP3, MP2) and AC-3 frames,
//! which AVI chunks need not follow, are found again across chunk boundaries
//! ([`filmcraft_mpegts::frame_bytes`]) and decoded as packets.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use filmcraft_avi::{AviFile, BitmapInfo, StreamKind, WaveFormat};
use filmcraft_frame::{AudioBuffer, VideoFrame};
use filmcraft_isobmff::{PcmConfig, SampleEntry};
use filmcraft_media::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, SharedSource, VideoStreamInfo};
use filmcraft_time::{FrameRate, Tick};
use crate::audio::{PacketDecoder, decode_pcm};
use crate::gop::{GopCache, VideoSamples};
use crate::video::{RawFormat, RawVideoDecoder, VideoDecoder};
use crate::{CodecError, make_video_decoder};
pub fn sniff(b: &[u8]) -> bool {
filmcraft_avi::sniff(b)
}
#[derive(Clone, Debug, PartialEq)]
enum VCodec {
Jpeg,
H264,
Hevc,
Raw(RawFormat),
/// No decoder: the codec's name.
Unsupported(String),
}
fn fourcc_text(f: [u8; 4]) -> String {
let t: String = f.iter().map(|&c| if c.is_ascii_graphic() { c as char } else { '?' }).collect();
t.trim_end_matches('?').to_string()
}
/// The decoder for a video stream and its label.
fn video_codec(b: &BitmapInfo) -> (VCodec, String) {
if let Some(raw) = RawFormat::from_avi(b.compression, b.bit_count) {
return (VCodec::Raw(raw), raw.label().into());
}
let mut f = b.compression;
f.make_ascii_uppercase();
match &f {
b"MJPG" | b"AVRN" | b"LJPG" | b"JPGL" | b"DMB1" | b"JPEG" | b"MJPA" => (VCodec::Jpeg, "Motion JPEG".into()),
b"H264" | b"X264" | b"AVC1" | b"DAVC" | b"VSSH" => (VCodec::H264, "H.264".into()),
b"HEVC" | b"H265" | b"HEV1" | b"HVC1" | b"X265" => (VCodec::Hevc, "HEVC".into()),
b"DIVX" | b"DX50" | b"XVID" | b"FMP4" | b"MP4V" | b"M4S2" | b"3IV2" | b"DIV3" | b"MP43" | b"MP42" => {
(VCodec::Unsupported("MPEG-4 Part 2 (DivX / Xvid)".into()), "MPEG-4 Part 2 (DivX / Xvid)".into())
}
b"DVSD" | b"DV25" | b"DV50" | b"DVHD" | b"DVSL" | b"CDVC" | b"DVCP" | b"DVPP" => (VCodec::Unsupported("DV".into()), "DV".into()),
_ => {
let name = fourcc_text(b.compression);
(VCodec::Unsupported(format!("\"{name}\"")), name)
}
}
}
/// One picture of the video stream: its frame slot (in `scale / rate` units) and chunk.
#[derive(Clone, Copy, Debug)]
struct VFrame {
pts: i64,
chunk: usize,
key: bool,
}
#[derive(Clone, Debug)]
enum ACodec {
Pcm(PcmConfig),
/// MPEG audio (layer 1-3) or AC-3 frames, found again across chunks.
Framed {
ts: filmcraft_mpegts::Codec,
layer: u8,
},
Unsupported(String),
}
fn audio_codec(w: &WaveFormat) -> (ACodec, String) {
let pcm = |float: bool| {
let bits = w.bits_per_sample;
let cfg = PcmConfig { bits, float, big_endian: false, signed: bits > 8, channels: u32::from(w.channels.max(1)), sample_rate: f64::from(w.sample_rate) };
(ACodec::Pcm(cfg), format!("PCM {bits}-bit{}", if float { " float" } else { "" }))
};
match w.format_tag {
1 if matches!(w.bits_per_sample, 8 | 16 | 24 | 32) => pcm(false),
3 if matches!(w.bits_per_sample, 32 | 64) => pcm(true),
0x55 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 3 }, "MP3".into()),
0x50 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 2 }, "MPEG Audio".into()),
0x2000 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::Ac3, layer: 0 }, "AC-3".into()),
tag => {
let name = format!("audio format 0x{tag:04X}");
(ACodec::Unsupported(name.clone()), name)
}
}
}
/// A slice of a chunk: (chunk, offset in it, length).
type Piece = (usize, usize, usize);
struct AudioState {
decoder: Option<PacketDecoder>,
packets: HashMap<usize, Arc<Vec<Vec<f32>>>>,
order: Vec<usize>,
last_decoded: Option<usize>,
}
struct AviAudio {
stream: usize,
codec: ACodec,
/// The pieces of packet `k` are `pieces[first[k]..first[k + 1]]`.
pieces: Vec<Piece>,
first: Vec<usize>,
/// Packet start positions in sample frames.
starts: Vec<i64>,
state: Mutex<AudioState>,
}
pub struct AviSource {
info: MediaInfo,
bytes: crate::Src,
file: AviFile,
vstream: Option<usize>,
vcodec: VCodec,
/// Pictures in stored (decode) order.
vframes: Vec<VFrame>,
/// `vframes` indices in presentation order (H.264 with B-frames is stored in decode order),
/// and their presentation times.
vorder: Vec<usize>,
vorder_pts: Vec<i64>,
/// Frame slots of the video stream (where the last picture ends).
vslots: i64,
rate: FrameRate,
bottom_up: bool,
video: GopCache,
audios: Vec<AviAudio>,
}
/// Whether an Annex B access unit holds an H.264 IDR or HEVC IRAP picture.
fn random_access(data: &[u8], hevc: bool) -> bool {
filmcraft_bitstream::annexb_nals(data).iter().any(|n| match n.first() {
Some(h) if hevc => (16..=23).contains(&((h >> 1) & 0x3F)),
Some(h) => h & 0x1F == 5,
None => false,
})
}
/// The packets of a framed audio stream: every frame header found in the stream of chunks, and
/// the samples each frame decodes to. Bytes that start no frame are skipped (resynchronisation).
fn split_frames(
file: &AviFile,
bytes: &crate::Src,
s: usize,
ts: &filmcraft_mpegts::Codec,
) -> (Vec<Piece>, Vec<usize>, Vec<u32>, Option<filmcraft_mpegts::FrameInfo>) {
let Some(st) = file.streams.get(s) else { return Default::default() };
// chunk start positions in the concatenated stream
let mut at = Vec::with_capacity(st.chunks.len());
let mut total = 0usize;
for c in &st.chunks {
at.push(total);
total = total.saturating_add(c.size as usize);
}
let locate = |pos: usize| -> Option<(usize, usize)> {
let k = at.partition_point(|&a| a <= pos).checked_sub(1)?;
Some((k, pos - at.get(k)?))
};
let (mut pieces, mut first, mut samples) = (Vec::new(), Vec::new(), Vec::new());
let mut info = None;
// a sliding window over the chunks: `win` holds the stream from `base`
let (mut win, mut base, mut next) = (Vec::<u8>::new(), 0usize, 0usize);
let mut pos = 0usize;
while pos < total {
// keep at least a header's worth (and the frame, when we know its length) in the window
while win.len() < (pos - base) + 4096 && next < st.chunks.len() {
win.extend(file.read_chunk(&bytes, s, next).unwrap_or_default());
next += 1;
}
let Some(here) = win.get(pos - base..) else { break };
match filmcraft_mpegts::frame_bytes(ts, here) {
Some(n) if pos + n <= total => {
if info.is_none() {
info = filmcraft_mpegts::frame_info(ts, here);
}
let k = filmcraft_mpegts::frame_info(ts, here).map_or(0, |f| f.samples);
first.push(pieces.len());
let mut p = pos;
while p < pos + n {
let Some((c, off)) = locate(p) else { break };
let len = (st.chunks.get(c).map_or(0, |ch| ch.size as usize) - off).min(pos + n - p);
if len == 0 {
break;
}
pieces.push((c, off, len));
p += len;
}
samples.push(k);
pos += n;
}
_ if here.len() < 8 && next >= st.chunks.len() => break,
_ => pos += 1,
}
// drop what is behind
if pos - base > 1 << 16 {
let cut = pos - base;
win.drain(..cut.min(win.len()));
base += cut;
}
}
(pieces, first, samples, info)
}
impl AviSource {
pub fn open(name: &str, bytes: Arc<[u8]>) -> crate::Result<Self> {
Self::open_reader(name, Arc::new(filmcraft_media::reader::MemReader(bytes)))
}
/// Open from a random-access reader: the index now, frames and audio on demand.
pub fn open_reader(name: &str, reader: filmcraft_media::SharedReader) -> crate::Result<Self> {
let bytes = crate::Src(reader);
let file = filmcraft_avi::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?;
let vstream = file.streams.iter().position(|s| s.kind == StreamKind::Video && s.video.is_some() && !s.chunks.is_empty());
let astreams: Vec<usize> = file
.streams
.iter()
.enumerate()
.filter(|(_, s)| s.kind == StreamKind::Audio && s.audio.is_some() && !s.chunks.is_empty())
.map(|(i, _)| i)
.take(filmcraft_media::MAX_AUDIO_STREAMS)
.collect();
if vstream.is_none() && astreams.is_empty() {
return Err(CodecError::Unsupported("no playable streams".into()));
}
// video: frame slots, pictures and key frames
let (mut vcodec, mut vframes, mut vslots, mut rate, mut bottom_up, mut video) =
(VCodec::Unsupported(String::new()), Vec::new(), 0i64, FrameRate::new(25, 1), false, None);
if let Some(vs) = vstream.and_then(|i| file.streams.get(i)) {
let b = vs.video.clone().unwrap_or_default();
let (codec, label) = video_codec(&b);
rate = FrameRate::new(i64::from(vs.rate), i64::from(vs.scale));
bottom_up = b.height > 0;
let (w, h) = (b.width.unsigned_abs(), b.height.unsigned_abs());
for (k, c) in vs.chunks.iter().enumerate() {
if c.size == 0 {
continue;
}
let key = match &codec {
VCodec::H264 | VCodec::Hevc if !vs.keyframes_known => {
file.read_chunk(&bytes, vstream.unwrap_or(0), k).is_ok_and(|d| random_access(&d, codec == VCodec::Hevc))
}
VCodec::H264 | VCodec::Hevc => c.key,
_ => true,
};
vframes.push(VFrame { pts: i64::from(vs.start) + k as i64, chunk: k, key });
}
vslots = i64::from(vs.start) + vs.chunks.len() as i64;
// AVI has no timestamps: H.264 with B-frames is stored in decode order, and the
// presentation order comes from the pictures' order counts. A first look at the start
// of the stream decides whether it reorders at all (reading every frame's header is the
// cost of a reordering stream only).
if codec == VCodec::H264
&& let Some(si) = vstream
{
let heads = |n: usize| -> Vec<Vec<u8>> {
vframes
.iter()
.take(n)
.map(|f| {
file.streams
.get(si)
.and_then(|s| s.chunks.get(f.chunk))
.and_then(|c| filmcraft_media::reader::read_range(&*bytes.0, c.offset, (c.size as usize).min(8 << 10)).ok())
.unwrap_or_default()
})
.collect()
};
let reorders = |p: &[i64]| p.windows(2).any(|w| w[1] < w[0]);
if crate::mxf::avc_presentation_order(&heads(300)).is_some_and(|p| reorders(&p))
&& let Some(ranks) = crate::mxf::avc_presentation_order(&heads(vframes.len()))
{
// the pictures keep the slots they occupy (empty drop-frame chunks between them
// stay), shown in their presentation order
let mut slots: Vec<i64> = vframes.iter().map(|f| f.pts).collect();
slots.sort_unstable();
for (f, r) in vframes.iter_mut().zip(ranks) {
if let Some(&p) = usize::try_from(r).ok().and_then(|r| slots.get(r)) {
f.pts = p;
}
}
}
}
let secs = vslots as f64 / rate.as_f64().max(1e-9);
// The rate the pictures play at: a writer that kept a fine time base (ffmpeg copying an
// MP4's 1/600) fills the slots between pictures with empty chunks, every 20th slot
// holding a picture at 30 fps. A regular spacing makes the rate; slots stay the units.
let shown = if vframes.len() > 1 {
let mut gaps: Vec<i64> = vframes.windows(2).map(|w| w[1].chunk as i64 - w[0].chunk as i64).filter(|g| *g > 0).take(1000).collect();
gaps.sort_unstable();
let gap = gaps.get(gaps.len() / 2).copied().unwrap_or(1).max(1);
let regular = gaps.iter().filter(|g| **g == gap).count() * 10 >= gaps.len() * 9;
if gap > 1 && regular { FrameRate::new(i64::from(vs.rate), i64::from(vs.scale).saturating_mul(gap)) } else { rate }
} else {
rate
};
let bitrate = (secs > 0.0).then(|| (vs.chunks.iter().map(|c| u64::from(c.size)).sum::<u64>() as f64 * 8.0 / secs) as u64);
video = Some(VideoStreamInfo {
width: w,
height: h,
frame_rate: shown,
par: (1, 1),
codec: label,
pixel_format: String::new(),
color: crate::stream_color::resolve(w, h, &[]),
has_alpha: false,
bitrate,
hdr: None,
});
vcodec = codec;
}
// audio: packets and their start positions
let mut audios = Vec::new();
let mut audio_streams = Vec::new();
for &s in &astreams {
let Some(st) = file.streams.get(s) else { continue };
let w = st.audio.clone().unwrap_or_default();
let (codec, label) = audio_codec(&w);
let mut sample_rate = w.sample_rate.max(1);
let mut channels = u32::from(w.channels.max(1));
let (pieces, first, starts) = match &codec {
ACodec::Pcm(cfg) => {
let block = usize::from(w.block_align.max(1)).max((usize::from(cfg.bits).div_ceil(8)) * channels as usize);
let mut t = 0i64;
let mut starts = Vec::with_capacity(st.chunks.len());
for c in &st.chunks {
starts.push(t);
t = t.saturating_add((c.size as usize / block) as i64);
}
let pieces: Vec<Piece> = st.chunks.iter().enumerate().map(|(k, c)| (k, 0, c.size as usize)).collect();
let first = (0..pieces.len()).collect();
(pieces, first, starts)
}
ACodec::Framed { ts, .. } => {
let (pieces, first, samples, info) = split_frames(&file, &bytes, s, ts);
if let Some(i) = info {
sample_rate = i.sample_rate.max(1);
channels = i.channels.max(1);
}
let mut t = 0i64;
let starts = samples
.iter()
.map(|&k| {
let s = t;
t = t.saturating_add(i64::from(k));
s
})
.collect();
(pieces, first, starts)
}
ACodec::Unsupported(_) => (Vec::new(), Vec::new(), Vec::new()),
};
// a stream that starts late (dwStart, in scale / rate units)
let delay = (i128::from(st.start) * i128::from(st.scale) * i128::from(sample_rate) / i128::from(st.rate.max(1))) as i64;
let starts: Vec<i64> = starts.into_iter().map(|s: i64| s.saturating_add(delay)).collect();
audio_streams.push(AudioStreamInfo {
sample_rate,
channels,
codec: label,
bits_per_sample: (w.bits_per_sample > 0).then_some(u32::from(w.bits_per_sample)),
});
let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None });
audios.push(AviAudio { stream: s, codec, pieces, first, starts, state });
}
// the longer of the picture and the sound
let vdur = if vslots > 0 { rate.tick_of(vslots) } else { Tick::ZERO };
let adur = audios
.iter()
.zip(&audio_streams)
.filter_map(|(a, i)| {
let last = a.starts.last()?;
let tail = a.starts.len().checked_sub(2).and_then(|k| a.starts.get(k)).map_or(0, |p| last - p);
Some(Tick::from_rational(last + tail, 1, i64::from(i.sample_rate)))
})
.max()
.unwrap_or_default();
let info = MediaInfo {
name: name.to_string(),
kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly },
duration: vdur.max(adur),
video,
audio_streams,
container: "AVI".into(),
start_timecode: None,
file_size: Some(bytes.0.len()),
};
let mut vorder: Vec<usize> = (0..vframes.len()).collect();
vorder.sort_by_key(|&i| vframes.get(i).map_or(0, |f| f.pts));
let vorder_pts = vorder.iter().map(|&i| vframes.get(i).map_or(0, |f| f.pts)).collect();
Ok(Self { info, bytes, file, vstream, vcodec, vframes, vorder, vorder_pts, vslots, rate, bottom_up, video: GopCache::new(None), audios })
}
fn read(&self, stream: usize, chunk: usize) -> crate::Result<Vec<u8>> {
self.file.read_chunk(&self.bytes, stream, chunk).map_err(|e| CodecError::Container(e.to_string()))
}
/// The bytes of audio packet `k`.
fn packet(&self, a: &AviAudio, k: usize) -> crate::Result<Vec<u8>> {
let (from, to) = (
a.first.get(k).copied().ok_or_else(|| CodecError::Container("audio packet out of range".into()))?,
a.first.get(k + 1).copied().unwrap_or(a.pieces.len()),
);
let mut out = Vec::new();
for &(c, off, len) in a.pieces.get(from..to).unwrap_or_default() {
let chunk = self.read(a.stream, c)?;
out.extend_from_slice(chunk.get(off..off + len).ok_or_else(|| CodecError::Container("audio chunk shorter than its index".into()))?);
}
Ok(out)
}
fn audio_packet(&self, a: &AviAudio, st: &mut AudioState, ainfo: &AudioStreamInfo, i: usize) -> crate::Result<Arc<Vec<Vec<f32>>>> {
if let Some(p) = st.packets.get(&i) {
return Ok(p.clone());
}
let data = self.packet(a, i)?;
let decoded = match &a.codec {
ACodec::Pcm(cfg) => decode_pcm(&data, cfg),
ACodec::Unsupported(name) => return Err(CodecError::Unsupported(name.clone())),
ACodec::Framed { ts, layer } => {
if st.decoder.is_none() {
st.decoder = Some(match ts {
filmcraft_mpegts::Codec::Ac3 => PacketDecoder::ac3()?,
_ => PacketDecoder::mpeg_audio(*layer, ainfo.sample_rate)?,
});
}
let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?;
// non-sequential access: reset and prime with the packet before (the bit reservoir)
if st.last_decoded.is_none_or(|l| l + 1 != i) {
d.reset();
if let Some(prev) = i.checked_sub(1).and_then(|j| self.packet(a, j).ok()) {
let _ = d.decode(&prev, 0);
}
}
let start = a.starts.get(i).copied().unwrap_or(0).max(0) as u64;
let r = d.decode(&data, start);
st.last_decoded = Some(i);
r.unwrap_or_default()
}
};
let p = Arc::new(decoded);
st.packets.insert(i, p.clone());
st.order.push(i);
if st.order.len() > 4096 {
let old = st.order.remove(0);
st.packets.remove(&old);
}
Ok(p)
}
}
/// The video stream as a [`VideoSamples`] table.
struct AviVideo<'a> {
src: &'a AviSource,
stream: usize,
}
impl VideoSamples for AviVideo<'_> {
fn count(&self) -> usize {
self.src.vframes.len()
}
fn pts(&self, i: usize) -> i64 {
self.src.vframes.get(i).map_or(0, |f| f.pts)
}
fn sync_before(&self, i: usize) -> usize {
let i = i.min(self.src.vframes.len().saturating_sub(1));
self.src.vframes.get(..=i).and_then(|f| f.iter().rposition(|f| f.key)).unwrap_or(0)
}
fn sample_at(&self, t: i64) -> Option<usize> {
// the picture shown in slot `t`: the last one at or before it (empty chunks repeat it)
if t >= self.src.vslots {
return None;
}
let k = self.src.vorder_pts.partition_point(|&p| p <= t).checked_sub(1)?;
self.src.vorder.get(k).copied()
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
let f = self.src.vframes.get(i).ok_or_else(|| CodecError::Container("frame out of range".into()))?;
self.src.read(self.stream, f.chunk)
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
let v = self.src.info.video.as_ref().ok_or_else(|| CodecError::Unsupported("no video stream".into()))?;
let (w, h) = (v.width.min(u32::from(u16::MAX)) as u16, v.height.min(u32::from(u16::MAX)) as u16);
match &self.src.vcodec {
VCodec::Jpeg => make_video_decoder(&SampleEntry::jpeg(w, h)),
VCodec::H264 => Ok(Box::new(crate::video::H264Decoder::annexb())),
VCodec::Hevc => Ok(Box::new(crate::video::HevcDecoder::annexb())),
VCodec::Raw(f) => Ok(Box::new(RawVideoDecoder::new(*f, v.width, v.height, self.src.bottom_up)?)),
VCodec::Unsupported(name) => Err(CodecError::Unsupported(format!("no decoder for {name} video"))),
}
}
}
impl MediaSource for AviSource {
fn info(&self) -> &MediaInfo {
&self.info
}
fn video_frame(&self, req: FrameRequest) -> Result<Arc<VideoFrame>, MediaError> {
let s = self.vstream.ok_or(MediaError::NoStream("video"))?;
let t = req.time.max(Tick::ZERO);
let slot = |t: Tick| self.rate.frame_at(t);
let late = filmcraft_media::cancel::catch_up().map(|m| slot(t - m));
Ok(self.video.frame_late(&AviVideo { src: self, stream: s }, slot(t), late)?)
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
self.audio_stream(0, start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?;
if let ACodec::Unsupported(name) = &a.codec {
return Err(MediaError::Unsupported(format!("{name} audio")));
}
let ch = ainfo.channels.max(1) as usize;
let ratio = f64::from(ainfo.sample_rate) / f64::from(sample_rate.max(1));
let s0 = (start as f64 * ratio).floor() as i64;
let need = (frames as f64 * ratio).ceil() as i64 + 2;
let mut src: Vec<Vec<f32>> = vec![vec![0.0; need.max(0) as usize]; ch];
let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner());
let mut i = a.starts.partition_point(|&x| x <= s0).saturating_sub(1);
while let Some(&pk_start) = a.starts.get(i) {
if pk_start >= s0 + need {
break;
}
let pk = self.audio_packet(a, &mut st, ainfo, i)?;
for (c, dst) in src.iter_mut().enumerate() {
let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue };
for (k, v) in chan.iter().enumerate() {
let pos = pk_start + k as i64 - s0;
if let Some(d) = usize::try_from(pos).ok().and_then(|p| dst.get_mut(p)) {
*d = *v;
}
}
}
i += 1;
}
drop(st);
let frac = s0 as f64 - start as f64 * ratio;
let mut out = AudioBuffer::silence(sample_rate, ch, frames);
for (c, o) in out.channels.iter_mut().enumerate() {
let Some(s) = src.get(c) else { continue };
for (k, d) in o.iter_mut().enumerate() {
let pos = k as f64 * ratio - frac;
let i0 = pos.floor().max(0.0) as usize;
let f = (pos - i0 as f64) as f32;
let a = s.get(i0).copied().unwrap_or(0.0);
let b = s.get(i0 + 1).copied().unwrap_or(a);
*d = a + (b - a) * f;
}
}
Ok(out)
}
}
pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource, MediaError>> {
if !sniff(&bytes) {
return None;
}
Some(AviSource::open(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
/// [`filmcraft_media::ReaderOpener`] for AVI.
pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedReader) -> Option<Result<SharedSource, MediaError>> {
if !sniff(head) {
return None;
}
Some(AviSource::open_reader(name, reader.clone()).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}

232
crates/codecs/src/avi_tests.rs Normal file
View File

@ -0,0 +1,232 @@
//! AVI files made by ffmpeg (skipped without it), checked against ffmpeg's decode of the same file:
//! bit-exact where the decoder is (H.264, uncompressed video, PCM), within decoder tolerance where
//! it differs (JPEG, MP3 / MP2 / AC-3).
use std::path::Path;
use std::process::Command;
use std::sync::Arc;
use filmcraft_media::FrameRequest;
use crate::tests::{ffmpeg, fixture_path, reference_yuv, yuv_bytes};
const PIC: [&str; 4] = ["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2"];
const TONE: [&str; 4] = ["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=2"];
fn avi(name: &str, video: &[&str], audio: &[&str]) -> Option<(std::path::PathBuf, filmcraft_media::SharedSource)> {
let mut args: Vec<&str> = PIC.to_vec();
args.extend_from_slice(&TONE);
args.extend_from_slice(video);
args.extend_from_slice(audio);
let path = fixture_path(name, &args)?;
let bytes: Arc<[u8]> = std::fs::read(&path).ok()?.into();
Some((path.clone(), crate::open_bytes(name, bytes).unwrap_or_else(|e| panic!("{name}: {e}"))))
}
/// ffmpeg's decode of the first audio stream as interleaved f32 at its own rate.
fn reference_audio(path: &Path) -> Vec<f32> {
let out = Command::new(ffmpeg().unwrap())
.args(["-v", "error", "-i"])
.arg(path)
.args(["-map", "0:a:0", "-f", "f32le", "-c:a", "pcm_f32le", "-"])
.output()
.unwrap();
out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
}
/// The first `n` mono samples of stream 0 as FilmCraft decodes them.
fn ours(src: &filmcraft_media::SharedSource, n: usize) -> Vec<f32> {
src.audio(0, n, 48_000).unwrap().channels[0].clone()
}
fn check_info(name: &str, src: &filmcraft_media::SharedSource, vcodec: &str, acodec: &str) {
let info = src.info();
assert_eq!(info.container, "AVI", "{name}");
let v = info.video.as_ref().unwrap();
assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, vcodec), "{name}");
assert_eq!(v.frame_rate.as_f64(), 25.0, "{name}");
let a = &info.audio_streams[0];
assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (48_000, 1, acodec), "{name}");
let secs = info.duration.seconds();
assert!((secs - 2.0).abs() < 0.1, "{name}: duration {secs}");
}
#[test]
fn h264_frames_are_bit_exact_with_random_access_and_mp3_plays() {
let Some((path, src)) =
avi("avi_h264_mp3.avi", &["-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p"], &["-c:a", "libmp3lame", "-b:a", "128k"])
else {
return;
};
check_info("h264", &src, "H.264", "MP3");
let reference = reference_yuv(&path, "yuv420p").unwrap();
let frame = reference.len() / 50;
let rate = src.info().frame_rate();
for k in [37, 3, 24, 49, 0, 13, 12, 11, 30, 31, 32] {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "H.264 frame {k} differs from ffmpeg");
}
// MP3 (a different decoder): the tone, in time with ffmpeg's decode
let r = reference_audio(&path);
let a = ours(&src, 48_000);
let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200);
// ffmpeg's test tone is at 1/8 of full scale
assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "MP3: tone {rms_ref}, difference {rms_diff}");
// random access into the middle matches sequential decoding
let mid = src.audio(24_000, 4_800, 48_000).unwrap().channels[0].clone();
let worst = mid.iter().zip(&a[24_000..28_800]).map(|(x, y)| (x - y).abs()).fold(0.0, f32::max);
assert!(worst < 0.01, "MP3 seek: {worst}");
}
fn rms_and_diff(reference: &[f32], ours: &[f32], range: std::ops::Range<usize>) -> (f32, f32) {
let n = range.len() as f32;
let rms = (range.clone().map(|i| reference[i].powi(2)).sum::<f32>() / n).sqrt();
let diff = (range.map(|i| (reference[i] - ours[i]).powi(2)).sum::<f32>() / n).sqrt();
(rms, diff)
}
#[test]
fn mjpeg_and_pcm() {
let Some((path, src)) = avi("avi_mjpeg_pcm.avi", &["-c:v", "mjpeg", "-q:v", "3", "-pix_fmt", "yuvj420p"], &["-c:a", "pcm_s16le"]) else { return };
check_info("mjpeg", &src, "Motion JPEG", "PCM 16-bit");
let reference = reference_yuv(&path, "rgba").unwrap();
let frame = reference.len() / 50;
let rate = src.info().frame_rate();
for k in [10, 0, 49] {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap().to_rgba8();
let r = &reference[k * frame..(k + 1) * frame];
let mean = f.iter().zip(r).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::<u32>() as f64 / f.len() as f64;
// another JPEG decoder and chroma upsampling than ffmpeg's: close, not identical
assert!(mean < 6.0, "JPEG frame {k}: mean difference {mean}");
}
// PCM: exactly ffmpeg's samples
let r = reference_audio(&path);
assert_eq!(ours(&src, 48_000), r[..48_000].to_vec());
}
#[test]
fn uncompressed_video_is_bit_exact() {
for (name, pix, codec, reference_fmt) in [
("avi_bgr24.avi", "bgr24", "Uncompressed RGB 24-bit", "rgba"),
("avi_yuyv.avi", "yuyv422", "Uncompressed YUV 4:2:2", "yuv422p"),
("avi_i420.avi", "yuv420p", "Uncompressed YUV 4:2:0", "yuv420p"),
("avi_nv12.avi", "nv12", "Uncompressed YUV 4:2:0", "yuv420p"),
] {
let Some((path, src)) = avi(name, &["-c:v", "rawvideo", "-pix_fmt", pix], &["-c:a", "pcm_s16le"]) else { return };
check_info(name, &src, codec, "PCM 16-bit");
let reference = reference_yuv(&path, reference_fmt).unwrap();
let frame = reference.len() / 50;
let rate = src.info().frame_rate();
for k in [7, 0, 49] {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
let got = if reference_fmt == "rgba" { f.to_rgba8() } else { yuv_bytes(&f) };
assert!(got == reference[k * frame..(k + 1) * frame], "{name}: frame {k} differs from ffmpeg");
}
}
}
#[test]
fn mp2_and_ac3_audio() {
for (name, codec, label) in [("avi_mp2.avi", "mp2", "MPEG Audio"), ("avi_ac3.avi", "ac3", "AC-3")] {
let Some((path, src)) = avi(name, &["-c:v", "mjpeg"], &["-c:a", codec, "-b:a", "192k"]) else { return };
assert_eq!(src.info().audio_streams[0].codec, label, "{name}");
let r = reference_audio(&path);
let a = ours(&src, 48_000);
let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200);
assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "{name}: tone {rms_ref}, difference {rms_diff}");
}
}
/// Dropped frames are stored as empty chunks: their slot shows the picture before.
#[test]
fn drop_frames_hold_the_previous_picture() {
let args = [
"-f",
"lavfi",
"-i",
"testsrc2=s=160x120:r=25:d=2",
"-vf",
"select='not(eq(n\\,10))'",
"-fps_mode",
"passthrough",
"-c:v",
"rawvideo",
"-pix_fmt",
"bgr24",
];
let Some(path) = fixture_path("avi_drop.avi", &args) else { return };
let src = crate::open_bytes("avi_drop.avi", std::fs::read(&path).unwrap().into()).unwrap();
let rate = src.info().frame_rate();
let at = |k: i64| src.video_frame(FrameRequest::full(rate.tick_of(k))).unwrap().to_rgba8();
assert!(at(10) == at(9), "the dropped slot holds frame 9");
assert!(at(11) != at(9));
}
/// DivX / Xvid has no decoder yet: the file opens, the codec is named, the picture reports it, and
/// the sound plays.
#[test]
fn mpeg4_part2_is_named_and_its_audio_plays() {
let Some((_, src)) = avi("avi_xvid.avi", &["-c:v", "mpeg4", "-vtag", "XVID"], &["-c:a", "pcm_s16le"]) else { return };
assert_eq!(src.info().video.as_ref().unwrap().codec, "MPEG-4 Part 2 (DivX / Xvid)");
let e = src.video_frame(FrameRequest::full(filmcraft_time::Tick::ZERO)).unwrap_err().to_string();
assert!(e.contains("MPEG-4 Part 2"), "{e}");
assert!(ours(&src, 4_800).iter().any(|s| s.abs() > 0.1));
}
/// A Motion JPEG frame without Huffman tables (the AVI1 convention) decodes with the standard ones:
/// exactly as the same frame with them.
#[test]
fn jpeg_without_huffman_tables_uses_the_standard_ones() {
let Some(path) = fixture_path(
"default_huffman.jpg",
&["-f", "lavfi", "-i", "testsrc2=s=160x120", "-frames:v", "1", "-c:v", "mjpeg", "-huffman", "default", "-q:v", "3"],
) else {
return;
};
let jpeg = std::fs::read(&path).unwrap();
// remove every DHT segment
let mut stripped = jpeg[..2].to_vec();
let mut at = 2;
while at + 4 <= jpeg.len() && jpeg[at] == 0xFF && jpeg[at + 1] != 0xDA {
let len = u16::from_be_bytes([jpeg[at + 2], jpeg[at + 3]]) as usize;
if jpeg[at + 1] != 0xC4 {
stripped.extend_from_slice(&jpeg[at..at + 2 + len]);
}
at += 2 + len;
}
stripped.extend_from_slice(&jpeg[at..]);
assert!(stripped.len() < jpeg.len() && !stripped.windows(2).any(|w| w == [0xFF, 0xC4]));
use crate::video::VideoDecoder;
let mut d = crate::video::MjpegDecoder;
let a = d.decode(&jpeg, 0).unwrap().remove(0).frame.to_rgba8();
let b = d.decode(&stripped, 0).unwrap().remove(0).frame.to_rgba8();
assert!(a == b);
// the inserted segment is 418 bytes and well-formed: 4 tables of 12, 12, 162, 162 codes
let with = crate::video::with_default_huffman(&stripped);
assert_eq!(with.len(), stripped.len() + 420);
}
/// H.264 with B-frames copied from an MP4 whose time base is 1/600 (ffmpeg keeps it): 600 frame
/// slots a second, a picture in every 24th at 25 fps, empty drop chunks between. The pictures stay
/// at their times in presentation order, and the stream reports the rate they play at.
#[test]
fn copied_h264_keeps_its_timing_with_a_fine_time_base() {
let mp4 =
["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2", "-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p", "-video_track_timescale", "600"];
let Some(src_mp4) = fixture_path("avi_copy_source.mp4", &mp4) else { return };
let src_path = src_mp4.to_string_lossy().into_owned();
let Some(path) = fixture_path("avi_copy_600.avi", &["-i", &src_path, "-c:v", "copy", "-bsf:v", "h264_mp4toannexb", "-f", "avi"]) else { return };
let src = crate::open_bytes("avi_copy_600.avi", std::fs::read(&path).unwrap().into()).unwrap();
let v = src.info().video.clone().unwrap();
assert_eq!(v.frame_rate.as_f64(), 25.0, "the rate the pictures play at, not the 600 slots");
let reference = reference_yuv(&path, "yuv420p").unwrap();
let frame = reference.len() / 50;
for k in [37usize, 3, 24, 49, 0, 13, 12, 11, 30] {
let at = filmcraft_time::Tick::from_rational(k as i64, 1, 25);
let f = src.video_frame(FrameRequest::full(at)).unwrap();
assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "picture {k} at {:.2} s differs from ffmpeg", k as f64 / 25.0);
}
// between pictures, the one before holds
let mid = src.video_frame(FrameRequest::full(filmcraft_time::Tick::from_rational(1, 1, 50))).unwrap();
assert!(yuv_bytes(&mid) == reference[0..frame]);
}

303
crates/codecs/src/gop.rs
View File

@ -109,8 +109,9 @@ impl std::ops::Sub for GopStats {
}
}
/// A container's video sample table, in decode (file) order.
pub trait VideoSamples {
/// A container's video sample table, in decode (file) order. `Sync`: a cache decodes on its own
/// threads ([`isolated`]).
pub trait VideoSamples: Sync {
fn count(&self) -> usize;
/// Presentation timestamp of sample `i` (track units).
fn pts(&self, i: usize) -> i64;
@ -152,6 +153,10 @@ struct State {
bytes: usize,
/// When a frame was last asked of this cache.
last_used: Instant,
/// Presentation pts of the frame the last request wanted: eviction keeps a window around it
/// (not around the newest decoded picture). Walking a long GOP dumps pictures in decode
/// order; a window around those would throw away the playhead frame and force another walk.
pin: i64,
/// Only background requests (thumbnails, [`filmcraft_media::cancel::with_background`]) have
/// used the decoder since it was made: it is given up as soon as another background request
/// is done, whatever the caches' recency.
@ -437,6 +442,57 @@ const MIN_FRAMES: usize = 64;
/// rather than restarting at the sync sample.
const CONTINUE_THROUGH: usize = 48;
/// Threads that run the caches' decoding (see [`isolated`]); `None` where none can be made.
#[cfg(not(target_arch = "wasm32"))]
fn decode_pool() -> Option<&'static rayon::ThreadPool> {
static POOL: std::sync::OnceLock<Option<rayon::ThreadPool>> = std::sync::OnceLock::new();
POOL.get_or_init(|| {
let n = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
rayon::ThreadPoolBuilder::new().num_threads(n).thread_name(|i| format!("decode-{i}")).build().ok()
})
.as_ref()
}
/// Run a cache's decoding, its lock held by the caller, on [`decode_pool`], the caller waiting
/// without doing other rayon work meanwhile (#480).
///
/// That decoding starts rayon work (a decoder's slices, cropping and rotating the frames). On the
/// pool of the render jobs that ask the cache for frames, a worker that took part of it could,
/// waiting inside it, pick up another of those jobs and block on the lock: the lock holder then
/// waits for its work on the blocked worker's stack, and that worker for the lock, for good (an
/// export of rotated iPhone clips stopped making frames). On the decode pool that work only ever
/// runs beside other caches' decoding, which never waits for a cache's lock. A caller that is a
/// rayon worker would also run other jobs of its pool while it waits for the decode pool, some
/// of them asking for a cache's lock while it holds this one: it waits on a plain thread instead.
fn isolated<T: Send>(f: impl FnOnce() -> crate::Result<T> + Send) -> crate::Result<T> {
let ctx = filmcraft_media::cancel::Context::current();
let job = move || ctx.run(f);
#[cfg(not(target_arch = "wasm32"))]
if let Some(pool) = decode_pool() {
if rayon::current_thread_index().is_none() {
return pool.install(job);
}
let slot = Mutex::new(Some(job));
let waited = std::thread::scope(|scope| {
let spawned = std::thread::Builder::new().name("decode-wait".into()).spawn_scoped(scope, || {
let job = slot.lock().unwrap_or_else(PoisonError::into_inner).take();
job.map(|job| pool.install(job))
});
spawned.map(|h| h.join())
});
return match waited {
Ok(Ok(Some(r))) => r,
Ok(Err(_)) => Err(CodecError::Decode("internal error while decoding".into())),
// no thread to wait on: decode here, as before
Ok(Ok(None)) | Err(_) => match slot.into_inner().unwrap_or_else(PoisonError::into_inner) {
Some(job) => job(),
None => Err(CodecError::Decode("internal error while decoding".into())),
},
};
}
job()
}
impl GopCache {
pub fn new(explicit_color: Option<ColorInfo>) -> Self {
// unit tests count decoder restarts: each of their caches gets a pool of its own
@ -457,6 +513,7 @@ impl GopCache {
drafts: Default::default(),
bytes: 0,
last_used: Instant::now(),
pin: i64::MIN,
background: false,
}),
pool: pool.clone(),
@ -494,7 +551,11 @@ impl GopCache {
}
fn store(&self, st: &mut State, pts: i64, f: VideoFrame, draft: bool) {
let f = self.finish(f);
self.store_finished(st, pts, self.finish(f), draft);
}
/// Cache a frame [`Self::finish`] has been applied to.
fn store_finished(&self, st: &mut State, pts: i64, f: VideoFrame, draft: bool) {
st.last_used = Instant::now();
if draft {
st.drafts.insert(pts);
@ -509,10 +570,19 @@ impl GopCache {
st.bytes -= old.byte_size();
filmcraft_frame::pool::recycle(old);
}
// evict frames far from the most recent (keep a window around the working position)
// Evict frames far from the request (pin), not the newest decode. A frame-threaded
// decoder can emit a burst past the playhead; pivoting on those pts used to drop the
// wanted picture and walk the GOP again.
let pivot = if st.pin != i64::MIN { st.pin } else { pts };
while st.bytes > budget && st.frames.len() > 2 {
let (Some(&first), Some(&last)) = (st.frames.keys().next(), st.frames.keys().next_back()) else { break };
let victim = if pts - first > last - pts { first } else { last };
let mut victim = if pivot - first > last - pivot { first } else { last };
if victim == pivot || victim == pts {
victim = if victim == first { last } else { first };
if victim == pivot || victim == pts {
break;
}
}
if let Some(v) = st.frames.remove(&victim) {
st.drafts.remove(&victim);
st.bytes -= v.byte_size();
@ -583,6 +653,7 @@ impl GopCache {
let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner());
let _decoding = DecodingGuard::enter(me);
st.last_used = Instant::now();
st.pin = want_pts;
let background = filmcraft_media::cancel::background();
if let Some(f) = st.cached(want_pts, draft).cloned() {
HITS.fetch_add(1, Ordering::Relaxed);
@ -616,6 +687,22 @@ impl GopCache {
if st.intra {
return self.intra_frame(st, s, i, want_pts);
}
// with the lock held: see `isolated`
let st = &mut *st;
isolated(|| self.decode_until(st, s, i, want_pts, late_before, draft))
}
/// Decode (the lock held) until the frame with `want_pts` (sample `i`) is cached; return it.
fn decode_until(
&self,
st: &mut State,
s: &dyn VideoSamples,
i: usize,
want_pts: i64,
late_before: Option<i64>,
draft: bool,
) -> crate::Result<Arc<VideoFrame>> {
let n = s.count();
let mut key = decode_start(s, i);
// Continue the running decoder when it has passed the wanted sample's sync sample and
// either has not reached the sample yet or has been fed it without outputting it yet
@ -669,14 +756,14 @@ impl GopCache {
let out = timed(|| decoder.decode(&data, s.pts(k)))?;
st.next += 1;
DECODED.fetch_add(1, Ordering::Relaxed);
self.store_output(&mut st, out);
self.store_output(st, out);
if st.cached(want_pts, draft).is_some() {
break;
}
}
if st.cached(want_pts, draft).is_none() {
let out = st.decoder.as_mut().map(|d| timed(|| d.flush())).unwrap_or_default();
self.store_output(&mut st, out);
self.store_output(st, out);
st.next = usize::MAX;
}
// nearest decoded frame at or before the wanted pts (robust to decoder pts quirks)
@ -761,14 +848,17 @@ impl GopCache {
DECODED.fetch_add(1, Ordering::Relaxed);
timed(|| dec.decode(&data, want_pts))
});
// crop and rotate before taking the lock: that work runs on rayon (see `isolated`)
let res = res.map(|out| {
count_frames(&out);
out.into_iter().map(|d| (d.pts, self.finish(d.frame), d.draft)).collect::<Vec<_>>()
});
let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner());
if st.spare.len() < 16 {
st.spare.push(dec);
}
let res = res?;
count_frames(&res);
for d in res {
self.store(&mut st, d.pts, d.frame, d.draft);
for (pts, frame, draft) in res? {
self.store_finished(&mut st, pts, frame, draft);
}
st.at_or_before(want_pts, true).ok_or_else(|| CodecError::Decode("frame not produced".into()))
}
@ -900,6 +990,77 @@ mod tests {
assert!(s.decodes.load(Ordering::Relaxed) <= 60 + 24 + 1);
}
/// A decoder that, like a slice-threaded one, splits each picture's work over rayon.
struct ParDec(Box<dyn VideoDecoder>);
impl VideoDecoder for ParDec {
fn decode(&mut self, sample: &[u8], pts: i64) -> crate::Result<Vec<DecodedFrame>> {
use rayon::prelude::*;
let rows: u64 = (0..64u32)
.into_par_iter()
.with_max_len(1)
.map(|r| (0..20_000u64).fold(u64::from(r), |a, x| a.wrapping_mul(31).wrapping_add(x)))
.reduce(|| 0, u64::wrapping_add);
std::hint::black_box(rows);
self.0.decode(sample, pts)
}
fn flush(&mut self) -> Vec<DecodedFrame> {
self.0.flush()
}
fn reset(&mut self) {
self.0.reset()
}
fn name(&self) -> &str {
"par"
}
}
struct ParSamples(Samples);
impl VideoSamples for ParSamples {
fn count(&self) -> usize {
self.0.count()
}
fn pts(&self, i: usize) -> i64 {
self.0.pts(i)
}
fn sync_before(&self, i: usize) -> usize {
self.0.sync_before(i)
}
fn sample_at(&self, t: i64) -> Option<usize> {
self.0.sample_at(t)
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
self.0.read(i)
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
Ok(Box::new(ParDec(self.0.make_decoder()?)))
}
}
/// #480: render jobs on rayon asking one source for frames while its decoding (a decoder's
/// slices, rotating the frames) runs on rayon too. A worker that took part of that work and,
/// waiting inside it, picked up another frame request blocked on the cache's lock for good:
/// an export of rotated iPhone clips stopped making frames.
#[test]
fn rayon_frame_requests_while_decoding_on_rayon_never_deadlock() {
use rayon::prelude::*;
let s = Arc::new(ParSamples(samples(240, 30, 4, false)));
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for round in 0..40usize {
// export-like: many frame workers, frames of several GOPs of a fresh source at once
let c = GopCache::new(None).with_rotation(1);
(0..96usize).into_par_iter().with_max_len(1).for_each(|k| {
let i = (k * 37 + round * 53) % 240;
assert_eq!(index_of(&c.frame(&*s, i as i64 * 1000).expect("frame")), i);
});
}
let _ = tx.send(());
});
assert!(rx.recv_timeout(Duration::from_secs(120)).is_ok(), "frame requests deadlocked");
}
#[test]
fn running_decoder_continues_through_a_nearby_sync_sample() {
let s = samples(200, 30, 0, false);
@ -919,19 +1080,27 @@ mod tests {
fn intra_frames_decode_in_parallel_once_each() {
let s = Arc::new(samples(64, 1, 0, true));
let c = Arc::new(GopCache::new(None));
std::thread::scope(|scope| {
for t in 0..4usize {
let (s, c) = (s.clone(), c.clone());
scope.spawn(move || {
for k in 0..64usize {
let i = (k * 7 + t * 16) % 64;
assert_eq!(index_of(&c.frame(&*s, i as i64 * 1000).expect("frame")), i);
}
});
}
});
// a frame two threads miss at the same moment may decode twice; nothing more
assert!(s.decodes.load(Ordering::Relaxed) <= 64 + 4 * 4);
// First decode 64 distinct cold frames: each of the four workers owns
// 16 indices. Multiplication by seven permutes them without overlap.
// Then let every worker request all 64 cached frames in overlapping order.
// Concurrent misses for the same cold frame may legitimately decode twice,
// so they cannot support a scheduler-independent duplicate allowance.
for requests_per_worker in [16usize, 64] {
let start = Arc::new(std::sync::Barrier::new(4));
std::thread::scope(|scope| {
for t in 0..4usize {
let (s, c, start) = (s.clone(), c.clone(), start.clone());
scope.spawn(move || {
start.wait();
for k in 0..requests_per_worker {
let i = (k * 7 + t * 16) % 64;
assert_eq!(index_of(&c.frame(&*s, i as i64 * 1000).expect("frame")), i);
}
});
}
});
assert_eq!(s.decodes.load(Ordering::Relaxed), 64, "cold frames decode once; warm requests hit the cache");
}
assert_eq!(s.resets.load(Ordering::Relaxed), 0);
}
@ -1364,4 +1533,90 @@ mod tests {
}
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
}
/// Many pictures in one GOP; a decoder that emits a burst past the wanted sample (frame-
/// threaded H.264). Eviction used to pivot on the newest pts and drop the playhead frame.
struct Burst {
n: usize,
/// Pictures held before a decode dumps them all.
burst: usize,
held: VecDeque<i64>,
resets: Arc<AtomicUsize>,
decodes: Arc<AtomicUsize>,
}
impl VideoDecoder for Burst {
fn decode(&mut self, _sample: &[u8], pts: i64) -> crate::Result<Vec<DecodedFrame>> {
self.decodes.fetch_add(1, Ordering::Relaxed);
self.held.push_back(pts);
if self.held.len() < self.burst {
return Ok(Vec::new());
}
Ok(self.held.drain(..).map(|p| picture((p, false))).collect())
}
fn flush(&mut self) -> Vec<DecodedFrame> {
self.held.drain(..).map(|p| picture((p, false))).collect()
}
fn reset(&mut self) {
self.resets.fetch_add(1, Ordering::Relaxed);
self.held.clear();
}
fn name(&self) -> &str {
"burst"
}
}
impl VideoSamples for Burst {
fn count(&self) -> usize {
self.n
}
fn pts(&self, i: usize) -> i64 {
i as i64 * 1000
}
fn sync_before(&self, _i: usize) -> usize {
0
}
fn sample_at(&self, t: i64) -> Option<usize> {
let i = (t / 1000) as usize;
(i < self.n).then_some(i)
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
Ok(vec![i as u8])
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
Ok(Box::new(Burst { n: self.n, burst: self.burst, held: VecDeque::new(), resets: self.resets.clone(), decodes: self.decodes.clone() }))
}
}
#[test]
fn long_gop_seek_keeps_the_target_and_does_not_walk_twice() {
let s = Burst { n: 200, burst: 90, held: VecDeque::new(), resets: Default::default(), decodes: Default::default() };
let mut c = GopCache::new(None);
c.budget = 0;
// Want a picture early in the GOP; the burst dumps 90 pictures at once (past MIN_FRAMES).
assert_eq!(index_of(&c.frame(&s, 10_000).expect("frame")), 10);
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
let decodes = s.decodes.load(Ordering::Relaxed);
assert!(decodes <= 90 + 1, "walked {decodes} samples");
// The wanted picture is still cached: asking again must not restart or re-feed.
assert_eq!(index_of(&c.frame(&s, 10_000).expect("cached")), 10);
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
assert_eq!(s.decodes.load(Ordering::Relaxed), decodes);
// Playback continues from the same run.
assert_eq!(index_of(&c.frame(&s, 11_000).expect("frame")), 11);
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
}
#[test]
fn long_gop_continue_does_not_restart_for_the_next_frame() {
let s = samples(300, 250, 8, false);
let mut c = GopCache::new(None);
c.budget = 0;
assert_eq!(index_of(&c.frame(&s, 80_000).expect("frame")), 80);
let resets = s.resets.load(Ordering::Relaxed);
let decodes = s.decodes.load(Ordering::Relaxed);
assert_eq!(index_of(&c.frame(&s, 81_000).expect("frame")), 81);
assert_eq!(s.resets.load(Ordering::Relaxed), resets);
assert!(s.decodes.load(Ordering::Relaxed) - decodes <= 8 + 2);
}
}

4
crates/codecs/src/hw.rs
View File

@ -258,7 +258,7 @@ impl NalStreamInfo {
bit_depth_chroma: sps.bit_depth_chroma,
interlaced: !sps.frame_mbs_only,
profile_idc: sps.profile_idc,
color: vui_color(crop.2, crop.3, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
color: vui_color(crop.2, crop.3, vui.colour_primaries, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
par: sar_par(vui.sar),
// as the software decoder's DPB (`max_reorder`)
reorder: sps.max_num_reorder_frames().min(sps.max_dpb_frames()),
@ -304,7 +304,7 @@ impl NalStreamInfo {
bit_depth_chroma: sps.bit_depth_chroma,
interlaced: false,
profile_idc: sps.ptl.profile_idc,
color: vui_color(crop.2, crop.3, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
color: vui_color(crop.2, crop.3, vui.colour_primaries, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
par: sar_par(vui.sar),
reorder: sps.max_num_reorder as usize,
})

32
crates/codecs/src/lib.rs
View File

@ -13,6 +13,8 @@
//! - [`MpegSource`]: MPEG-2 transport streams (`.ts`, `.m2ts`, `.mts`), program streams (`.mpg`,
//! `.vob`, `.mod`) and MPEG-1/2 video elementary streams: MPEG-1/2, H.264 and HEVC video; MPEG
//! audio, AAC (ADTS / LATM), AC-3 and LPCM.
//! - [`AviSource`]: AVI (OpenDML included): Motion JPEG, H.264, HEVC and uncompressed video; PCM,
//! MP3 / MP2 and AC-3 audio.
//! - [`AudioFileSource`]: standalone compressed audio files (MP3, FLAC, AIFF, …).
//! - [`openers`]: the openers to register with the engine's media pool.
@ -20,6 +22,7 @@
pub mod apv;
pub mod audio;
pub mod avi;
pub mod gop;
pub mod hw;
mod hw_frame;
@ -36,6 +39,7 @@ use std::sync::{Arc, RwLock};
pub use apv::ApvSource;
pub use audio::AudioFileSource;
pub use avi::AviSource;
pub use gop::{FRAME_BUDGET, GopStats, cached_bytes, gop_stats, live_decoders};
pub use mkv::MkvSource;
pub use mp4::Mp4Source;
@ -70,9 +74,10 @@ impl From<CodecError> for filmcraft_media::MediaError {
}
/// Our own (pure-Rust) decoders, in the order they are tried.
const BUILTIN_FACTORIES: [VideoDecoderFactory; 9] = [
const BUILTIN_FACTORIES: [VideoDecoderFactory; 10] = [
video::h264_factory,
video::hevc_factory,
video::vp8_factory,
video::vp9_factory,
video::av1_factory,
video::apv_factory,
@ -125,12 +130,22 @@ pub fn software_video_decoder(entry: &filmcraft_isobmff::SampleEntry) -> Result<
/// Openers for the engine's media pool (MP4/MOV, Matroska/WebM, MXF, Ogg Opus/Vorbis, MPEG TS/PS and
/// MPEG-1/2 video elementary streams, APV raw bitstreams, standalone audio).
pub fn openers() -> Vec<filmcraft_media::Opener> {
vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, audio::opener]
vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, avi::opener, audio::opener]
}
fn reader_registry() -> &'static RwLock<Vec<filmcraft_media::ReaderOpener>> {
static R: std::sync::OnceLock<RwLock<Vec<filmcraft_media::ReaderOpener>>> = std::sync::OnceLock::new();
R.get_or_init(|| RwLock::new(vec![mp4::reader_opener, mkv::reader_opener, mxf::reader_opener, ogg::reader_opener, mpeg::reader_opener, apv::reader_opener]))
R.get_or_init(|| {
RwLock::new(vec![
mp4::reader_opener,
mkv::reader_opener,
mxf::reader_opener,
ogg::reader_opener,
mpeg::reader_opener,
apv::reader_opener,
avi::reader_opener,
])
})
}
/// Openers that read containers through a [`filmcraft_media::ByteReader`] (index now, samples on
@ -161,6 +176,15 @@ impl filmcraft_isobmff::ByteSource for Src {
}
}
impl filmcraft_avi::ByteSource for Src {
fn len(&self) -> u64 {
self.0.len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> {
self.0.read_at(offset, buf)
}
}
impl filmcraft_matroska::ByteSource for Src {
fn len(&self) -> u64 {
self.0.len()
@ -178,6 +202,8 @@ pub fn open_bytes(name: &str, bytes: Arc<[u8]>) -> std::result::Result<filmcraft
#[cfg(test)]
mod audio_timing_tests;
#[cfg(test)]
mod avi_tests;
#[cfg(test)]
mod rounded_pts_tests;
#[cfg(test)]
mod stream_color_tests;

87
crates/codecs/src/mkv.rs
View File

@ -58,7 +58,7 @@ pub struct MkvSource {
/// Video codec as an ISO-BMFF sample entry (for the decoder factories).
ventry: Option<SampleEntry>,
video: GopCache,
/// The alpha layer of a VP9 track with `AlphaMode` 1 (WebM transparency, #402).
/// The alpha layer of a VP8 / VP9 track with `AlphaMode` 1 (WebM transparency, #402).
alpha: Option<Arc<crate::webm_alpha::AlphaLayer>>,
/// The playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
audios: Vec<MkvAudio>,
@ -71,15 +71,14 @@ pub struct MkvSource {
/// length (AAC, MPEG audio, AC-3, E-AC-3) therefore run on from one another, resynchronising to the
/// timestamp only across gaps of more than half a packet (and more than two ticks); see
/// [`crate::audio::contiguous_starts`]. Opus packets: [`opus_starts`]. Other codecs start at their
/// timestamps.
/// timestamps. Timestamps are taken with the exact `CodecDelay` off (see [`stamp_to_samples`]).
fn audio_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, rate: i64) -> Vec<i64> {
use crate::audio::{FixedFrames, PacketTime, contiguous_starts, fixed_packet_samples};
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
if let Some(h) = opus_head(&t.codec) {
return opus_starts(file, bytes, ti, &h, rate);
}
let (n, d) = tb(t);
let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64;
let at = stamp_to_samples(t, rate, codec_delay_samples(t, rate));
let first = || file.read_sample(bytes, ti, 0).unwrap_or_default();
let out_rate = u32::try_from(rate).unwrap_or(0);
let frame = match &t.codec {
@ -98,18 +97,13 @@ fn audio_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, rate: i64) -> Vec
/// Sample-exact Opus packet start positions (48 kHz frames, pre-skip removed).
///
/// The demuxer also subtracts a rounded `CodecDelay` from the timestamps. Starts are accumulated
/// from each packet's TOC duration, resynchronising to the timestamp only across real gaps (more
/// than two ticks off). Pre-skip is the exact `CodecDelay` (or the header's pre-skip when it is
/// absent).
/// Starts are accumulated from each packet's TOC duration, resynchronising to the timestamp only
/// across real gaps (more than two ticks off). Pre-skip is the exact `CodecDelay` (or the header's
/// pre-skip when it is absent).
fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_opus::OpusHead, rate: i64) -> Vec<i64> {
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
let (n, d) = tb(t);
let delay_ns = t.codec_delay_ns as i128;
// The demuxer's rounding of CodecDelay to ticks, undone here.
let delay_ticks = codec_delay_ticks(t);
let skip = if t.codec_delay_ns > 0 { (delay_ns * rate as i128 / 1_000_000_000) as i64 } else { head.pre_skip as i64 };
let at = |pts: i64| ((pts.saturating_add(delay_ticks) as i128 * n as i128 * rate as i128 / d as i128) as i64).saturating_sub(skip);
let skip = if t.codec_delay_ns > 0 { codec_delay_samples(t, rate) } else { head.pre_skip as i64 };
let at = stamp_to_samples(t, rate, skip);
let packets = t.samples.iter().enumerate().map(|(i, s)| crate::audio::PacketTime {
stamp: at(s.pts),
stamped: own_stamp(s),
@ -118,6 +112,28 @@ fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_o
crate::audio::contiguous_starts(packets, stamp_tolerance(t, rate))
}
/// A block timestamp (with the demuxer's rounded `CodecDelay` taken off) as a position in sample
/// frames at `rate`, `skip` frames of priming removed (#790). The demuxer's rounding is undone first:
/// `CodecDelay` is rarely a whole number of ticks (1024 AAC samples at 48 kHz are 21.333 ms, 256
/// AC-3 samples at 44.1 kHz 5.805 ms), so starting the first packet at the rounded delay played the
/// whole track up to 16 samples late.
fn stamp_to_samples(t: &filmcraft_matroska::Track, rate: i64, skip: i64) -> impl Fn(i64) -> i64 {
let (n, d) = tb(t);
let delay_ticks = codec_delay_ticks(t);
move |pts: i64| {
let ticks = pts.saturating_add(delay_ticks) as i128;
let at = ticks.checked_mul(n as i128).and_then(|x| x.checked_mul(rate as i128)).map_or(ticks.signum() * i128::MAX, |x| x / d as i128);
i64::try_from(at).unwrap_or(if at < 0 { i64::MIN } else { i64::MAX }).saturating_sub(skip)
}
}
/// `CodecDelay` in sample frames at `rate`, rounded to the nearest (the element holds whole
/// nanoseconds: 1024 samples at 48 kHz are stored as 21 333 333 ns).
fn codec_delay_samples(t: &filmcraft_matroska::Track, rate: i64) -> i64 {
let ns = (t.codec_delay_ns as i128).saturating_mul(rate.max(0) as i128);
i64::try_from(ns.saturating_add(500_000_000) / 1_000_000_000).unwrap_or(i64::MAX)
}
/// `CodecDelay` in timestamp ticks, rounded as the demuxer rounds it (half away from zero) before
/// subtracting it from every timestamp of the track.
fn codec_delay_ticks(t: &filmcraft_matroska::Track) -> i64 {
@ -232,6 +248,7 @@ fn vp9_config(private: &[u8], v: Option<&filmcraft_matroska::VideoInfo>) -> VpcC
fn sample_entry(c: &Codec, private: &[u8], v: Option<&filmcraft_matroska::VideoInfo>, w: u16, h: u16) -> Option<SampleEntry> {
Some(match c {
Codec::Vp8 => SampleEntry::video(FourCc(*b"vp08"), CodecConfig::Unknown { fourcc: FourCc(*b"vp08"), raw: Vec::new() }, w, h),
Codec::Vp9 { private } => SampleEntry::video(FourCc(*b"vp09"), CodecConfig::Vp9(vp9_config(private, v)), w, h),
Codec::Av1 { av1c } => SampleEntry::video(FourCc(*b"av01"), CodecConfig::Av1(filmcraft_isobmff::Av1Config::parse(av1c).unwrap_or_default()), w, h),
Codec::Apv { apvc } => SampleEntry::apv(filmcraft_isobmff::ApvConfig::parse(apvc).unwrap_or_default(), w, h),
@ -358,7 +375,11 @@ impl MkvSource {
let rate = match t.default_duration_ns {
Some(ns) if ns > 0 => FrameRate::from_f64(1e9 / ns as f64),
_ => {
let mut d: Vec<i64> = t.samples.windows(2).take(240).map(|p| (p[1].pts - p[0].pts).abs()).filter(|d| *d > 0).collect();
// blocks are stored in decode order: with B-frames, neighbours in the file are
// up to a few frames apart, so step between presentation times (#923)
let mut pts: Vec<i64> = t.samples.iter().take(241).map(|s| s.pts).collect();
pts.sort_unstable();
let mut d: Vec<i64> = pts.windows(2).map(|p| p[1].saturating_sub(p[0])).filter(|d| *d > 0).collect();
d.sort_unstable();
let (n, dd) = tb(t);
let step = d.get(d.len() / 2).copied().unwrap_or(1).max(1) as f64 * n as f64 / dd as f64;
@ -429,9 +450,11 @@ impl MkvSource {
Some(MkvAudio { track: i, state, starts, preroll })
})
.collect();
// WebM transparency: VP9 alpha in each block's BlockAdditional (ID 1)
// WebM transparency: VP8 / VP9 alpha in each block's BlockAdditional (ID 1)
let alpha = vtrack.and_then(|i| file.tracks.get(i)).filter(|t| {
matches!(t.codec, Codec::Vp9 { .. }) && t.video.as_ref().is_some_and(|v| v.alpha_mode != 0) && t.samples.iter().any(|s| s.addition.is_some())
matches!(t.codec, Codec::Vp8 | Codec::Vp9 { .. })
&& t.video.as_ref().is_some_and(|v| v.alpha_mode != 0)
&& t.samples.iter().any(|s| s.addition.is_some())
});
let alpha = alpha.map(|t| Arc::new(crate::webm_alpha::AlphaLayer::new(bytes.clone(), t.samples.iter().map(|s| (s.pts, s.addition)))));
Ok(Self { info, bytes, file, vtrack, ventry, video: GopCache::new(explicit_color).with_rotation(rotation), alpha, audios })
@ -441,6 +464,21 @@ impl MkvSource {
self.file.read_sample(&self.bytes, track, i).map_err(|e| CodecError::Container(e.to_string()))
}
/// Whether sample `i` of a VP8 track is a hidden frame (`show_frame` 0, RFC 6386 9.1): only the
/// first byte of the frame is read.
fn vp8_hidden(&self, track: usize, i: usize) -> bool {
let Some(t) = self.file.tracks.get(track) else { return false };
let Some(s) = t.samples.get(i) else { return false };
let first = match t.frame_prefix(u64::from(s.size)).first() {
Some(&b) => Some(b),
None => {
let mut b = [0u8; 1];
(s.size > 0 && self.bytes.0.read_at(s.offset, &mut b).is_ok()).then_some(b[0])
}
};
first.is_some_and(|b| b & 0x10 == 0)
}
fn audio_decoder(&self, c: &Codec, rate: u32) -> crate::Result<PacketDecoder> {
use symphonia::core::codecs::{CODEC_TYPE_FLAC, CODEC_TYPE_MP3, CODEC_TYPE_VORBIS};
match c {
@ -534,7 +572,20 @@ impl VideoSamples for MkvVideo<'_> {
self.src.file.tracks[self.track].sync_sample_before(i)
}
fn sample_at(&self, t: i64) -> Option<usize> {
self.src.file.tracks[self.track].sample_at_pts(t)
let track = &self.src.file.tracks[self.track];
let i = track.sample_at_pts(t)?;
if !matches!(track.codec, Codec::Vp8) {
return Some(i);
}
// VP8 has no superframes: an encoder stores a hidden (altref) frame as a block of its own
// with the time of the frame that follows it, which vpxenc moves 1 ms later. The hidden
// frame shows nothing, so its time asks for that next shown frame (as decoders that pass
// the two on as one packet show it).
let mut k = i;
while k + 1 < track.samples.len() && k < i + 8 && self.src.vp8_hidden(self.track, k) {
k += 1;
}
Some(k)
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
self.src.read(self.track, i)

4
crates/codecs/src/mp4.rs
View File

@ -540,7 +540,8 @@ fn audio_stream_info(t: &filmcraft_isobmff::Track, bytes: &crate::Src) -> Option
None => ap.sample_rate as u32,
}
},
if a.channel_config > 0 { a.channel_config as u32 } else { ap.channels },
// the configuration is an index (7 and 12 are both eight channels), not a count
if a.channels() > 0 { a.channels() } else { ap.channels },
None,
),
CodecConfig::Pcm(p) => (p.sample_rate as u32, p.channels, Some(p.bits as u32)),
@ -577,6 +578,7 @@ fn codec_label(c: &CodecConfig) -> String {
),
CodecConfig::Hevc(_) => "HEVC".into(),
CodecConfig::Vp9(c) => format!("VP9 (Profile {})", c.profile),
CodecConfig::Unknown { fourcc, .. } if fourcc.0 == *b"vp08" => "VP8".into(),
CodecConfig::Av1(c) => format!("AV1 ({} Profile)", ["Main", "High", "Professional"].get(c.seq_profile as usize).unwrap_or(&"Main")),
CodecConfig::Apv(a) => match filmcraft_apv::Profile::from_idc(a.profile_idc) {
Some(p) => p.name().into(),

185
crates/codecs/src/mpeg.rs
View File

@ -7,8 +7,9 @@
//! first random-access picture; display order comes from the PTS (or, without timestamps, the
//! GOP and temporal reference). Leading pictures of an open GOP decode from the previous GOP.
//! - Audio: MPEG audio layers I-III (symphonia), AAC in ADTS or LATM (our decoder), AC-3 (our
//! decoder), Blu-ray / AVCHD and DVD LPCM. Audio frames are placed on the timeline from their
//! PTS relative to the first video frame, sample-contiguously (re-anchored only at gaps).
//! decoder), Blu-ray / AVCHD and DVD LPCM. Every audio PID is a stream (capped). Audio frames
//! are placed on the timeline from their PTS relative to the first video frame,
//! sample-contiguously (re-anchored only at gaps).
use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, Mutex};
@ -104,7 +105,8 @@ pub struct MpegSource {
es: Option<(Arc<[u8]>, Vec<std::ops::Range<usize>>)>,
video: Option<VideoTrack>,
gop: GopCache,
audio: Option<AudioTrack>,
/// Playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
audios: Vec<AudioTrack>,
/// Why the video or audio stream cannot be decoded.
unsupported_video: Option<String>,
unsupported_audio: Option<String>,
@ -201,8 +203,8 @@ impl MpegSource {
log::warn!("{name}: {w}");
}
let vstream = file.find(Kind::Video, |c| vkind(c).is_some()).or_else(|| file.find(Kind::Video, |_| true));
let astream = file.find(Kind::Audio, audio_supported).or_else(|| file.find(Kind::Audio, |_| true));
if vstream.is_none() && astream.is_none() {
let has_audio = file.streams.iter().any(|s| s.kind() == Kind::Audio && !s.units.is_empty());
if vstream.is_none() && !has_audio {
return Err(CodecError::Unsupported(format!("{}: no video or audio streams", file.format.name())));
}
let mut src = MpegSource {
@ -221,7 +223,7 @@ impl MpegSource {
es: None,
video: None,
gop: GopCache::new(None),
audio: None,
audios: Vec::new(),
unsupported_video: None,
unsupported_audio: None,
};
@ -256,7 +258,7 @@ impl MpegSource {
src.video = Some(track);
src.describe_video(rate_from_pts)?;
if let Some(file) = src.file.take() {
src.setup_audio(file, astream, origin);
src.setup_audio(file, origin);
}
}
None => {
@ -273,12 +275,12 @@ impl MpegSource {
bitrate: None,
hdr: None,
});
src.setup_audio(file, astream, None);
src.setup_audio(file, None);
}
}
} else {
src.info.kind = MediaKind::AudioOnly;
src.setup_audio(file, astream, None);
src.setup_audio(file, None);
}
Ok(src)
}
@ -326,7 +328,7 @@ impl MpegSource {
es: Some((all, aus)),
video: Some(track),
gop: GopCache::new(None),
audio: None,
audios: Vec::new(),
unsupported_video: None,
unsupported_audio: None,
};
@ -347,15 +349,15 @@ impl MpegSource {
/// Timeline position (samples at the audio stream's rate) of the first audio frame, and the
/// number of samples in the audio stream.
pub fn audio_extent(&self) -> Option<(i64, i64)> {
let a = self.audio.as_ref()?;
let a = self.audios.first()?;
let first = *a.starts.first()?;
let end = a.starts.last()? + *a.lens.last()? as i64;
Some((first, end - first))
}
/// The demuxed stream indices of the presented video and audio.
/// The demuxed stream indices of the presented video and first audio.
pub fn stream_indices(&self) -> (Option<usize>, Option<usize>) {
(self.video.as_ref().filter(|_| self.es.is_none()).map(|v| v.stream), self.audio.as_ref().map(|a| a.stream))
(self.video.as_ref().filter(|_| self.es.is_none()).map(|v| v.stream), self.audios.first().map(|a| a.stream))
}
/// Presentation timestamps (90 kHz) of the presented video frames in display order. Frames
@ -475,15 +477,31 @@ impl MpegSource {
Ok(())
}
/// Choose and index the audio stream; `origin`: PTS of the first video frame.
fn setup_audio(&mut self, file: File, astream: Option<usize>, origin: Option<i64>) {
let track = astream.and_then(|ai| self.audio_track(&file, ai, origin));
/// Index every audio stream the container carries (capped); `origin`: PTS of the first video frame.
fn setup_audio(&mut self, file: File, origin: Option<i64>) {
let indices: Vec<usize> = file
.streams
.iter()
.enumerate()
.filter(|(_, s)| s.kind() == Kind::Audio && !s.units.is_empty())
.map(|(i, _)| i)
.take(filmcraft_media::MAX_AUDIO_STREAMS)
.collect();
let mut tracks = Vec::new();
for ai in indices {
if let Some(t) = self.audio_track(&file, ai, origin) {
tracks.push(t);
}
}
self.file = Some(file);
self.audio = track;
self.audios = tracks;
}
/// Mark the audio stream as not decodable (it still shows in the media info).
/// Mark the audio stream as not decodable (it still shows in the media info when nothing plays).
fn audio_unsupported(&mut self, codec_name: &str, why: String) -> Option<AudioTrack> {
if !self.info.audio_streams.is_empty() {
return None;
}
self.unsupported_audio = Some(why);
self.info.audio_streams =
vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None }];
@ -594,14 +612,16 @@ impl MpegSource {
next = Some(s + n as i64);
}
let total = next.unwrap_or(0).max(0);
if self.info.video.is_none() || self.unsupported_video.is_some() {
if self.unsupported_audio.take().is_some() {
self.info.audio_streams.clear();
}
if self.info.audio_streams.is_empty() && (self.info.video.is_none() || self.unsupported_video.is_some()) {
self.info.duration = Tick::from_units(total, rate.max(1) as i64);
if self.info.video.is_none() {
self.info.kind = MediaKind::AudioOnly;
}
}
self.info.audio_streams =
vec![AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits }];
self.info.audio_streams.push(AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits });
Some(AudioTrack {
stream: ai,
codec,
@ -783,10 +803,16 @@ impl MediaSource for MpegSource {
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
if let Some(why) = &self.unsupported_audio {
self.audio_stream(0, start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
if self.audios.is_empty()
&& let Some(why) = &self.unsupported_audio
{
return Err(MediaError::Unsupported(why.clone()));
}
let a = self.audio.as_ref().ok_or(MediaError::NoStream("audio"))?;
let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
let ch = a.channels;
let ratio = a.rate as f64 / sample_rate as f64;
let exact = a.rate == sample_rate;
@ -852,6 +878,7 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedRe
#[cfg(test)]
mod tests {
use super::*;
use filmcraft_media::MediaSource;
/// A mutated transport stream whose PTS span gave ~0 fps panicked on open with "attempt to
/// divide by zero" (`FrameRate::tick_of`).
@ -868,6 +895,118 @@ mod tests {
assert_eq!(stream_rate(None, Some(50.0)), FrameRate::from_f64(50.0));
}
/// MPEG-1 layer II frame, 48 kHz 192 kb/s (576 bytes).
fn mp2_frame(k: u8) -> Vec<u8> {
let mut v = vec![0xFF, 0xFD, 0xA4, 0x00];
v.extend((0..572).map(|i| (i as u8) ^ k));
v
}
fn ts_bytes(prefix: u8, v: i64) -> [u8; 5] {
let v = v & ((1 << 33) - 1);
[(prefix << 4) | (((v >> 30) & 7) as u8) << 1 | 1, (v >> 22) as u8, (((v >> 15) & 0x7F) as u8) << 1 | 1, (v >> 7) as u8, ((v & 0x7F) as u8) << 1 | 1]
}
fn pes(stream_id: u8, pts: Option<i64>, payload: &[u8]) -> Vec<u8> {
let mut h = Vec::new();
if let Some(p) = pts {
h.extend(ts_bytes(2, p));
}
let flags = if pts.is_some() { 0x80 } else { 0 };
let len = 3 + h.len() + payload.len();
let mut v = vec![0, 0, 1, stream_id, (len >> 8) as u8, len as u8, 0x80, flags, h.len() as u8];
v.extend(h);
v.extend_from_slice(payload);
v
}
/// Tiny TS muxer: PAT, PMT and PES packets in 188-byte packets.
struct TsMux {
out: Vec<u8>,
cc: std::collections::HashMap<u16, u8>,
}
impl TsMux {
fn packet(&mut self, pid: u16, pusi: bool, payload: &[u8]) -> usize {
let cc = self.cc.entry(pid).or_insert(0);
let mut p = vec![0x47, (pusi as u8) << 6 | (pid >> 8) as u8, pid as u8];
let n = payload.len().min(184);
if n < 184 {
let al = 183 - n;
p.push(0x30 | *cc);
p.push(al as u8);
if al > 0 {
p.push(0);
p.extend(std::iter::repeat_n(0xFF, al - 1));
}
} else {
p.push(0x10 | *cc);
}
*cc = (*cc + 1) & 15;
p.extend_from_slice(&payload[..n]);
self.out.extend(&p);
n
}
fn section(&mut self, pid: u16, body: &[u8]) {
let mut s = body.to_vec();
s.extend(crc32_mpeg(&s).to_be_bytes());
let mut payload = vec![0];
payload.extend(s);
self.packet(pid, true, &payload);
}
fn psi(&mut self, streams: &[(u8, u16)]) {
let pat = [0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00];
self.section(0, &pat);
let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x00, 0xF0, 0];
for &(st, pid) in streams {
pmt.extend([st, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]);
}
let len = pmt.len() - 3 + 4;
pmt[1] = 0xB0 | (len >> 8) as u8;
pmt[2] = len as u8;
self.section(0x100, &pmt);
}
fn pes(&mut self, pid: u16, data: &[u8]) {
let mut rest = data;
let mut first = true;
while !rest.is_empty() {
let n = self.packet(pid, first, rest);
rest = &rest[n..];
first = false;
}
}
}
fn crc32_mpeg(data: &[u8]) -> u32 {
let mut crc = 0xFFFF_FFFFu32;
for &b in data {
crc ^= (b as u32) << 24;
for _ in 0..8 {
crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
}
}
crc
}
/// Dual MPEG audio in one transport stream: import used to keep only the first PID.
#[test]
fn every_audio_pid_is_a_stream() {
let mut mux = TsMux { out: Vec::new(), cc: Default::default() };
mux.psi(&[(0x03, 0x1100), (0x03, 0x1101)]);
for k in 0..4u16 {
mux.pes(0x1100, &pes(0xC0, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8)));
mux.pes(0x1101, &pes(0xC1, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8 + 16)));
}
let source = MpegSource::open("dual.ts", mux.out.into()).unwrap();
assert_eq!(source.info().audio_streams.len(), 2, "{:?}", source.info().audio_streams);
assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::<Vec<_>>(), vec![48_000, 48_000]);
let a0 = source.audio(0, 64, 48_000).unwrap();
let a1 = source.audio_stream(1, 0, 64, 48_000).unwrap();
assert_eq!(a0.channels.len(), 2);
assert_eq!(a1.channels.len(), 2);
assert!(source.audio_stream(2, 0, 64, 48_000).is_err());
}
#[test]
fn presentation_drops_leading_pictures_and_orders_by_key() {
// decode order I(2) B(0) B(1) P(5) B(3) B(4): the leading B pictures are not presented

2
crates/codecs/src/stream_color.rs
View File

@ -42,6 +42,8 @@ pub fn from_codec_config(codec: &CodecConfig) -> Option<ColorCodes> {
CodecConfig::Avc(c) => avc(c),
CodecConfig::Hevc(c) => hevc(c),
CodecConfig::Av1(c) => av1(&c.config_obus),
// VP8 has no colour signalling: its one colour space is BT.601 YUV, studio swing (RFC 6386 9.2)
CodecConfig::Unknown { fourcc, .. } if fourcc.0 == *b"vp08" => Some(ColorCodes { primaries: 2, transfer: 2, matrix: 6, full_range: Some(false) }),
_ => None,
}
}

164
crates/codecs/src/tests.rs
View File

@ -7,7 +7,7 @@ use std::sync::Arc;
use filmcraft_media::FrameRequest;
use filmcraft_time::{TICKS_PER_SECOND, Tick};
fn ffmpeg() -> Option<&'static str> {
pub(crate) fn ffmpeg() -> Option<&'static str> {
["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].into_iter().find(|p| std::path::Path::new(p).exists())
}
@ -224,20 +224,63 @@ fn matroska_hevc_flac() {
check_mkv("blue_hevc_flac.mkv", &["-c:v", "libx265", "-x265-params", "log-level=error", "-pix_fmt", "yuv420p"], &["-c:a", "flac"]);
}
/// FLAC in MP4 (`fLaC` + `dfLa`): the decoder gets the STREAMINFO from `dfLa` (it was given
/// nothing, and every FLAC track in an MP4 failed with "missing extra data").
#[test]
fn mp4_flac_audio() {
let args = ["-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=48000:d=2", "-c:a", "flac", "-strict", "-2"];
let Some(b) = fixture("tone_flac.mp4", &args) else { return };
let src = crate::open_bytes("tone_flac.mp4", b).expect("open");
let a = src.audio(24_000, 4800, 48_000).expect("audio");
let peak = a.channels[0].iter().fold(0f32, |m, s| m.max(s.abs()));
assert!(peak > 0.05 && peak < 1.0, "audio peak {peak}");
}
#[test]
fn matroska_prores_pcm() {
check_mkv("blue_prores_pcm.mkv", &["-c:v", "prores_ks", "-profile:v", "2"], &["-c:a", "pcm_s16le"]);
}
/// `bytes` with its first `DefaultDuration` element (0x23E383) overwritten by an EBML Void of the
/// same length, as a muxer that leaves the optional element out would write the track.
fn without_default_duration(bytes: &[u8]) -> Vec<u8> {
let mut out = bytes.to_vec();
let at = out.windows(3).position(|w| w == [0x23, 0xE3, 0x83]).expect("DefaultDuration");
// ffmpeg writes a one-byte size (0x80 | n)
let total = 4 + (out[at + 3] & 0x7F) as usize;
out[at] = 0xEC;
out[at + 1] = 0x80 | (total - 2) as u8;
out[at + 2..at + total].fill(0);
out
}
#[test]
fn matroska_reordered_video_without_default_duration_keeps_its_rate() {
// blocks are in decode order, so B-frames put file neighbours 2–3 frames apart (#923)
for (bf, name) in [(0, "rate25_bf0.mkv"), (1, "rate25_bf1.mkv"), (2, "rate25_bf2.mkv")] {
let params = format!("bframes={bf}:b-adapt=0:b-pyramid=0:keyint=250:scenecut=0");
let args =
["-f", "lavfi", "-i", "testsrc2=size=64x48:rate=25", "-frames:v", "25", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-x264-params", &params, "-an"];
let Some(control) = fixture(name, &args) else { return };
let stripped = without_default_duration(&control);
for (label, bytes) in [("with DefaultDuration", control.to_vec()), ("without", stripped)] {
let src = crate::open_bytes(name, bytes.into()).expect("open");
let rate = src.info().video.as_ref().expect("video").frame_rate;
assert_eq!((rate.num, rate.den), (25, 1), "{bf} B-frames, {label}");
assert_eq!(src.info().duration, Tick::from_rational(1, 1, 1), "{bf} B-frames, {label}");
}
}
}
/// Path of a fixture under `target/fixtures/codecs/`, generated with ffmpeg when missing
/// (`None` when ffmpeg or the encoder is unavailable).
fn fixture_path(name: &str, args: &[&str]) -> Option<PathBuf> {
pub(crate) fn fixture_path(name: &str, args: &[&str]) -> Option<PathBuf> {
fixture(name, args)?;
Some(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/codecs").join(name))
}
/// ffmpeg's decode of `src` as raw planar video (the oracle).
fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option<Vec<u8>> {
pub(crate) fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option<Vec<u8>> {
let name = format!("{}.{pix_fmt}.yuv", src.file_name()?.to_str()?);
let path = src.to_str()?;
let b = fixture(&name, &["-i", path, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", pix_fmt])?;
@ -245,7 +288,7 @@ fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option<Vec<u8>> {
}
/// Planes of a decoded YUV frame as little-endian bytes (ffmpeg rawvideo layout).
fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec<u8> {
pub(crate) fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec<u8> {
match &f.data {
filmcraft_frame::PixelData::Yuv8 { planes, .. } => planes.iter().flat_map(|p| p.iter().copied()).collect(),
filmcraft_frame::PixelData::Yuv16 { planes, .. } => planes.iter().flat_map(|p| p.iter().flat_map(|s| s.to_le_bytes())).collect(),
@ -449,6 +492,119 @@ fn vp9_webm_alpha_layer_matches_libvpx() {
}
}
/// #432: VP8 in WebM with hidden altref frames (vpxenc, two-pass with ARNR: 11 of its 161
/// frames are not shown). Frames requested out of order (seeks back and forth, then sequential
/// playback across the hidden frames) are sample-exact against libvpx's decode, and the stream's
/// colour is BT.601 (RFC 6386 9.2). Skipped without ffmpeg or vpxenc.
#[test]
fn vp8_webm_random_access_matches_libvpx() {
let frames = 150usize;
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/codecs");
let path = dir.join("vp8_hidden_altref.webm");
if !path.exists() {
let Some(y4m) =
fixture_path("mandelbrot_352x288_150.y4m", &["-f", "lavfi", "-i", "mandelbrot=s=352x288:r=25", "-frames:v", "150", "-pix_fmt", "yuv420p"])
else {
return;
};
let tmp = dir.join(format!("vp8_hidden_altref.{}.webm", std::process::id()));
let enc = Command::new("vpxenc")
.args(["--codec=vp8", "--passes=2", "--auto-alt-ref=1", "--lag-in-frames=25", "--arnr-maxframes=7", "--arnr-strength=5"])
.args(["--target-bitrate=300", "--kf-max-dist=150", "-q", "-o"])
.arg(&tmp)
.arg(&y4m)
.status();
if !enc.is_ok_and(|s| s.success()) {
eprintln!("vpxenc not available; skipping");
return;
}
let _ = std::fs::rename(&tmp, &path);
}
let p = path.to_string_lossy().into_owned();
let Some(reference) =
fixture("vp8_hidden_altref.webm.yuv", &["-c:v", "libvpx", "-i", &p, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "yuv420p"])
else {
return;
};
let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into();
let src = crate::open_bytes("vp8_hidden_altref.webm", bytes).unwrap();
let info = src.info().clone();
assert_eq!(info.container, "WebM");
let v = info.video.as_ref().unwrap();
assert_eq!(v.codec, "VP8");
assert_eq!((v.width, v.height), (352, 288));
assert_eq!(v.color.matrix, filmcraft_color::Matrix::Bt601);
assert_eq!(reference.len(), frames * 352 * 288 * 3 / 2);
let frame_len = reference.len() / frames;
let rate = info.frame_rate();
let order: Vec<usize> = [147, 3, 90, 26, 24, 149, 0, 112, 61].into_iter().chain(40..frames).collect();
for k in order {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
assert_eq!((f.width, f.height), (352, 288));
assert_eq!(f.color.matrix, filmcraft_color::Matrix::Bt601);
assert!(yuv_bytes(&f) == reference[k * frame_len..(k + 1) * frame_len], "frame {k} differs from libvpx");
}
}
/// #432 / #402: a VP8 WebM with an alpha layer (`AlphaMode` 1, a second VP8 stream in each
/// block's `BlockAdditional`, as browsers and GIF converters write transparent WebM) decodes
/// with that alpha; frames requested out of order match libvpx exactly, colour and alpha.
#[test]
fn vp8_webm_alpha_layer_matches_libvpx() {
let (w, h, frames) = (176usize, 144usize, 30usize);
let Some(path) = fixture_path(
"alpha_vp8.webm",
&[
"-f",
"lavfi",
"-i",
"testsrc2=s=176x144:r=25,format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='mod(X*2+N*7\\,256)'",
"-frames:v",
"30",
"-c:v",
"libvpx",
"-pix_fmt",
"yuva420p",
"-g",
"10",
"-auto-alt-ref",
"0",
"-b:v",
"600k",
],
) else {
return;
};
let src_path = path.to_string_lossy().into_owned();
let Some(reference) =
fixture("alpha_vp8.webm.yuva420p.yuv", &["-c:v", "libvpx", "-i", &src_path, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "yuva420p"])
else {
return;
};
let (luma, chroma) = (w * h, (w / 2) * (h / 2));
let frame_len = luma * 2 + chroma * 2;
if reference.len() != frame_len * frames {
eprintln!("libvpx reference has {} bytes, not {}; skipping", reference.len(), frame_len * frames);
return;
}
let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into();
let src = crate::open_bytes("alpha_vp8.webm", bytes).unwrap();
let info = src.info().clone();
assert_eq!(info.container, "WebM");
assert!(info.video.as_ref().unwrap().has_alpha);
let rate = info.frame_rate();
let order: Vec<usize> = [29, 3, 17, 12, 0, 25, 21, 9].into_iter().chain(10..frames).collect();
for k in order {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
let r = &reference[k * frame_len..(k + 1) * frame_len];
let filmcraft_frame::PixelData::Yuv8 { planes, alpha, .. } = &f.data else { panic!("frame {k}: not 8-bit YUV: {}", f.format_label()) };
assert_eq!(planes[0].as_slice(), &r[..luma], "frame {k}: luma");
assert_eq!(planes[1].as_slice(), &r[luma..luma + chroma], "frame {k}: U");
let alpha = alpha.as_ref().unwrap_or_else(|| panic!("frame {k}: no alpha plane"));
assert_eq!(alpha.as_slice(), &r[luma + 2 * chroma..], "frame {k}: alpha");
}
}
/// Every sample flagged as sync (as in an MP4 without `stss`): the GOP cache must still start
/// decoding at a real VP9 key frame.
#[test]

289
crates/codecs/src/video.rs
View File

@ -140,11 +140,14 @@ pub(crate) fn tight_plane<T: Copy>(src: Vec<T>, stride: usize, w: usize, h: usiz
}
/// Colour of an H.264 / HEVC picture from its VUI code points (ITU-T H.273), as the decoders
/// report it: BT.709 by default with the matrix guessed from the size, the signalled matrix and
/// transfer when known, and full range when flagged. Shared by the software and hardware
/// decoders so the two are interchangeable.
pub fn vui_color(width: u32, height: u32, matrix: u8, transfer: u8, full_range: bool) -> filmcraft_color::ColorInfo {
/// report it: BT.709 by default with the matrix guessed from the size, the signalled primaries,
/// matrix and transfer when known, and full range when flagged. Shared by the software and
/// hardware decoders so the two are interchangeable.
pub fn vui_color(width: u32, height: u32, primaries: u8, matrix: u8, transfer: u8, full_range: bool) -> filmcraft_color::ColorInfo {
let mut color = filmcraft_color::ColorInfo { matrix: filmcraft_frame::default_matrix(width, height), ..filmcraft_color::ColorInfo::REC709 };
if let Some(p) = primaries_from_code(primaries) {
color.primaries = p;
}
if let Some(m) = filmcraft_color::Matrix::from_code(matrix) {
color.matrix = m;
}
@ -178,13 +181,69 @@ pub fn hvcc_length_size_and_tid(hvcc: &[u8]) -> (usize, Option<u8>) {
/// A factory returns `None` when it does not handle the entry.
pub type VideoDecoderFactory = fn(&SampleEntry) -> Option<Result<Box<dyn VideoDecoder>>>;
/// The typical Huffman tables of ITU-T T.81 Annex K.3 as one DHT segment (luminance and
/// chrominance, DC and AC): what a Motion JPEG frame without its own tables is coded with (AVI's
/// "MJPG", the AVI1 convention of capture cards and cameras).
const DEFAULT_DHT: &[u8] = &[
0xFF, 0xC4, 0x01, 0xA2, //
0x00, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, //
0x01, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, //
0x10, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7D, //
0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1, 0x08, //
0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, //
0x29, 0x2A, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, //
0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, //
0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, //
0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xE1, 0xE2, //
0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, //
0x11, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77, //
0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, //
0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62, 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25, 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, //
0x27, 0x28, 0x29, 0x2A, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, //
0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, //
0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, //
0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, //
0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA,
];
/// A JPEG with the [`DEFAULT_DHT`] inserted before its first scan when it defines no Huffman
/// table of its own (else unchanged, borrowed).
pub fn with_default_huffman(jpeg: &[u8]) -> std::borrow::Cow<'_, [u8]> {
use std::borrow::Cow;
if jpeg.get(..2) != Some(&[0xFF, 0xD8]) {
return Cow::Borrowed(jpeg);
}
// walk the marker segments up to the start of scan
let mut at = 2usize;
while let (Some(&0xFF), Some(&m)) = (jpeg.get(at), jpeg.get(at + 1)) {
match m {
0xFF => at += 1, // fill byte
0xC4 => return Cow::Borrowed(jpeg),
0xDA => {
let mut out = Vec::with_capacity(jpeg.len() + DEFAULT_DHT.len());
out.extend_from_slice(jpeg.get(..at).unwrap_or_default());
out.extend_from_slice(DEFAULT_DHT);
out.extend_from_slice(jpeg.get(at..).unwrap_or_default());
return Cow::Owned(out);
}
0xD0..=0xD9 | 0x01 => at += 2,
_ => {
let Some(len) = jpeg.get(at + 2..at + 4).map(|b| usize::from(u16::from_be_bytes([b[0], b[1]]))) else { break };
at += 2 + len;
}
}
}
Cow::Borrowed(jpeg)
}
/// Motion-JPEG / Photo-JPEG (each sample is a complete JPEG).
pub struct MjpegDecoder;
impl VideoDecoder for MjpegDecoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let sample = with_default_huffman(sample);
// mjpa samples may contain two fields; decode the first image (field-merging lands with interlace support).
let img = image::load_from_memory_with_format(sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?;
let img = image::load_from_memory_with_format(&sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?;
let rgba = img.to_rgba8();
let (w, h) = rgba.dimensions();
Ok(vec![DecodedFrame { pts, frame: VideoFrame::rgba8(w, h, rgba.into_raw()), draft: false }])
@ -250,7 +309,7 @@ impl H264Decoder {
let y = tight_plane(p.y, p.y_stride, w, h);
let u = tight_plane(p.u, p.uv_stride, cw, ch);
let v = tight_plane(p.v, p.uv_stride, cw, ch);
let color = vui_color(p.width, p.height, p.color.matrix, p.color.transfer, p.color.full_range);
let color = vui_color(p.width, p.height, p.color.primaries, p.color.matrix, p.color.transfer, p.color.full_range);
let par = sar_par(p.sar);
let frame = VideoFrame {
width: p.width,
@ -371,7 +430,7 @@ impl HevcDecoder {
alpha: None,
},
};
let color = vui_color(p.width, p.height, p.color.matrix, p.color.transfer, p.color.full_range);
let color = vui_color(p.width, p.height, p.color.primaries, p.color.matrix, p.color.transfer, p.color.full_range);
let par = sar_par(p.sar);
let frame = VideoFrame { width: p.width, height: p.height, data, color, par, pts: filmcraft_time::Tick::ZERO };
DecodedFrame { pts: p.pts, frame, draft: p.draft }
@ -547,6 +606,60 @@ pub fn vp9_factory(e: &SampleEntry) -> Option<Result<Box<dyn VideoDecoder>>> {
}
}
/// Our pure-Rust VP8 decoder (RFC 6386; #432): WebM / Matroska `V_VP8` and MP4 `vp08`.
pub struct Vp8Decoder {
dec: filmcraft_vp8::Decoder,
}
impl Default for Vp8Decoder {
fn default() -> Self {
Self::new()
}
}
impl Vp8Decoder {
pub fn new() -> Self {
Self { dec: filmcraft_vp8::Decoder::new() }
}
fn convert(p: filmcraft_vp8::Picture) -> DecodedFrame {
use filmcraft_frame::{Chroma, PixelData};
use std::sync::Arc;
// color_space 0 is BT.601 YUV (RFC 6386 9.2), studio swing; a container's Colour wins
let color = filmcraft_color::ColorInfo { matrix: filmcraft_color::Matrix::Bt601, ..filmcraft_color::ColorInfo::REC709 };
let data = PixelData::Yuv8 { planes: [Arc::new(p.y), Arc::new(p.u), Arc::new(p.v)], chroma: Chroma::C420, alpha: None };
let frame = VideoFrame { width: p.width, height: p.height, data, color, par: (1, 1), pts: filmcraft_time::Tick::ZERO };
DecodedFrame { pts: p.pts, frame, draft: false }
}
}
impl VideoDecoder for Vp8Decoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let pic = self.dec.decode(sample, pts).map_err(|e| CodecError::Decode(e.to_string()))?;
Ok(pic.into_iter().map(Self::convert).collect())
}
fn flush(&mut self) -> Vec<DecodedFrame> {
// no reordering: every picture comes out with its frame
Vec::new()
}
fn reset(&mut self) {
self.dec.reset();
}
fn name(&self) -> &str {
"FilmCraft VP8"
}
fn is_random_access(&self, sample: &[u8]) -> Option<bool> {
Some(filmcraft_vp8::is_keyframe(sample))
}
fn is_disposable(&self, sample: &[u8]) -> bool {
self.dec.is_disposable(sample)
}
}
pub fn vp8_factory(e: &SampleEntry) -> Option<Result<Box<dyn VideoDecoder>>> {
(e.format.0 == *b"vp08").then(|| Ok(Box::new(Vp8Decoder::new()) as Box<dyn VideoDecoder>))
}
/// Our AV1 decoder. The `av1C` configuration OBUs (sequence header) are fed before the first
/// sample and again after every reset.
pub struct Av1Decoder {
@ -1035,3 +1148,165 @@ mod disposable_tests {
assert!(!hevc_disposable(&sample(&[&nal(0, 1)]), 4, None));
}
}
/// Uncompressed video layouts (AVI `BI_RGB` and YUV FourCCs).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RawFormat {
/// 24-bit B, G, R; rows padded to 4 bytes.
Bgr24,
/// 32-bit B, G, R, X.
Bgrx32,
/// Packed 4:2:2: Y0 U Y1 V (`YUY2`, `YUYV`).
Yuy2,
/// Packed 4:2:2: U Y0 V Y1 (`UYVY`, `2vuy`).
Uyvy,
/// Planar 4:2:0: Y, U, V (`I420`, `IYUV`).
I420,
/// Planar 4:2:0: Y, V, U (`YV12`).
Yv12,
/// Y plane, interleaved U V at half resolution (`NV12`).
Nv12,
/// Luma only (`Y800`, `GREY`, `Y8 `).
Gray8,
}
impl RawFormat {
/// The layout of an AVI video stream: `BI_RGB` with its bit count, or a YUV FourCC.
pub fn from_avi(compression: [u8; 4], bit_count: u16) -> Option<RawFormat> {
if matches!(u32::from_le_bytes(compression), 0 | 3) {
return match bit_count {
24 => Some(RawFormat::Bgr24),
32 => Some(RawFormat::Bgrx32),
_ => None,
};
}
let mut upper = compression;
upper.make_ascii_uppercase();
Some(match &upper {
b"YUY2" | b"YUYV" | b"YUNV" | b"V422" => RawFormat::Yuy2,
b"UYVY" | b"2VUY" | b"UYNV" | b"HDYC" => RawFormat::Uyvy,
b"I420" | b"IYUV" => RawFormat::I420,
b"YV12" => RawFormat::Yv12,
b"NV12" => RawFormat::Nv12,
b"Y800" | b"GREY" | b"GRAY" | b"Y8 " => RawFormat::Gray8,
_ => return None,
})
}
pub fn label(self) -> &'static str {
match self {
RawFormat::Bgr24 => "Uncompressed RGB 24-bit",
RawFormat::Bgrx32 => "Uncompressed RGB 32-bit",
RawFormat::Yuy2 | RawFormat::Uyvy => "Uncompressed YUV 4:2:2",
RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 => "Uncompressed YUV 4:2:0",
RawFormat::Gray8 => "Uncompressed greyscale",
}
}
}
/// Uncompressed frames (each sample one picture of a [`RawFormat`]).
pub struct RawVideoDecoder {
format: RawFormat,
width: u32,
height: u32,
/// RGB rows are stored bottom-up (a positive `BITMAPINFOHEADER` height).
bottom_up: bool,
}
impl RawVideoDecoder {
pub fn new(format: RawFormat, width: u32, height: u32, bottom_up: bool) -> Result<Self> {
if width == 0 || height == 0 || width > 16_384 || height > 16_384 {
return Err(CodecError::Unsupported(format!("{width}x{height} uncompressed video")));
}
Ok(Self { format, width, height, bottom_up })
}
fn yuv(&self, planes: [Vec<u8>; 3], chroma: filmcraft_frame::Chroma) -> VideoFrame {
use std::sync::Arc;
let [y, u, v] = planes;
VideoFrame {
width: self.width,
height: self.height,
data: filmcraft_frame::PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma, alpha: None },
color: vui_color(self.width, self.height, 2, 2, 2, false),
par: (1, 1),
pts: filmcraft_time::Tick::ZERO,
}
}
fn picture(&self, s: &[u8]) -> Result<VideoFrame> {
let (w, h) = (self.width as usize, self.height as usize);
let short = || CodecError::Decode(format!("{} frame of {} bytes is too short for {w}x{h}", self.format.label(), s.len()));
let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
Ok(match self.format {
RawFormat::Bgr24 | RawFormat::Bgrx32 => {
let bpp = if self.format == RawFormat::Bgr24 { 3 } else { 4 };
let stride = (w * bpp).div_ceil(4) * 4;
if s.len() < stride * h {
return Err(short());
}
let mut rgba = Vec::with_capacity(w * h * 4);
for row in 0..h {
let src = if self.bottom_up { h - 1 - row } else { row };
for px in s.get(src * stride..src * stride + w * bpp).unwrap_or_default().chunks_exact(bpp) {
rgba.extend_from_slice(&[px[2], px[1], px[0], 255]);
}
}
VideoFrame::rgba8(self.width, self.height, rgba)
}
RawFormat::Yuy2 | RawFormat::Uyvy => {
let stride = cw * 4;
if s.len() < stride * h {
return Err(short());
}
let (yo, uo, vo, y2) = if self.format == RawFormat::Yuy2 { (0, 1, 3, 2) } else { (1, 0, 2, 3) };
let (mut y, mut u, mut v) = (Vec::with_capacity(w * h), Vec::with_capacity(cw * h), Vec::with_capacity(cw * h));
for row in s.chunks_exact(stride).take(h) {
for (k, q) in row.as_chunks::<4>().0.iter().enumerate() {
y.push(q[yo]);
if 2 * k + 1 < w {
y.push(q[y2]);
}
u.push(q[uo]);
v.push(q[vo]);
}
}
self.yuv([y, u, v], filmcraft_frame::Chroma::C422)
}
RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 | RawFormat::Gray8 => {
let luma = w * h;
let need = if self.format == RawFormat::Gray8 { luma } else { luma + 2 * cw * ch };
if s.len() < need {
return Err(short());
}
let y = s.get(..luma).unwrap_or_default().to_vec();
let (u, v) = match self.format {
RawFormat::Gray8 => (vec![128; cw * ch], vec![128; cw * ch]),
RawFormat::Nv12 => s.get(luma..luma + 2 * cw * ch).unwrap_or_default().as_chunks::<2>().0.iter().map(|p| (p[0], p[1])).unzip(),
_ => {
let a = s.get(luma..luma + cw * ch).unwrap_or_default().to_vec();
let b = s.get(luma + cw * ch..luma + 2 * cw * ch).unwrap_or_default().to_vec();
if self.format == RawFormat::Yv12 { (b, a) } else { (a, b) }
}
};
self.yuv([y, u, v], filmcraft_frame::Chroma::C420)
}
})
}
}
impl VideoDecoder for RawVideoDecoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
Ok(vec![DecodedFrame { pts, frame: self.picture(sample)?, draft: false }])
}
fn flush(&mut self) -> Vec<DecodedFrame> {
Vec::new()
}
fn reset(&mut self) {}
fn name(&self) -> &str {
self.format.label()
}
fn intra_only(&self) -> bool {
true
}
}

11
crates/codecs/src/webm_alpha.rs
View File

@ -1,9 +1,10 @@
//! WebM alpha (#402): a VP9 track with `AlphaMode` 1 carries, in each block's `BlockAdditional`
//! with `BlockAddID` 1, a second VP9 stream coded independently of the colour one. Its luma plane,
//! WebM alpha (#402): a VP8 or VP9 track with `AlphaMode` 1 carries, in each block's
//! `BlockAdditional` with `BlockAddID` 1, a second stream of the same codec coded independently of
//! the colour one. Its luma plane,
//! full range, is the frame's alpha (the WebM convention, as libvpx-based decoders read it).
//!
//! [`AlphaDecoder`] wraps the colour decoder: every sample it decodes, it also decodes that
//! sample's alpha frame with a second (software) VP9 decoder and attaches the luma plane to the
//! sample's alpha frame with a second (software) decoder and attaches the luma plane to the
//! colour picture with the same timestamp, holding a colour picture back until its alpha is out. The alpha stream has key frames of its own, which need
//! not coincide with the colour ones: after a seek (or any other break in the order samples are
//! fed), the alpha decoder restarts from the nearest alpha key frame at or before the sample.
@ -21,7 +22,7 @@ use crate::video::{DecodedFrame, VideoDecoder};
/// How far back the alpha decoder looks for a key frame when it resynchronises.
const MAX_RESYNC: usize = 600;
/// Decoded alpha pictures waiting for their colour picture, and colour pictures waiting for their
/// alpha (VP9 pictures come out in order, so a few are plenty; past that a picture goes out opaque
/// alpha (VP8 / VP9 pictures come out in order, so a few are plenty; past that a picture goes out opaque
/// rather than stall).
const MAX_PENDING: usize = 8;
@ -106,7 +107,7 @@ impl AlphaDecoder {
self.alpha.reset();
self.pending.clear();
self.alpha_max = None;
let is_key = |k: usize| self.layer.read(k).is_some_and(|d| filmcraft_vp9::is_keyframe(&d));
let is_key = |k: usize| self.layer.read(k).is_some_and(|d| self.alpha.is_random_access(&d) == Some(true));
let start = (i.saturating_sub(MAX_RESYNC)..=i).rev().find(|&k| is_key(k));
if let Some(start) = start {
for k in start..i {

108
crates/codecs/tests/aac_channels_oracle.rs Normal file
View File

@ -0,0 +1,108 @@
//! Multichannel AAC in MP4 against ffmpeg (#922). The AudioSpecificConfig's channel configuration
//! is an index into ISO/IEC 14496-3 Table 1.19 (with Amd 4), not a channel count: 7 and 12 are
//! both 7.1 (eight channels) and 11 is 6.1 (seven). Each channel carries its own tone, so a
//! dropped or swapped channel shows up against ffmpeg's decode.
mod common;
use std::io::Cursor;
use common::*;
use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
const RATE: u32 = 48_000;
/// A one-second AAC MP4 of `layout` (`n` channels), channel `i` a tone of 200 + 100·i Hz, except
/// the LFE (fourth in all these layouts) at 60 Hz: AAC band-limits it.
fn tones(ff: &std::path::Path, name: &str, layout: &str, n: usize) -> Option<std::path::PathBuf> {
let mut graph: String =
(0..n).map(|i| format!("sine=frequency={}:sample_rate={RATE}:duration=1[a{i}];", if i == 3 { 60 } else { 200 + 100 * i })).collect();
graph.extend((0..n).map(|i| format!("[a{i}]")));
graph.push_str(&format!("join=inputs={n}:channel_layout={layout}[out]"));
fixture(ff, name, &["-filter_complex", &graph, "-map", "[out]", "-c:a", "aac", "-b:a", "512k"])
}
/// The AAC access units of `path`'s audio track, rewrapped in a new MP4 whose AudioSpecificConfig
/// says `channel_config` (AAC LC, 48 kHz), keeping each sample's duration and the priming edit.
fn rewrap(path: &std::path::Path, name: &str, channel_config: u8, channels: u32) -> std::path::PathBuf {
let data = bytes(path);
let file = filmcraft_isobmff::open(&data).unwrap();
let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio).unwrap();
// object type 2, rate index 3 (48 kHz), then the configuration
let asc = (((2u16 << 11) | (3 << 7) | ((channel_config as u16) << 3)).to_be_bytes()).to_vec();
let mut mux = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap();
let id = mux.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).unwrap();
for s in &track.samples {
let au = &data[s.offset as usize..s.offset as usize + s.size as usize];
mux.write_sample(id, WriteSample { data: au, duration: s.duration, composition_offset: 0, is_sync: true }).unwrap();
}
let out = dir().join(name);
std::fs::write(&out, mux.finish().unwrap().into_inner()).unwrap();
out
}
/// Our decode of the whole file, channel by channel.
fn ours(path: &std::path::Path, name: &str) -> (u32, Vec<Vec<f32>>) {
let src = filmcraft_codecs::open_bytes(name, bytes(path)).unwrap();
let ch = src.info().audio().map(|a| a.channels).unwrap();
(ch, src.audio(0, RATE as usize * 9 / 10, RATE).unwrap().channels)
}
/// ffmpeg's decode, deinterleaved into `n` channels.
fn theirs(ff: &std::path::Path, path: &std::path::Path, n: usize) -> Vec<Vec<f32>> {
let all = ffmpeg_audio_f32(ff, path, &[]);
(0..n).map(|c| all.iter().skip(c).step_by(n).copied().collect()).collect()
}
/// Largest sample difference over the first `len` samples, best over a small lag either way (the
/// two decoders may place the priming differently by a few samples).
fn diff(a: &[f32], b: &[f32]) -> f32 {
let len = a.len().min(b.len()).saturating_sub(64);
(0..=32).flat_map(|l| [(l, 0), (0, l)]).map(|(x, y)| (32..len).map(|i| (a[i + x] - b[i + y]).abs()).fold(0.0, f32::max)).fold(f32::MAX, f32::min)
}
fn peak(x: &[f32]) -> f32 {
x.iter().fold(0.0, |m, v| m.max(v.abs()))
}
#[test]
fn aac_channel_configurations_report_and_decode_every_channel() {
let ff = filmcraft_testkit::require_ffmpeg!();
// ffmpeg 8 writes 7.1 as configuration 7 and ffmpeg 9 as 12: the same five elements
let Some(seven) = tones(&ff, "aac_71.mp4", "7.1", 8) else {
eprintln!("SKIPPED: this ffmpeg can't encode 7.1 AAC");
return;
};
let asc = filmcraft_isobmff::open(bytes(&seven)).unwrap().tracks.iter().find_map(|t| match &t.entries.first()?.codec {
filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.channel_config),
_ => None,
});
let twelve = match asc {
Some(7) => rewrap(&seven, "aac_71_cfg12.mp4", 12, 8),
Some(12) => seven.clone(),
other => panic!("ffmpeg wrote 7.1 AAC as configuration {other:?}"),
};
let seven = if asc == Some(7) { seven } else { rewrap(&seven, "aac_71_cfg7.mp4", 7, 8) };
// configuration 7: eight channels reported and returned, the last one too
let (ch, got) = ours(&seven, "aac_71_cfg7.mp4");
assert_eq!((ch, got.len()), (8, 8), "configuration 7");
assert!(got.iter().all(|c| peak(c) > 0.05), "configuration 7: a silent channel");
// configuration 12 (7.1 back): decoded, in ffmpeg's (and WAV's) L R C LFE Lb Rb Ls Rs order
let (ch, got) = ours(&twelve, "aac_71_cfg12.mp4");
assert_eq!((ch, got.len()), (8, 8), "configuration 12");
let want = theirs(&ff, &twelve, 8);
for (c, (a, b)) in got.iter().zip(&want).enumerate() {
let d = diff(a, b);
println!("configuration 12, channel {c}: peak {:.3}, max |ours - ffmpeg| {d:.5}", peak(a));
assert!(peak(a) > 0.05 && d < 0.01, "configuration 12, channel {c}: off ffmpeg's decode by {d}");
}
// controls: 5.1 (configuration 6) and 6.1 (ffmpeg writes a PCE)
for (layout, n, name) in [("5.1", 6, "aac_51.mp4"), ("6.1", 7, "aac_61.mp4")] {
let path = tones(&ff, name, layout, n).unwrap();
let (ch, got) = ours(&path, name);
assert_eq!((ch as usize, got.len()), (n, n), "{layout}");
}
}

18
crates/codecs/tests/eac3_oracle.rs
View File

@ -53,12 +53,18 @@ fn eac3_in_mp4_matroska_and_transport_streams() {
};
let ours = src.audio(start, n, 48_000).unwrap().channels;
assert_eq!(ours.len(), ch, "{name}");
// Matroska stores CodecDelay (the encoder's 256-sample priming) in ns and timestamps in
// ms, and we place packets at the rounded value (240 samples): a shared Matroska timing
// limit (AAC and AC-3 alike), not a decoding one. Elsewhere the samples line up exactly.
let (worst, lag) = (-32..=32).map(|lag| (snr(&ours, &want, ch, lag), lag)).fold((f64::MIN, 0), |a, b| if b.0 > a.0 { b } else { a });
let max_lag = if name.ends_with(".mkv") { 16 } else { 0 };
assert!(lag.abs() <= max_lag, "{name}: decode is {lag} samples off ffmpeg's");
// Matroska stores the encoder's 256-sample priming as CodecDelay, which we trim exactly
// (checked to the sample against the encoded signal in `matroska_audio_oracle`). Whether
// ffmpeg's CLI trims it too depends on its version (9.x does; 2023-24 builds keep the
// priming samples), so for .mkv its decode may start 256 samples earlier than ours.
// Elsewhere the samples line up exactly, whatever the version.
let lags: Vec<i64> = match name.ends_with(".mkv") {
true => (-32..=32).chain(-256 - 32..=-256 + 32).collect(),
false => (-32..=32).collect(),
};
let (worst, lag) = lags.into_iter().map(|lag| (snr(&ours, &want, ch, lag), lag)).fold((f64::MIN, 0), |a, b| if b.0 > a.0 { b } else { a });
let ok = if name.ends_with(".mkv") { lag == 0 || lag == -256 } else { lag == 0 };
assert!(ok, "{name}: decode is {lag} samples off ffmpeg's");
// a window read on a fresh source (random access) gives the same samples
let fresh = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap();
let at = n / 3;

61
crates/codecs/tests/frame_color_oracle.rs Normal file
View File

@ -0,0 +1,61 @@
//! Decoded H.264 / HEVC frames carry the colour primaries their VUI signals (#924). ffmpeg writes
//! MP4 without a `colr` box unless asked, so for most files the bitstream is the only place the
//! primaries are described: stream info already read them, the decoded frames reported BT.709.
mod common;
use common::*;
use filmcraft_color::{ColorInfo, ColorSpace, Matrix, Primaries, Range, Transfer};
use filmcraft_media::FrameRequest;
use filmcraft_time::Tick;
/// One 64×64 frame of `codec` with the colour `params` (x264 / x265 syntax), and our stream and
/// frame colour for it.
fn colors(ff: &std::path::Path, name: &str, codec: &[&str]) -> Option<(ColorInfo, ColorInfo)> {
let mut args = vec!["-f", "lavfi", "-i", "testsrc2=size=64x64:rate=1", "-frames:v", "1"];
args.extend_from_slice(codec);
let path = fixture(ff, name, &args)?;
let isobmff = filmcraft_isobmff::open(bytes(&path)).unwrap();
let track = isobmff.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).unwrap();
assert!(track.entries[0].video.as_ref().unwrap().color.is_none(), "{name}: the fixture has a colr box");
let src = filmcraft_codecs::open_bytes(name, bytes(&path)).unwrap();
let stream = src.info().video.as_ref().unwrap().color;
let frame = src.video_frame(FrameRequest::full(Tick::ZERO)).unwrap().color;
Some((stream, frame))
}
#[test]
fn decoded_frames_keep_the_signalled_primaries() {
let ff = filmcraft_testkit::require_ffmpeg!();
let x264 = |p: &'static str| ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-x264-params", p];
let x265 = |p: &'static str, pix: &'static str| ["-c:v", "libx265", "-pix_fmt", pix, "-tag:v", "hvc1", "-x265-params", p];
let sdr2020 = ColorInfo { matrix: Matrix::Bt709, transfer: Transfer::Bt709, primaries: Primaries::Bt2020, range: Range::Limited };
let cases: [(&str, Vec<&str>, ColorInfo, ColorSpace); 4] = [
("prim2020_h264.mp4", x264("colorprim=bt2020:transfer=bt709:colormatrix=bt709").to_vec(), sdr2020, ColorSpace::Rec2020),
("prim2020_hevc.mp4", x265("log-level=error:colorprim=bt2020:transfer=bt709:colormatrix=bt709", "yuv420p").to_vec(), sdr2020, ColorSpace::Rec2020),
// control: BT.709 throughout
(
"prim709_h264.mp4",
x264("colorprim=bt709:transfer=bt709:colormatrix=bt709").to_vec(),
ColorInfo { primaries: Primaries::Bt709, ..sdr2020 },
ColorSpace::Rec709,
),
// HDR keeps its transfer and now its primaries too
(
"prim2020_pq_hevc.mp4",
x265("log-level=error:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc", "yuv420p10le").to_vec(),
ColorInfo { matrix: Matrix::Bt2020Ncl, transfer: Transfer::Pq, primaries: Primaries::Bt2020, range: Range::Limited },
ColorSpace::Rec2100Pq,
),
];
for (name, codec, want, space) in cases {
let Some((stream, frame)) = colors(&ff, name, &codec) else {
eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
continue;
};
println!("{name}: stream {stream:?}, frame {frame:?}");
assert_eq!(stream, want, "{name}: stream");
assert_eq!(frame, want, "{name}: decoded frame");
assert_eq!(ColorSpace::from_info(&frame), space, "{name}");
}
}

97
crates/codecs/tests/matroska_audio_oracle.rs
View File

@ -48,3 +48,100 @@ fn matroska_audio_plays_without_holes() {
assert!(r_ours < 2.0 * r_theirs + 0.005, "{name}: clicks between packets (|Δ²| {r_ours} vs ffmpeg's {r_theirs})");
}
}
/// Mean |difference| between `ours[i]` and `theirs[i + lag]` over `from..from + len`.
fn mean_err(ours: &[f32], theirs: &[f32], from: usize, len: usize, lag: i64) -> f64 {
(from..from + len).map(|i| (ours[i] as f64 - theirs[(i as i64 + lag) as usize] as f64).abs()).sum::<f64>() / len as f64
}
/// The lag in `lags` at which `ours` best matches `theirs` (least [`mean_err`]), and that error.
fn best_lag(ours: &[f32], theirs: &[f32], from: usize, len: usize, lags: std::ops::RangeInclusive<i64>) -> (i64, f64) {
lags.map(|lag| (lag, mean_err(ours, theirs, from, len, lag))).min_by(|a, b| a.1.total_cmp(&b.1)).unwrap()
}
/// A reference signal we control: low-passed white noise from a fixed xorshift seed. Its
/// autocorrelation dies out within a few samples, so even a one-sample misalignment shows.
fn noise(n: usize) -> Vec<f32> {
let (mut state, mut y) = (0x9E37_79B9_7F4A_7C15u64, 0f32);
(0..n)
.map(|_| {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
let white = (state >> 40) as f32 / (1u64 << 24) as f32 - 0.5;
y = 0.5 * y + 0.4 * white;
y
})
.collect()
}
/// #790: ffmpeg stores the encoder priming as `CodecDelay` (1024 samples = 21.333 ms for AAC at
/// 48 kHz, 256 = 5.805 ms for AC-3 at 44.1 kHz, 1105 = 25.057 ms for MP3), and block timestamps to
/// the millisecond. Taking the rounded delay as the start of the first packet played the whole
/// track up to 16 samples late.
///
/// The strict check is version-independent: ffmpeg encodes a signal we generate, and our decode
/// must line up with that signal to the sample (the stream's `CodecDelay` trimmed exactly). ffmpeg's
/// own decode is compared only after a bounded best-lag search: how its CLI handles Matroska
/// priming depends on the version (9.x trims `CodecDelay`, 2023-24 builds keep the priming
/// samples), so its lag is reported, not asserted.
#[test]
fn matroska_audio_is_sample_aligned_with_ffmpeg() {
let ff = filmcraft_testkit::require_ffmpeg!();
let cases: [(&str, u32, &[&str]); 6] = [
("ref_aac_48k.mkv", 48_000, &["-c:a", "aac", "-b:a", "256k"]),
("ref_aac_44k.mkv", 44_100, &["-c:a", "aac", "-b:a", "256k"]),
("ref_ac3_44k.mkv", 44_100, &["-c:a", "ac3", "-b:a", "448k"]),
("ref_ac3_48k.mkv", 48_000, &["-c:a", "ac3", "-b:a", "448k"]),
("ref_eac3_48k.mkv", 48_000, &["-c:a", "eac3", "-b:a", "384k"]),
("ref_mp3_44k.mkv", 44_100, &["-c:a", "libmp3lame", "-b:a", "320k"]),
];
for (name, rate, codec) in cases {
let reference = noise(rate as usize * 2);
let raw = format!("ref_noise_{rate}.f32");
let raw_path = filmcraft_testkit::fixtures::generate(&dir().join(&raw), |tmp| {
std::fs::write(tmp, reference.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>()).is_ok()
})
.unwrap();
let (raw_arg, rate_arg) = (raw_path.to_str().unwrap().to_string(), rate.to_string());
let args: Vec<&str> = ["-f", "f32le", "-ar", &rate_arg, "-ac", "1", "-i", &raw_arg].iter().copied().chain(codec.iter().copied()).collect();
let Some(path) = fixture(&ff, name, &args) else {
eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
continue;
};
let file = bytes(&path);
let mkv = filmcraft_matroska::open(file.clone()).unwrap();
let delay_ns = mkv.tracks.first().map_or(0, |t| t.codec_delay_ns) as i64;
let delay = (delay_ns * rate as i64 + 500_000_000) / 1_000_000_000;
assert!(delay > 0, "{name}: no CodecDelay");
let src = filmcraft_codecs::open_bytes(name, file).unwrap();
let frames = rate as usize * 3 / 2;
let mut ours = Vec::with_capacity(frames);
while ours.len() < frames {
let n = 1000.min(frames - ours.len());
ours.extend_from_slice(&src.audio(ours.len() as i64, n, rate).unwrap().channels[0]);
}
let (from, len, bound) = (rate as usize / 4, rate as usize / 4, delay + 32);
// strict: the encoder's input, to the sample
let (lag, err) = best_lag(&ours, &reference, from, len, -bound..=bound);
let level = reference[from..from + len].iter().map(|v| v.abs() as f64).sum::<f64>() / len as f64;
assert_eq!(lag, 0, "{name}: our decode is {lag} samples off the encoded signal (CodecDelay {delay} samples)");
assert!(err < 0.25 * level, "{name}: decode error {err:.2e} against a signal level of {level:.2e}");
// ffmpeg's decode, wherever its version puts the priming
let theirs = ffmpeg_audio_f32(&ff, &path, &[]);
assert!(theirs.len() >= frames, "{name}: ffmpeg decoded {} samples", theirs.len());
let (ff_lag, ff_err) = best_lag(&ours, &theirs, from, len, -bound..=bound);
let priming = match ff_lag {
0 => "trims CodecDelay".to_string(),
l if l == delay => "keeps the priming samples".to_string(),
l => format!("is {l} samples off"),
};
println!("{name}: CodecDelay {delay} samples; vs the encoded signal lag 0, error {err:.2e} (level {level:.2e}); ffmpeg {priming}, error {ff_err:.2e}");
assert!(ff_err < 1e-3, "{name}: differs from ffmpeg's decode by {ff_err:.2e} at its best lag {ff_lag}");
// read from the middle, as a seek does
let at = rate as i64 / 2;
let mid = src.audio(at, 2000, rate).unwrap().channels[0].clone();
let diff = mid.iter().zip(&ours[at as usize..]).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max);
assert!(diff < 1e-3, "{name}: a read from {at} differs from continuous decoding by {diff}");
}
}

92
crates/codecs/tests/media_oracle.rs
View File

@ -171,6 +171,98 @@ fn dpx_stills_every_layout() {
}
}
/// A big-endian DPX file holding `data` as its one image element (`w` × `h`, the given
/// descriptor, bit depth, packing and colorimetric code; reference codes undefined).
fn dpx_file(w: u32, h: u32, descriptor: u8, bits: u8, packing: u16, colorimetric: u8, data: &[u8]) -> Vec<u8> {
let mut f = vec![0u8; 2048];
let put = |f: &mut Vec<u8>, at: usize, v: &[u8]| f[at..at + v.len()].copy_from_slice(v);
put(&mut f, 0, b"SDPX");
put(&mut f, 4, &2048u32.to_be_bytes());
put(&mut f, 8, b"V2.0");
put(&mut f, 16, &(2048 + data.len() as u32).to_be_bytes());
put(&mut f, 770, &1u16.to_be_bytes());
put(&mut f, 772, &w.to_be_bytes());
put(&mut f, 776, &h.to_be_bytes());
let el = 780;
put(&mut f, el + 4, &u32::MAX.to_be_bytes());
put(&mut f, el + 12, &u32::MAX.to_be_bytes());
f[el + 20] = descriptor;
f[el + 22] = colorimetric;
f[el + 23] = bits;
put(&mut f, el + 24, &packing.to_be_bytes());
put(&mut f, el + 28, &2048u32.to_be_bytes());
f.extend_from_slice(data);
f
}
/// `pix_fmt` raw frame of ffmpeg's test picture at `w` × `h`.
fn raw_picture(ff: &Path, w: u32, h: u32, pix_fmt: &str) -> Vec<u8> {
let alpha = if pix_fmt.contains('a') { ",format=rgba,geq=r='r(X,Y)':g='g(X,Y)':b='b(X,Y)':a='X*2+Y'" } else { "" };
let vf = format!("scale={w}:{h}{alpha}");
ffmpeg_out(ff, &["-f", "lavfi", "-i", "testsrc2=size=96x64", "-frames:v", "1", "-vf", &vf, "-f", "rawvideo", "-pix_fmt", pix_fmt, "-"])
}
/// A DPX file we built must import exactly like ffmpeg decodes it, within `tol` of its 16-bit
/// decode rounded to 8 bits.
fn dpx_matches(ff: &Path, name: &str, file: Vec<u8>, tol: u8) {
let path = dir().join(name);
std::fs::create_dir_all(dir()).unwrap();
std::fs::write(&path, &file).unwrap();
let src = filmcraft_codecs::open_bytes(name, file.into()).unwrap_or_else(|e| panic!("{name}: {e}"));
assert_eq!(src.info().kind, MediaKind::Still, "{name}");
let ours = rgba(&src.video_frame(FrameRequest::full(filmcraft_time::Tick(0))).unwrap());
let wide = ffmpeg_out(ff, &["-i", path.to_str().unwrap(), "-f", "rawvideo", "-pix_fmt", "rgba64le", "-"]);
let want: Vec<u8> = wide.as_chunks::<2>().0.iter().map(|c| ((u32::from(u16::from_le_bytes(*c)) * 255 + 32767) / 65535) as u8).collect();
assert_eq!(ours.len(), want.len(), "{name}");
let d = ours.iter().zip(&want).map(|(a, b)| a.abs_diff(*b)).max().unwrap();
assert!(d <= tol, "{name}: differs by {d}");
eprintln!("{name}: max diff {d}");
}
/// Packed 12-bit RGB (packing 0: components back to back, low bits first, lines on 32-bit
/// boundaries) and 8-bit Y'CbCr (4:2:2, 4:4:4, 4:4:4:4 with alpha; BT.709, BT.601 and unspecified
/// colorimetry), layouts ffmpeg reads but does not write, built here from ffmpeg's test picture
/// (#841).
#[test]
fn dpx_packed_and_ycbcr_match_ffmpeg() {
let ff = filmcraft_testkit::require_ffmpeg!();
let (w, h) = (95u32, 63u32);
let rgb: Vec<u32> = raw_picture(&ff, w, h, "rgb48le").as_chunks::<2>().0.iter().map(|c| u32::from(u16::from_le_bytes(*c)) >> 4).collect();
let mut data = Vec::new();
for line in rgb.chunks(w as usize * 3) {
let (mut acc, mut n) = (0u64, 0u32);
for &v in line {
acc |= u64::from(v) << n;
n += 12;
while n >= 32 {
data.extend_from_slice(&(acc as u32).to_be_bytes());
acc >>= 32;
n -= 32;
}
}
if n > 0 {
data.extend_from_slice(&(acc as u32).to_be_bytes());
}
}
dpx_matches(&ff, "dpx_packed12.dpx", dpx_file(w, h, 50, 12, 0, 6, &data), 1);
// Y'CbCr: even width, scan lines a multiple of 4 bytes
let (w, h) = (96u32, 64u32);
let px = (w * h) as usize;
let uyvy = raw_picture(&ff, w, h, "uyvy422");
let yuv = raw_picture(&ff, w, h, "yuv444p");
let yuva = raw_picture(&ff, w, h, "yuva444p");
let interleave = |planes: &[u8], order: &[usize]| -> Vec<u8> { (0..px).flat_map(|i| order.iter().map(move |&p| planes[p * px + i])).collect() };
// planes are Y, U (Cb), V (Cr), A; DPX stores Cb Y Cr A
let cbycr = interleave(&yuv, &[1, 0, 2]);
let cbycra = interleave(&yuva, &[1, 0, 2, 3]);
for colorimetric in [6u8, 7, 0] {
dpx_matches(&ff, &format!("dpx_cbycry_{colorimetric}.dpx"), dpx_file(w, h, 100, 8, 0, colorimetric, &uyvy), 1);
dpx_matches(&ff, &format!("dpx_cbycr_{colorimetric}.dpx"), dpx_file(w, h, 102, 8, 0, colorimetric, &cbycr), 1);
}
dpx_matches(&ff, "dpx_cbycra.dpx", dpx_file(w, h, 103, 8, 0, 6, &cbycra), 1);
}
#[test]
fn missing_frames_hold_the_previous_frame() {
let ff = filmcraft_testkit::require_ffmpeg!();

59
crates/codecs/tests/mpeg_oracle.rs
View File

@ -16,6 +16,9 @@ use filmcraft_media::{FrameRequest, MediaError, MediaSource};
const NOISE: &[&str] =
&["-f", "lavfi", "-i", "aevalsrc=exprs=0.3*sin(2*PI*441*t)+0.1*sin(2*PI*3100*t)|0.25*sin(2*PI*660*t)+0.1*sin(2*PI*(500+4000*t)*t):s=48000"];
/// A mono tone.
const MONO: &[&str] = &["-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=48000"];
fn spec(name: &str) -> Vec<String> {
let cat = |p: &[&[&str]]| p.concat().into_iter().map(String::from).collect::<Vec<_>>();
match name {
@ -142,6 +145,12 @@ fn spec(name: &str) -> Vec<String> {
cat(&[&["-f", "lavfi", "-i", "testsrc2=size=320x240:rate=25"], NOISE, &["-t", "0.6", "-c:v", "mpeg2video", "-bf", "2", "-g", "6", "-c:a", "mp2"]])
}
"audio.mp2" => cat(&[NOISE, &["-t", "1", "-c:a", "mp2", "-b:a", "224k", "-f", "mp2"]]),
// mono MP2 at ffmpeg's default bit rate, 384 kbit/s, which ISO/IEC 11172-3 does not allow
"audio_mono.mp2" => cat(&[MONO, &["-t", "1", "-c:a", "mp2", "-f", "mp2"]]),
"ts_mpeg2_mono_mp2.ts" => {
cat(&[&["-f", "lavfi", "-i", "testsrc2=size=320x240:rate=25"], MONO, &["-t", "1", "-c:v", "mpeg2video", "-c:a", "mp2", "-f", "mpegts"]])
}
"mkv_mono_mp2.mkv" => cat(&[MONO, &["-t", "1", "-c:a", "mp2"]]),
"es_mpeg2.m2v" => {
cat(&[&["-f", "lavfi", "-i", "testsrc2=size=352x288:rate=25"], &["-t", "1", "-c:v", "mpeg2video", "-bf", "2", "-g", "9", "-f", "mpeg2video"]])
}
@ -164,6 +173,9 @@ pub const ALL: &[&str] = &[
"mp4_mpeg2.mp4",
"mkv_mpeg2.mkv",
"audio.mp2",
"audio_mono.mp2",
"ts_mpeg2_mono_mp2.ts",
"mkv_mono_mp2.mkv",
"es_mpeg2.m2v",
];
@ -515,6 +527,53 @@ fn standalone_mp2_file() {
assert!(worst <= 2e-4, "{worst}");
}
/// #809: mono MP2 at 384 kbit/s, ffmpeg's default, played as silence in every container because
/// symphonia rejects the bit rate / mode combinations ISO/IEC 11172-3 does not allow.
#[test]
fn mono_mp2_at_ffmpegs_default_bit_rate() {
let ff = filmcraft_testkit::require_ffmpeg!();
let f = make(&ff, "audio_mono.mp2");
assert_eq!(std::fs::read(&f).unwrap()[2] >> 4, 14, "384 kbit/s");
let worst = check_audio(&ff, &make(&ff, "ts_mpeg2_mono_mp2.ts"), 2e-4);
println!("ts_mpeg2_mono_mp2.ts: max error {worst}");
// Matroska stores the encoder's 481-sample priming as `CodecDelay`. Whether ffmpeg's CLI trims
// it depends on its version (9.x does; 2023-24 builds keep the priming samples), and until
// #790 we trim it rounded to the millisecond. So for .mkv ffmpeg's decode is searched at
// lags within a sample or two of 0 (trimmed) and of `CodecDelay` (kept). The plain .mp2 file
// has no such ambiguity and must line up exactly.
for name in ["audio_mono.mp2", "mkv_mono_mp2.mkv"] {
let f = make(&ff, name);
let file = bytes(&f);
let src = filmcraft_codecs::open_bytes(name, file.clone()).unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!((a.sample_rate, a.channels), (48_000, 1), "{name}");
let want = ffmpeg_audio_f32(&ff, &f, &[]);
let (delay, lags): (usize, Vec<i64>) = match name.ends_with(".mkv") {
true => {
let mkv = filmcraft_matroska::open(file).unwrap();
let ns = mkv.tracks.first().map_or(0, |t| t.codec_delay_ns);
let delay = ((ns * 48_000 + 500_000_000) / 1_000_000_000) as usize;
assert!(delay > 0, "{name}: no CodecDelay");
(delay, (-2..=2).chain(delay as i64 - 2..=delay as i64 + 2).collect())
}
false => (0, vec![0]),
};
// ours from sample 2 on, against ffmpeg's from sample 2 + lag (inside both decodes)
let n = want.len() - delay - 6;
let buf = src.audio(0, n, 48_000).unwrap();
let (worst, lag) = lags
.into_iter()
.map(|lag| {
let err = (2..n - 2).map(|i| (want[(i as i64 + lag) as usize] - buf.channels[0][i]).abs()).fold(0f32, f32::max);
(err, lag)
})
.fold((f32::INFINITY, 0), |m, x| if x.0 < m.0 { x } else { m });
let peak = want.iter().fold(0f32, |m, x| m.max(x.abs()));
println!("{name}: max error {worst} at lag {lag} (CodecDelay {delay}), peak {peak}");
assert!(peak > 0.1 && worst <= 2e-4, "{name}: peak {peak}, max error {worst} at lag {lag}");
}
}
#[test]
fn truncated_and_corrupt_files_never_panic() {
let ff = filmcraft_testkit::require_ffmpeg!();

69
crates/codecs/tests/rf64_metadata.rs Normal file
View File

@ -0,0 +1,69 @@
//! #925: a chunk after an RF64 `data` sentinel must not become audio.
use std::sync::Arc;
use filmcraft_time::Tick;
fn chunk(id: &[u8; 4], body: &[u8]) -> Vec<u8> {
let mut out = id.to_vec();
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(body);
if body.len() % 2 == 1 {
out.push(0);
}
out
}
/// Four 16-bit mono frames of silence at 48 kHz, as RIFF or RF64, optionally with a trailing iXML.
fn fixture(rf64: bool, trailing: bool) -> Vec<u8> {
let mut fmt = Vec::new();
for v in [1u16, 1] {
fmt.extend_from_slice(&v.to_le_bytes());
}
fmt.extend_from_slice(&48_000u32.to_le_bytes());
fmt.extend_from_slice(&96_000u32.to_le_bytes());
fmt.extend_from_slice(&2u16.to_le_bytes());
fmt.extend_from_slice(&16u16.to_le_bytes());
let mut out = if rf64 { b"RF64\xff\xff\xff\xffWAVE".to_vec() } else { b"RIFF\0\0\0\0WAVE".to_vec() };
if rf64 {
let mut ds = vec![0u8; 28];
ds[8..16].copy_from_slice(&8u64.to_le_bytes());
ds[16..24].copy_from_slice(&4u64.to_le_bytes());
out.extend(chunk(b"ds64", &ds));
}
out.extend(chunk(b"fmt ", &fmt));
let offset = out.len();
out.extend(chunk(b"data", &[0u8; 8]));
if rf64 {
out[offset + 4..offset + 8].copy_from_slice(&u32::MAX.to_le_bytes());
}
if trailing {
out.extend(chunk(b"iXML", b"<BWFXML></BWFXML>"));
}
let size = out.len() - 8;
if rf64 {
out[20..28].copy_from_slice(&(size as u64).to_le_bytes());
} else {
out[4..8].copy_from_slice(&(size as u32).to_le_bytes());
}
out
}
#[test]
fn rf64_metadata_stays_outside_audio() {
for (name, rf64, trailing) in [("rf64.wav", true, false), ("riff_ixml.wav", false, true), ("rf64_ixml.wav", true, true)] {
let bytes: Arc<[u8]> = fixture(rf64, trailing).into();
let src = filmcraft_codecs::open_bytes(name, bytes.clone()).unwrap();
let stream = filmcraft_media::reader::open_reader(
name,
Arc::new(filmcraft_media::reader::MemReader(bytes)),
&filmcraft_codecs::reader_openers(),
&filmcraft_codecs::openers(),
)
.unwrap();
for s in [&src, &stream] {
assert_eq!(s.info().duration, Tick::from_units(4, 48_000), "{name}");
assert_eq!(s.audio(0, 20, 48_000).unwrap().channels.remove(0), vec![0.0; 20], "{name}");
}
}
}

2
crates/color/src/lib.rs
View File

@ -20,7 +20,7 @@ pub use grade::GradeSpace;
pub use log::LogCurve;
pub use lut::{Lut, Lut1d, Lut3d, LutFormat};
pub use spaces::{ColorPipeline, ColorSpace, Curve, Gamut, WorkingSpace};
pub use transform::{DecodeTable, InputTransform, OutputTransform, REFERENCE_WHITE_NITS, ToneMap};
pub use transform::{DecodeTable, GpuCurve, GpuInputParams, InputTransform, OutputTransform, REFERENCE_WHITE_NITS, ToneMap};
/// YUV ↔ RGB matrix coefficients.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]

15
crates/color/src/lut.rs
View File

@ -186,7 +186,7 @@ impl LutFormat {
}
fn floats(line: &str) -> Result<Vec<f32>, String> {
line.split_whitespace().map(|x| x.parse::<f32>().map_err(|_| format!("not a number: {x:?}"))).collect()
line.split_whitespace().map(|x| x.parse::<f32>().ok().filter(|v| v.is_finite()).ok_or_else(|| format!("not a finite number: {x:?}"))).collect()
}
impl Lut {
@ -252,6 +252,9 @@ impl Lut {
if v.len() != 2 {
return Err(err(&format!("{key} needs 2 values")));
}
if v[1] <= v[0] {
return Err(err(&format!("{key} maximum must exceed its minimum")));
}
let r = Some([v[0], v[1]]);
if key == "LUT_1D_INPUT_RANGE" { range1 = r } else { range3 = r }
}
@ -591,6 +594,16 @@ mod tests {
assert!(Lut::parse_cube("TITLE \"x\"\n").is_err());
assert!(Lut::parse_cube("LUT_3D_SIZE 2\n0 0 0\nLUT_1D_SIZE 2\n").is_err());
assert!(Lut::parse_cube("LUT_3D_SIZE 2\nDOMAIN_MIN 1 1 1\nDOMAIN_MAX 0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n").is_err());
for key in ["LUT_1D_INPUT_RANGE", "LUT_3D_INPUT_RANGE"] {
let (size, rows) = if key.starts_with("LUT_1D") { ("LUT_1D_SIZE 2", 2) } else { ("LUT_3D_SIZE 2", 8) };
let table = "0 0 0\n".repeat(rows);
for range in ["0 0", "1 0", "NaN 1", "0 inf"] {
assert!(Lut::parse_cube(&format!("{size}\n{key} {range}\n{table}")).is_err(), "accepted {key} {range}");
}
assert!(Lut::parse_cube(&format!("{size}\n{key} -0.5 2\n{table}")).is_ok());
}
assert!(Lut::parse_cube("LUT_1D_SIZE 2\n0 0 0\nNaN 1 1\n").is_err());
assert!(Lut::parse_cube("LUT_1D_SIZE 2\nDOMAIN_MAX inf 1 1\n0 0 0\n1 1 1\n").is_err());
// comments, blank lines, Windows newlines, inline comments
let txt = "# hi\r\nTITLE \"t\"\r\n\r\nLUT_3D_SIZE 2 # two\r\n0 0 0\n1 0 0\n0 1 0\n1 1 0\n0 0 1\n1 0 1\n0 1 1\n1 1 1\n";
let l = Lut3d::parse_cube(txt).unwrap();

67
crates/color/src/transform.rs
View File

@ -259,6 +259,54 @@ impl InputTransform {
pub fn convert(&self, rgb: [f32; 3]) -> [f32; 3] {
self.apply(rgb.map(|v| self.table.lookup(v)))
}
/// The transform as plain numbers for a shader, which evaluates the curve and the BT.2390
/// EETF directly instead of sampling the tables. `None` for what a shader does not do: log
/// curves and the camera-gamut compression.
pub fn gpu_params(&self) -> Option<GpuInputParams> {
let curve = match self.source.curve() {
Curve::Sdr | Curve::Srgb => GpuCurve::Sdr,
Curve::Linear => GpuCurve::Linear,
Curve::Pq => GpuCurve::Pq,
Curve::Hlg => GpuCurve::Hlg,
Curve::Log(_) => return None,
};
if self.compress {
return None;
}
Some(GpuInputParams {
curve,
hlg: self.hlg,
matrix: (!self.identity_matrix).then_some(self.matrix),
tone: self.tone.as_ref().map(|t| (t.src_peak_nits as f32, t.dst_peak_nits as f32)),
desaturate: self.desaturate,
})
}
}
/// How a shader decodes a channel ([`decode_channel`] without the table): SDR with the sRGB curve,
/// linear, PQ in working units (cd/m² over [`REFERENCE_WHITE_NITS`]), HLG to scene light.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum GpuCurve {
Sdr,
Linear,
Pq,
Hlg,
}
/// [`InputTransform`] for a shader ([`InputTransform::gpu_params`]), stage by stage as
/// [`InputTransform::apply`] runs them.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct GpuInputParams {
pub curve: GpuCurve,
/// HLG OOTF: BT.2020 luma weights and the scale to working units.
pub hlg: Option<([f32; 3], f32)>,
/// Into the working primaries (row-major), when they differ.
pub matrix: Option<[[f32; 3]; 3]>,
/// BT.2390 tone mapping (source peak, target peak) in cd/m², on max(R,G,B) as a ratio.
pub tone: Option<(f32, f32)>,
/// Luma weights to desaturate into the working gamut with ([`desaturate_into_gamut`]).
pub desaturate: Option<[f32; 3]>,
}
/// Whether gamut `a` lies inside gamut `b` (so no compression is needed going a → b).
@ -462,6 +510,25 @@ mod tests {
}
}
#[test]
fn gpu_params_cover_display_referred_media_only() {
// HLG / PQ / BT.2020 into Rec. 709: every stage the shader has (#965)
let p = |cs| InputTransform::new(cs, Range::Limited, &ColorPipeline::REC709, None).gpu_params();
let hlg = p(ColorSpace::Rec2100Hlg).unwrap();
assert_eq!(hlg.curve, GpuCurve::Hlg);
assert!(hlg.hlg.is_some() && hlg.matrix.is_some() && hlg.desaturate.is_some());
assert_eq!(hlg.tone, Some((HDR_PEAK_NITS as f32, REFERENCE_WHITE_NITS as f32)));
let pq = InputTransform::new(ColorSpace::Rec2100Pq, Range::Limited, &ColorPipeline::REC709, Some(4000.0)).gpu_params().unwrap();
assert_eq!((pq.curve, pq.hlg, pq.tone), (GpuCurve::Pq, None, Some((4000.0, REFERENCE_WHITE_NITS as f32))));
let sdr2020 = p(ColorSpace::Rec2020).unwrap();
assert_eq!((sdr2020.curve, sdr2020.tone), (GpuCurve::Sdr, None));
assert!(sdr2020.matrix.is_some() && sdr2020.desaturate.is_some());
// log curves and camera-gamut compression stay on the CPU
for cs in ColorSpace::ALL.into_iter().filter(|c| c.is_log()) {
assert_eq!(p(cs), None, "{cs:?}");
}
}
#[test]
fn sdr_in_hdr_keeps_values_and_display_tone_maps() {
let pipe = hdr(WorkingSpace::Rec2100Pq);

23
crates/dng/Cargo.toml Normal file
View File

@ -0,0 +1,23 @@
[package]
name = "filmcraft-dng"
description = "Clean-room DNG / CinemaDNG camera RAW decoder (TIFF container, lossless JPEG, DNG colour pipeline)"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[features]
default = ["threads"]
threads = ["dep:rayon"]
# A small DNG and lossless JPEG writer for synthesising test files (not used by the app).
writer = []
[dependencies]
rayon = { workspace = true, optional = true }
thiserror.workspace = true
[dev-dependencies]
filmcraft-dng = { path = ".", features = ["writer"] }
[lints]
workspace = true

88
crates/dng/README.md Normal file
View File

@ -0,0 +1,88 @@
# filmcraft-dng
Clean-room DNG and CinemaDNG camera RAW decoder in pure Rust. Layer L0: depends only on
`thiserror` and (optional, feature `threads`, default on) `rayon`. Builds for
`wasm32-unknown-unknown`; no `unsafe`. Implemented from the public specifications below only; the
source of no other raw decoder (dcraw, LibRaw, rawspeed, darktable, RawTherapee, FFmpeg, the Adobe
DNG SDK…) was read. LibRaw's `unprocessed_raw` sample program is run as an external test oracle
when it is installed, never linked.
## Specifications
| Document | Edition | Used for |
|---|---|---|
| Adobe Digital Negative (DNG) Specification | 1.4.0.0 (2012) – 1.7.1.0 (2023) | raw IFD selection (NewSubFileType 0), CFA / LinearRaw data, BitsPerSample packing, LinearizationTable, BlackLevel (+ RepeatDim, DeltaH / DeltaV), WhiteLevel, ActiveArea, DefaultCrop, the colour model of chapter 6 (ColorMatrix, CameraCalibration, AnalogBalance, ForwardMatrix, AsShotNeutral / AsShotWhiteXY, CalibrationIlluminant interpolation, BaselineExposure) |
| TIFF Revision 6.0 | June 1992 | header, IFDs, field types, strips and tiles, PlanarConfiguration, row padding |
| TIFF/EP, ISO 12234-2 | 2001 | CFARepeatPatternDim, CFAPattern |
| TIFF Technical Note 1 (SubIFDs) | 1993 | the SubIFD tree |
| ITU-T T.81 \| ISO/IEC 10918-1 | 09/1992 | lossless JPEG (process 14, SOF3): Annex B syntax, Annex C Huffman tables, F.2.2.3 decoding, Annex H prediction and point transform |
| CinemaDNG Image Data Format Specification | 1.1.0.0 (2009) | frames as numbered DNG files, FrameRate, TimeCodes, ReelName tags |
| SMPTE ST 12-1 | 2014 | time code BCD packing (via the CinemaDNG TimeCodes tag) |
| CIPA DC-008 (Exif) | 2.3 | LightSource codes → illuminant temperatures |
| Malvar, He, Cutler, "High-quality linear interpolation for demosaicing of Bayer-patterned color images", IEEE ICASSP | 2004 | the Bayer demosaic filters |
| McCamy, "Correlated color temperature as an explicit function of chromaticity coordinates", Color Res. Appl. 17(2) | 1992 | white xy → temperature for the calibration interpolation |
| IEC 61966-2-1 (sRGB) | 1999 | Rec. 709 / sRGB primaries and D65 white |
## API
```rust
if filmcraft_dng::sniff(&bytes) {
let dng = filmcraft_dng::Dng::parse(&bytes)?;
let info = dng.info()?; // size after crop, bits, CFA or LinearRaw, frame rate, time code, camera, reel
let raw = dng.raw()?; // the stored samples (u16), bit-exact
let img = dng.develop()?; // linear Rec. 709 RGBA f32 (alpha 1), info.width × info.height
}
let img = filmcraft_dng::decode(&bytes)?; // parse + develop
```
`writer` (feature `writer`, tests only) writes synthetic DNGs: uncompressed at any depth or
lossless JPEG with any predictor, strips or tiles, either byte order, an optional preview IFD with
the raw image in a SubIFD, and arbitrary extra tags.
## What it decodes
- **Container:** little and big-endian TIFF, IFD chains and SubIFDs (at most 64 IFDs, 4 levels,
cycles end the walk). The main image is the largest NewSubFileType 0 IFD with a CFA or LinearRaw
photometric interpretation.
- **Raw data:** 1–16-bit unsigned samples, uncompressed (8 and 16-bit in file byte order, other
depths packed most significant bit first) or lossless JPEG (all predictors 1–7, 1–4 components,
point transform, restart intervals of whole lines), in strips or tiles; tiles decode in
parallel.
- **Processing:** ActiveArea, LinearizationTable, per-site black levels with row / column deltas,
per-sample white levels, as-shot white balance (clipped highlights stay neutral), demosaic
(Malvar–He–Cutler for 2 × 2 Bayer patterns of any phase, bilinear for other patterns up to
8 × 8), DefaultCrop, the DNG colour model to XYZ D50 and Bradford to linear Rec. 709, and
BaselineExposure. LinearRaw (3 samples) skips the demosaic; monochrome LinearRaw is grey.
- **Output:** scene-linear Rec. 709 floats: a white the sensor clipped is 1.0 × 2^BaselineExposure;
brighter values are kept (FilmCraft's compositor works in linear float). Negative results of the
colour matrix are clipped to 0. Black levels scale per pixel by (white − black).
- **White balance without a ForwardMatrix** follows the specification: the camera matrix takes the
values to XYZ and a Bradford adaptation moves the as-shot white to D50. Decoders in the dcraw
tradition (LibRaw included) scale the camera channels instead, so under a white far from D65
their saturated colours differ from ours (greys agree).
- **Speed:** a 4096 × 2160 12-bit frame develops in about 80–100 ms on 16 threads (raw decode
~15 ms, the rest demosaic and colour).
## Not supported (errors name them)
Lossy JPEG and JPEG XL DNG, deflate / floating-point DNG (HDR merges), planar raw data, colour
filters other than red / green / blue (CMY, RGBE), non-rectangular CFA layouts, opcode lists
(warp, vignette, bad pixels: ignored), DefaultScale and BestQualityScale (non-square pixels are
shown square), DNG 1.6 IlluminantData for illuminant 255 (calibration 1 is used), lossless JPEG
restart intervals that split a line, and MXF-wrapped CinemaDNG.
## Tests
- `tests/roundtrip.rs`: container parsing in both byte orders with and without a preview IFD;
uncompressed samples bit-exact at 8–16 bits, strips and tiles; lossless JPEG bit-exact for every
predictor and component count, full-swing 16-bit data (every difference category, SSSS = 16),
restarts and point transform; developed values against hand-computed ones (black / white,
white balance, ForwardMatrix, BaselineExposure, ActiveArea, DefaultCrop, LinearRaw); CinemaDNG
FrameRate / TimeCodes / ReelName; hostile files (every truncation, 12 000 mutations under
`catch_unwind`, cyclic IFDs, huge sizes, bad offsets, zero denominators).
- `tests/oracle_libraw.rs`: LibRaw's `unprocessed_raw` reads the same samples from our synthetic
files (uncompressed 8–16-bit in both byte orders, lossless JPEG predictors 1–7 with one and two
components). Skipped when it is not installed.
- Unit tests: matrix maths, the published sRGB matrix, McCamy temperatures, calibration
interpolation, sRGB / ForwardMatrix cameras mapping to themselves, demosaic on flat fields
(every Bayer phase), ramps and edges, black / white / linearisation maths.

385
crates/dng/src/color.rs Normal file
View File

@ -0,0 +1,385 @@
//! Camera colour to linear Rec. 709, following the DNG specification's "Mapping Camera Color Space
//! to CIE XYZ Space" (DNG 1.4–1.7 chapter 6):
//!
//! - The two calibrations (ColorMatrix, CameraCalibration, ForwardMatrix 1 and 2) are blended
//! linearly in inverse correlated colour temperature between their CalibrationIlluminants.
//! - The white balance is AsShotNeutral (found as a white xy by the specification's iteration) or
//! AsShotWhiteXY; without either, D65.
//! - With ForwardMatrix: CameraToXYZ_D50 = FM · D · (AB · CC)⁻¹, D = diag((AB · CC)⁻¹ · neutral)⁻¹.
//! Without: CameraToXYZ_D50 = CA · (AB · CC · CM)⁻¹, CA a linear Bradford adaptation from the
//! white to D50.
//! - XYZ (D50) goes to linear Rec. 709 / sRGB primaries (D65 white) through Bradford again.
//!
//! The correlated colour temperature of an xy white comes from McCamy's cubic approximation
//! (C. S. McCamy, "Correlated color temperature as an explicit function of chromaticity
//! coordinates", Color Research & Application 17(2), 1992); the illuminant temperatures are the
//! CIE / EXIF 2.3 LightSource values.
use crate::{Error, Result};
pub type M3 = [[f64; 3]; 3];
pub type V3 = [f64; 3];
pub const IDENTITY: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
/// CIE D50 and D65 white chromaticities (the DNG specification's D50).
pub const D50: (f64, f64) = (0.3457, 0.3585);
pub const D65: (f64, f64) = (0.3127, 0.3290);
/// Linear Bradford cone response matrix.
const BRADFORD: M3 = [[0.8951, 0.2664, -0.1614], [-0.7502, 1.7135, 0.0367], [0.0389, -0.0685, 1.0296]];
pub fn mul(a: &M3, b: &M3) -> M3 {
let mut m = [[0.0; 3]; 3];
for (i, row) in m.iter_mut().enumerate() {
for (j, v) in row.iter_mut().enumerate() {
*v = (0..3).map(|k| a[i][k] * b[k][j]).sum();
}
}
m
}
pub fn mulv(a: &M3, v: &V3) -> V3 {
[0, 1, 2].map(|i| a[i][0] * v[0] + a[i][1] * v[1] + a[i][2] * v[2])
}
pub fn diag(v: &V3) -> M3 {
[[v[0], 0.0, 0.0], [0.0, v[1], 0.0], [0.0, 0.0, v[2]]]
}
pub fn scale(a: &M3, s: f64) -> M3 {
a.map(|r| r.map(|v| v * s))
}
fn blend(a: &M3, b: &M3, w: f64) -> M3 {
let mut m = [[0.0; 3]; 3];
for i in 0..3 {
for j in 0..3 {
m[i][j] = w * a[i][j] + (1.0 - w) * b[i][j];
}
}
m
}
/// The inverse of `a`, or `None` when it is (nearly) singular.
pub fn inverse(a: &M3) -> Option<M3> {
let c = |r: usize, s: usize| a[(r + 1) % 3][(s + 1) % 3] * a[(r + 2) % 3][(s + 2) % 3] - a[(r + 1) % 3][(s + 2) % 3] * a[(r + 2) % 3][(s + 1) % 3];
let det = a[0][0] * c(0, 0) + a[0][1] * c(0, 1) + a[0][2] * c(0, 2);
let norm: f64 = a.iter().flatten().map(|v| v.abs()).fold(0.0, f64::max);
if !det.is_finite() || det.abs() <= 1e-12 * norm.powi(3).max(1e-300) {
return None;
}
let mut m = [[0.0; 3]; 3];
for (i, row) in m.iter_mut().enumerate() {
for (j, v) in row.iter_mut().enumerate() {
*v = c(j, i) / det;
}
}
Some(m)
}
/// XYZ (Y = 1) of a chromaticity.
pub fn xy_to_xyz((x, y): (f64, f64)) -> V3 {
let y = y.max(1e-6);
[x / y, 1.0, (1.0 - x - y) / y]
}
pub fn xyz_to_xy(v: &V3) -> Option<(f64, f64)> {
let s = v[0] + v[1] + v[2];
(s.is_finite() && s > 1e-12).then(|| (v[0] / s, v[1] / s))
}
/// Correlated colour temperature (kelvin) of a chromaticity, McCamy (1992), limited to
/// 1 000–50 000 K.
pub fn cct((x, y): (f64, f64)) -> f64 {
let d = 0.1858 - y;
if d.abs() < 1e-9 {
return 50_000.0;
}
let n = (x - 0.3320) / d;
let t = 449.0 * n.powi(3) + 3525.0 * n.powi(2) + 6823.3 * n + 5520.33;
if t.is_finite() { t.clamp(1_000.0, 50_000.0) } else { 50_000.0 }
}
/// Colour temperature of an EXIF LightSource code used as a DNG CalibrationIlluminant.
pub fn illuminant_kelvin(code: u64) -> Option<f64> {
Some(match code {
1 | 4 | 9 => 5500.0, // daylight, flash, fine weather
2 | 14 => 4150.0, // fluorescent, cool white fluorescent (W 3800–4500 K)
3 => 2850.0, // tungsten
10 => 6500.0, // cloudy
11 => 7500.0, // shade
12 => 6430.0, // daylight fluorescent (D 5700–7100 K)
13 => 5050.0, // day white fluorescent (N 4600–5500 K)
15 => 3525.0, // white fluorescent (WW 3250–3800 K)
16 => 2925.0, // warm white fluorescent (L 2600–3250 K)
17 => 2856.0, // standard light A
18 => 4874.0, // standard light B
19 => 6774.0, // standard light C
20 => 5503.0, // D55
21 => 6504.0, // D65
22 => 7504.0, // D75
23 => 5003.0, // D50
24 => 3200.0, // ISO studio tungsten
_ => return None,
})
}
/// Linear Bradford adaptation from one white (XYZ) to another.
pub fn bradford(src: &V3, dst: &V3) -> Option<M3> {
let s = mulv(&BRADFORD, src);
let d = mulv(&BRADFORD, dst);
if s.iter().any(|v| v.abs() < 1e-12) {
return None;
}
let g = [d[0] / s[0], d[1] / s[1], d[2] / s[2]];
Some(mul(&inverse(&BRADFORD)?, &mul(&diag(&g), &BRADFORD)))
}
/// Linear RGB (Rec. 709 / sRGB primaries, D65 white) → XYZ.
pub fn rec709_to_xyz() -> Option<M3> {
let p = [(0.64, 0.33), (0.30, 0.60), (0.15, 0.06)].map(xy_to_xyz);
let pm: M3 = [[p[0][0], p[1][0], p[2][0]], [p[0][1], p[1][1], p[2][1]], [p[0][2], p[1][2], p[2][2]]];
let s = mulv(&inverse(&pm)?, &xy_to_xyz(D65));
Some(mul(&pm, &diag(&s)))
}
/// XYZ relative to D50 → linear Rec. 709.
pub fn xyz_d50_to_rec709() -> Option<M3> {
let to_d65 = bradford(&xy_to_xyz(D50), &xy_to_xyz(D65))?;
Some(mul(&inverse(&rec709_to_xyz()?)?, &to_d65))
}
/// The colour tags of a three-colour DNG.
#[derive(Clone, Debug, Default)]
pub struct Calibration {
pub color_matrix: [Option<M3>; 2],
pub camera_calibration: [Option<M3>; 2],
pub forward_matrix: [Option<M3>; 2],
/// EXIF LightSource codes of the two calibrations.
pub illuminant: [Option<u64>; 2],
pub analog_balance: Option<V3>,
pub as_shot_neutral: Option<V3>,
pub as_shot_white_xy: Option<(f64, f64)>,
pub baseline_exposure: f64,
}
/// The result of [`Calibration::solve`].
#[derive(Clone, Debug, PartialEq)]
pub struct Solved {
/// Camera neutral: raw (normalised) camera values of a white surface. Dividing by it white
/// balances the camera values.
pub neutral: V3,
/// White-balanced camera values → linear Rec. 709, including the baseline exposure. The white
/// balanced white (1, 1, 1) goes to Y = 2^BaselineExposure.
pub matrix: M3,
/// The white point found, for information.
pub white_xy: (f64, f64),
}
impl Calibration {
/// Weight of calibration 1 for a white of temperature `t` (1 when only one is usable).
fn weight(&self, t: f64) -> f64 {
let (Some(_), Some(_)) = (self.color_matrix[0], self.color_matrix[1]) else { return 1.0 };
let (Some(t1), Some(t2)) = (self.illuminant[0].and_then(illuminant_kelvin), self.illuminant[1].and_then(illuminant_kelvin)) else {
return 1.0;
};
if (t1 - t2).abs() < 1.0 {
return 1.0;
}
// the formula needs t1 < t2; swap the roles otherwise
let (lo, hi, lo_is_1) = if t1 < t2 { (t1, t2, true) } else { (t2, t1, false) };
let w_lo = if t <= lo {
1.0
} else if t >= hi {
0.0
} else {
(1.0 / t - 1.0 / hi) / (1.0 / lo - 1.0 / hi)
};
if lo_is_1 { w_lo } else { 1.0 - w_lo }
}
fn pick(&self, m: &[Option<M3>; 2], w: f64, default: M3) -> M3 {
match (m[0], m[1]) {
(Some(a), Some(b)) => blend(&a, &b, w),
(Some(a), None) => a,
(None, Some(b)) if self.color_matrix[0].is_none() => b,
_ => default,
}
}
/// XYZ → camera at weight `w`: AB · CC · CM.
fn xyz_to_camera(&self, w: f64) -> Option<M3> {
let cm = match self.color_matrix {
[None, None] => return None,
_ => self.pick(&self.color_matrix, w, IDENTITY),
};
let cc = self.pick(&self.camera_calibration, w, IDENTITY);
let ab = diag(&self.analog_balance.unwrap_or([1.0; 3]));
Some(mul(&ab, &mul(&cc, &cm)))
}
/// Work out the white balance and the camera → linear Rec. 709 matrix.
pub fn solve(&self) -> Result<Solved> {
let out_matrix = xyz_d50_to_rec709().ok_or(Error::Malformed("colour maths"))?;
let exposure = 2f64.powf(self.baseline_exposure.clamp(-16.0, 16.0));
if self.color_matrix.iter().all(Option::is_none) {
// no colour calibration: the camera values are taken as Rec. 709, white balanced
let neutral = self.as_shot_neutral.unwrap_or([1.0; 3]);
if neutral.iter().any(|v| !v.is_finite() || *v <= 1e-9) {
return Err(Error::Malformed("camera neutral must be positive"));
}
return Ok(Solved { neutral, matrix: scale(&IDENTITY, exposure), white_xy: D65 });
}
let bad = || Error::Malformed("singular colour matrix");
// the white point and the camera neutral
let (white_xy, neutral) = if let Some(n) = self.as_shot_neutral {
let mut xy = D50;
for _ in 0..50 {
let w = self.weight(cct(xy));
let to_xyz = inverse(&self.xyz_to_camera(w).ok_or_else(bad)?).ok_or_else(bad)?;
let next = xyz_to_xy(&mulv(&to_xyz, &n)).ok_or(Error::Malformed("AsShotNeutral is not a colour"))?;
let done = (next.0 - xy.0).abs() + (next.1 - xy.1).abs() < 1e-9;
xy = next;
if done {
break;
}
}
(xy, n)
} else {
let xy = self.as_shot_white_xy.unwrap_or(D65);
let w = self.weight(cct(xy));
let n = mulv(&self.xyz_to_camera(w).ok_or_else(bad)?, &xy_to_xyz(xy));
(xy, n)
};
if neutral.iter().any(|v| !v.is_finite() || *v <= 1e-9) {
return Err(Error::Malformed("camera neutral must be positive"));
}
let w = self.weight(cct(white_xy));
let ab_cc = mul(&diag(&self.analog_balance.unwrap_or([1.0; 3])), &self.pick(&self.camera_calibration, w, IDENTITY));
let camera_to_xyz_d50 = if self.forward_matrix.iter().any(Option::is_some) {
let fm = self.pick(&self.forward_matrix, w, IDENTITY);
let inv_ab_cc = inverse(&ab_cc).ok_or_else(bad)?;
let reference = mulv(&inv_ab_cc, &neutral);
if reference.iter().any(|v| v.abs() < 1e-12) {
return Err(bad());
}
mul(&fm, &mul(&diag(&reference.map(|v| 1.0 / v)), &inv_ab_cc))
} else {
let to_xyz = inverse(&self.xyz_to_camera(w).ok_or_else(bad)?).ok_or_else(bad)?;
let ca = bradford(&xy_to_xyz(white_xy), &xy_to_xyz(D50)).ok_or_else(bad)?;
mul(&ca, &to_xyz)
};
// white-balanced camera values: camera = neutral · wb
let wb_to_xyz = mul(&camera_to_xyz_d50, &diag(&neutral));
let y = mulv(&wb_to_xyz, &[1.0; 3])[1];
if !y.is_finite() || y <= 1e-9 {
return Err(bad());
}
let matrix = scale(&mul(&out_matrix, &wb_to_xyz), exposure / y);
if matrix.iter().flatten().any(|v| !v.is_finite()) {
return Err(bad());
}
Ok(Solved { neutral, matrix, white_xy })
}
}
#[cfg(test)]
mod tests {
use super::*;
fn close(a: &M3, b: &M3, tol: f64) -> bool {
a.iter().flatten().zip(b.iter().flatten()).all(|(x, y)| (x - y).abs() <= tol)
}
#[test]
fn rec709_matrix_matches_the_published_one() {
// IEC 61966-2-1 (sRGB) linear RGB → XYZ, four decimals
let m = rec709_to_xyz().unwrap();
let want = [[0.4124, 0.3576, 0.1805], [0.2126, 0.7152, 0.0722], [0.0193, 0.1192, 0.9505]];
assert!(close(&m, &want, 1e-3), "{m:?}");
}
#[test]
fn inverse_round_trips_and_refuses_singular() {
let a = [[2.0, 1.0, 0.5], [0.1, 3.0, 0.2], [0.3, 0.4, 1.5]];
assert!(close(&mul(&a, &inverse(&a).unwrap()), &IDENTITY, 1e-12));
assert!(inverse(&[[1.0, 2.0, 3.0], [2.0, 4.0, 6.0], [0.0, 0.0, 1.0]]).is_none());
assert!(inverse(&[[f64::NAN; 3]; 3]).is_none());
}
#[test]
fn mccamy_temperatures() {
assert!((cct(D65) - 6504.0).abs() < 15.0, "{}", cct(D65));
assert!((cct(D50) - 5003.0).abs() < 15.0, "{}", cct(D50));
assert!((cct((0.44757, 0.40745)) - 2856.0).abs() < 15.0, "illuminant A");
assert_eq!(cct((0.3320, 0.1858)), 50_000.0, "no division by zero");
}
#[test]
fn interpolation_weight_follows_inverse_temperature() {
let c = Calibration { color_matrix: [Some(IDENTITY), Some(IDENTITY)], illuminant: [Some(17), Some(21)], ..Default::default() };
assert_eq!(c.weight(2000.0), 1.0);
assert_eq!(c.weight(9000.0), 0.0);
let mid = c.weight(4000.0);
let want = (1.0 / 4000.0 - 1.0 / 6504.0) / (1.0 / 2856.0 - 1.0 / 6504.0);
assert!((mid - want).abs() < 1e-12);
// swapped order: weight of calibration 1 (now D65)
let s = Calibration { illuminant: [Some(21), Some(17)], ..c.clone() };
assert!((s.weight(4000.0) - (1.0 - want)).abs() < 1e-12);
// unknown illuminant: calibration 1 only
let u = Calibration { illuminant: [Some(0), Some(21)], ..c };
assert_eq!(u.weight(4000.0), 1.0);
}
#[test]
fn an_srgb_camera_maps_to_itself() {
// ColorMatrix = XYZ → linear sRGB, white D65: the camera is Rec. 709 already
let cm = inverse(&rec709_to_xyz().unwrap()).unwrap();
let c = Calibration { color_matrix: [Some(cm), None], illuminant: [Some(21), None], as_shot_neutral: Some([1.0; 3]), ..Default::default() };
let s = c.solve().unwrap();
assert!(close(&s.matrix, &IDENTITY, 1e-6), "{:?}", s.matrix);
assert!((s.white_xy.0 - D65.0).abs() < 1e-4 && (s.white_xy.1 - D65.1).abs() < 1e-4, "{:?}", s.white_xy);
// a baseline exposure of +1 doubles everything
let e = Calibration { baseline_exposure: 1.0, ..c }.solve().unwrap();
assert!(close(&e.matrix, &scale(&IDENTITY, 2.0), 1e-6));
}
#[test]
fn forward_matrix_white_balances_by_the_neutral() {
// FM = linear Rec. 709 → XYZ D50: the white-balanced camera values come out unchanged
let fm = mul(&bradford(&xy_to_xyz(D65), &xy_to_xyz(D50)).unwrap(), &rec709_to_xyz().unwrap());
let cm = inverse(&rec709_to_xyz().unwrap()).unwrap();
let c = Calibration {
color_matrix: [Some(cm), None],
forward_matrix: [Some(fm), None],
illuminant: [Some(21), None],
as_shot_neutral: Some([0.5, 1.0, 0.8]),
..Default::default()
};
let s = c.solve().unwrap();
assert_eq!(s.neutral, [0.5, 1.0, 0.8]);
assert!(close(&s.matrix, &IDENTITY, 1e-6), "{:?}", s.matrix);
}
#[test]
fn white_xy_gives_the_neutral() {
let cm = inverse(&rec709_to_xyz().unwrap()).unwrap();
let c = Calibration { color_matrix: [Some(cm), None], illuminant: [Some(21), None], as_shot_white_xy: Some(D65), ..Default::default() };
let s = c.solve().unwrap();
for v in s.neutral {
assert!((v - s.neutral[1]).abs() < 1e-3, "D65 is neutral for an sRGB camera: {:?}", s.neutral);
}
}
#[test]
fn hostile_calibrations_are_errors() {
let zero = Calibration { color_matrix: [Some([[0.0; 3]; 3]), None], as_shot_neutral: Some([1.0; 3]), ..Default::default() };
assert!(zero.solve().is_err());
let cm = inverse(&rec709_to_xyz().unwrap()).unwrap();
let neg = Calibration { color_matrix: [Some(cm), None], as_shot_neutral: Some([-1.0, 1.0, 1.0]), ..Default::default() };
assert!(neg.solve().is_err());
let none = Calibration { as_shot_neutral: Some([0.5, 1.0, 0.5]), ..Default::default() }.solve().unwrap();
assert_eq!(none.matrix, IDENTITY);
}
}

255
crates/dng/src/demosaic.rs Normal file
View File

@ -0,0 +1,255 @@
//! Demosaicing: one colour sample per pixel (a colour filter array) to three.
//!
//! - A 2 × 2 Bayer pattern (one red, two green, one blue) uses the gradient-corrected linear
//! interpolation of H. S. Malvar, L.-W. He and R. Cutler, "High-quality linear interpolation for
//! demosaicing of Bayer-patterned color images", IEEE ICASSP 2004: bilinear interpolation plus a
//! Laplacian correction from the pixel's own colour, as 5 × 5 filters.
//! - Any other pattern (up to 8 × 8) uses bilinear interpolation: the mean of the nearest samples
//! of each missing colour in the 3 × 3 neighbourhood, else the 5 × 5 one.
//!
//! The picture is extended by two pixels on every side by mirroring (without repeating the edge),
//! which keeps a 2 × 2 pattern's phase.
#[cfg(feature = "threads")]
use rayon::prelude::*;
/// A CFA pattern: `rows` × `cols` colour indices (0 red, 1 green, 2 blue), origin at the top left
/// of the image.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Pattern {
pub rows: usize,
pub cols: usize,
pub colors: Vec<u8>,
}
impl Pattern {
pub fn color(&self, x: usize, y: usize) -> u8 {
self.colors.get((y % self.rows) * self.cols + x % self.cols).copied().unwrap_or(1)
}
/// Whether this is a 2 × 2 Bayer pattern.
pub fn is_bayer(&self) -> bool {
if (self.rows, self.cols) != (2, 2) {
return false;
}
let [a, b, c, d] = self.colors[..] else { return false };
let count = |k: u8| [a, b, c, d].iter().filter(|&&v| v == k).count();
// the two greens sit on a diagonal
count(0) == 1 && count(1) == 2 && count(2) == 1 && ((a == 1 && d == 1) || (b == 1 && c == 1))
}
}
const PAD: usize = 2;
/// Mirror index `i` (which may be up to PAD outside) into 0..n.
fn mirror(i: isize, n: usize) -> usize {
let n = n as isize;
let mut i = i;
if i < 0 {
i = -i;
}
if i >= n {
i = 2 * (n - 1) - i;
}
i.clamp(0, n - 1) as usize
}
/// The plane extended by PAD mirrored pixels on each side.
fn padded(plane: &[f32], w: usize, h: usize) -> (Vec<f32>, usize) {
let pw = w + 2 * PAD;
let ph = h + 2 * PAD;
let mut out = vec![0f32; pw * ph];
for y in 0..ph {
let sy = mirror(y as isize - PAD as isize, h);
for x in 0..pw {
let sx = mirror(x as isize - PAD as isize, w);
out[y * pw + x] = plane.get(sy * w + sx).copied().unwrap_or(0.0);
}
}
(out, pw)
}
/// Demosaic `plane` (`w` × `h`, one sample per pixel) into interleaved RGB.
pub fn demosaic(plane: &[f32], w: usize, h: usize, pattern: &Pattern) -> Vec<f32> {
let mut out = vec![0f32; w * h * 3];
if w == 0 || h == 0 || plane.len() < w * h {
return out;
}
let (p, pw) = padded(plane, w, h);
let bayer = pattern.is_bayer() && w >= 3 && h >= 3;
let row = |y: usize, o: &mut [f32]| {
if bayer {
malvar_row(&p, pw, y, w, pattern, o);
} else {
bilinear_row(&p, pw, y, w, h, pattern, o);
}
};
#[cfg(feature = "threads")]
out.par_chunks_mut(w * 3).enumerate().for_each(|(y, o)| row(y, o));
#[cfg(not(feature = "threads"))]
out.chunks_mut(w * 3).enumerate().for_each(|(y, o)| row(y, o));
out
}
fn malvar_row(p: &[f32], pw: usize, y: usize, w: usize, pat: &Pattern, out: &mut [f32]) {
let py = y + PAD;
for x in 0..w {
let px = x + PAD;
let at = |dx: isize, dy: isize| p[((py as isize + dy) as usize) * pw + (px as isize + dx) as usize];
let c = at(0, 0);
let cross1 = at(-1, 0) + at(1, 0) + at(0, -1) + at(0, 1);
let cross2 = at(-2, 0) + at(2, 0) + at(0, -2) + at(0, 2);
let diag = at(-1, -1) + at(1, -1) + at(-1, 1) + at(1, 1);
let me = pat.color(x, y);
let rgb = match me {
1 => {
let h1 = at(-1, 0) + at(1, 0);
let h2 = at(-2, 0) + at(2, 0);
let v1 = at(0, -1) + at(0, 1);
let v2 = at(0, -2) + at(0, 2);
// colour along the row and along the column
let along_row = (5.0 * c + 4.0 * h1 - h2 - diag + 0.5 * v2) / 8.0;
let along_col = (5.0 * c + 4.0 * v1 - v2 - diag + 0.5 * h2) / 8.0;
if pat.color(x + 1, y) == 0 { [along_row, c, along_col] } else { [along_col, c, along_row] }
}
_ => {
let g = (4.0 * c + 2.0 * cross1 - cross2) / 8.0;
let other = (6.0 * c + 2.0 * diag - 1.5 * cross2) / 8.0;
if me == 0 { [c, g, other] } else { [other, g, c] }
}
};
if let Some(o) = out.get_mut(x * 3..x * 3 + 3) {
o.copy_from_slice(&rgb);
}
}
}
fn bilinear_row(p: &[f32], pw: usize, y: usize, w: usize, h: usize, pat: &Pattern, out: &mut [f32]) {
for x in 0..w {
let me = pat.color(x, y);
let mut rgb = [0f32; 3];
for (k, v) in rgb.iter_mut().enumerate() {
if usize::from(me) == k {
*v = p[(y + PAD) * pw + x + PAD];
continue;
}
for r in 1..=PAD as isize {
let (mut sum, mut n) = (0f32, 0u32);
for dy in -r..=r {
for dx in -r..=r {
// the colour of a mirrored neighbour is the colour where it was mirrored from
let sx = mirror(x as isize + dx, w);
let sy = mirror(y as isize + dy, h);
if usize::from(pat.color(sx, sy)) == k {
sum += p[(sy + PAD) * pw + sx + PAD];
n += 1;
}
}
}
if n > 0 {
*v = sum / n as f32;
break;
}
}
}
if let Some(o) = out.get_mut(x * 3..x * 3 + 3) {
o.copy_from_slice(&rgb);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rggb() -> Pattern {
Pattern { rows: 2, cols: 2, colors: vec![0, 1, 1, 2] }
}
/// A mosaic of an RGB picture given per pixel.
fn mosaic(w: usize, h: usize, pat: &Pattern, rgb: impl Fn(usize, usize) -> [f32; 3]) -> Vec<f32> {
(0..h).flat_map(|y| (0..w).map(move |x| (x, y))).map(|(x, y)| rgb(x, y)[usize::from(pat.color(x, y))]).collect()
}
#[test]
fn bayer_detection() {
assert!(rggb().is_bayer());
for c in [[1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
assert!(Pattern { rows: 2, cols: 2, colors: c.to_vec() }.is_bayer(), "{c:?}");
}
assert!(!Pattern { rows: 2, cols: 2, colors: vec![0, 1, 2, 1] }.is_bayer(), "greens side by side");
assert!(!Pattern { rows: 2, cols: 2, colors: vec![0, 0, 1, 2] }.is_bayer());
assert!(!Pattern { rows: 1, cols: 3, colors: vec![0, 1, 2] }.is_bayer());
}
#[test]
fn flat_fields_stay_flat_for_every_bayer_phase_and_other_patterns() {
let pats = [[0, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]].map(|c| Pattern { rows: 2, cols: 2, colors: c.to_vec() });
let stripes = Pattern { rows: 1, cols: 3, colors: vec![0, 1, 2] };
for pat in pats.iter().chain([&stripes]) {
let (w, h) = (9, 7);
let m = mosaic(w, h, pat, |_, _| [0.25, 0.5, 0.75]);
let out = demosaic(&m, w, h, pat);
for (i, px) in out.chunks(3).enumerate() {
for (k, want) in [0.25, 0.5, 0.75].iter().enumerate() {
assert!((px[k] - want).abs() < 1e-6, "{pat:?} pixel {i} channel {k}: {}", px[k]);
}
}
}
}
#[test]
fn known_samples_are_kept_and_linear_ramps_are_exact() {
// a horizontal ramp is reconstructed exactly away from the border (all filters are
// symmetric and sum to one, the correction terms sum to zero)
let pat = rggb();
let (w, h) = (16, 12);
let f = |x: usize, _y: usize| {
let v = x as f32 / 16.0;
[v, 0.5 * v + 0.1, 0.2 + 0.25 * v]
};
let m = mosaic(w, h, &pat, f);
let out = demosaic(&m, w, h, &pat);
for y in 0..h {
for x in 0..w {
let px = &out[(y * w + x) * 3..][..3];
let own = usize::from(pat.color(x, y));
assert_eq!(px[own], m[y * w + x], "own sample kept");
if (2..w - 2).contains(&x) {
let want = f(x, y);
for k in 0..3 {
assert!((px[k] - want[k]).abs() < 1e-5, "({x},{y}) ch {k}: {} vs {}", px[k], want[k]);
}
}
}
}
}
#[test]
fn a_vertical_edge_does_not_smear_far() {
// white left half, black right half: pixels two columns from the edge are untouched
let pat = rggb();
let (w, h) = (12, 8);
let m = mosaic(w, h, &pat, |x, _| if x < 6 { [1.0; 3] } else { [0.0; 3] });
let out = demosaic(&m, w, h, &pat);
for y in 0..h {
for x in (0..3).chain(9..12) {
let want = if x < 6 { 1.0 } else { 0.0 };
for k in 0..3 {
assert!((out[(y * w + x) * 3 + k] - want).abs() < 1e-6, "({x},{y}) ch {k}");
}
}
}
}
#[test]
fn tiny_and_empty_pictures_do_not_panic() {
let pat = rggb();
for (w, h) in [(0, 0), (1, 1), (2, 1), (1, 5), (2, 2), (3, 2)] {
let m = vec![0.5; w * h];
let out = demosaic(&m, w, h, &pat);
assert_eq!(out.len(), w * h * 3);
}
assert_eq!(demosaic(&[0.5; 3], 4, 4, &pat).len(), 48, "short plane");
}
}

358
crates/dng/src/lib.rs Normal file
View File

@ -0,0 +1,358 @@
//! Clean-room DNG and CinemaDNG decoder, implemented from the public specifications only (see the
//! crate README for editions): the TIFF 6.0 / TIFF-EP container, uncompressed and lossless JPEG
//! (ITU-T T.81 process 14) raw data, and the DNG processing model (linearisation, black and white
//! levels, white balance, demosaic, default crop, colour matrices) to a scene-linear Rec. 709
//! picture.
//!
//! - [`sniff`]: is this a DNG (a TIFF with a DNGVersion tag)?
//! - [`Dng::parse`] → [`Dng::info`] (size, bit depth, CinemaDNG frame rate and time code),
//! [`Dng::raw`] (the raw samples) and [`Dng::develop`] (linear RGBA floats).
//! - [`decode`]: parse and develop in one call.
//!
//! Every number in the file is hostile: offsets, counts and sizes are checked, allocations are
//! capped by [`raw::MAX_SAMPLES`] and by the amount of data present, and the IFD walk is bounded.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
pub mod color;
pub mod demosaic;
pub mod ljpeg;
pub mod process;
pub mod raw;
pub mod tiff;
#[cfg(any(test, feature = "writer"))]
pub mod writer;
use color::{Calibration, M3, V3};
use demosaic::Pattern;
use process::Params;
use raw::{PHOTOMETRIC_CFA, RawImage};
use tiff::{Entry, Ifd, Tiff, tags};
/// DNG decoding errors.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum Error {
#[error("not a DNG file")]
NotDng,
#[error("truncated DNG data")]
Truncated,
#[error("raw image too large")]
TooLarge,
#[error("malformed DNG: {0}")]
Malformed(&'static str),
#[error("unsupported DNG: {0}")]
Unsupported(String),
}
pub type Result<T> = std::result::Result<T, Error>;
/// Whether `b` is a DNG: a TIFF file whose first IFD has a DNGVersion tag.
pub fn sniff(b: &[u8]) -> bool {
let Ok((t, first)) = Tiff::new(b) else { return false };
t.read_ifd(first, 0).is_ok_and(|(ifd, _)| ifd.get(tags::DNG_VERSION).is_some())
}
/// SMPTE ST 12-1 time code of a CinemaDNG frame.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Timecode {
pub hours: u8,
pub minutes: u8,
pub seconds: u8,
pub frames: u8,
pub drop_frame: bool,
}
impl Timecode {
/// The first time code of a CinemaDNG TimeCodes tag (SMPTE ST 331 / 12-1 packing: BCD frames,
/// seconds, minutes, hours in bytes 0–3, the drop-frame flag in bit 6 of byte 0).
pub fn from_bytes(b: &[u8]) -> Option<Timecode> {
let bcd = |v: u8, tens_mask: u8| {
let (u, t) = (v & 0x0F, (v >> 4) & tens_mask);
(u <= 9).then_some(t * 10 + u)
};
let [f, s, m, h, ..] = *b else { return None };
let tc = Timecode { frames: bcd(f, 0x3)?, seconds: bcd(s, 0x7)?, minutes: bcd(m, 0x7)?, hours: bcd(h, 0x3)?, drop_frame: f & 0x40 != 0 };
(tc.seconds < 60 && tc.minutes < 60 && tc.hours < 24).then_some(tc)
}
}
/// What a DNG holds, without decoding the pixels.
#[derive(Clone, Debug, PartialEq)]
pub struct Info {
/// Size of the developed picture (the default crop).
pub width: usize,
pub height: usize,
/// Size of the stored raw image.
pub raw_width: usize,
pub raw_height: usize,
pub bits: u32,
pub samples_per_pixel: usize,
/// Colour filter array data (else LinearRaw).
pub cfa: bool,
/// "uncompressed" or "lossless JPEG".
pub compression: &'static str,
pub dng_version: [u8; 4],
/// CinemaDNG FrameRate (numerator, denominator).
pub frame_rate: Option<(i64, i64)>,
/// CinemaDNG time code of this frame.
pub timecode: Option<Timecode>,
/// UniqueCameraModel, else Make and Model.
pub camera: Option<String>,
/// CinemaDNG ReelName.
pub reel: Option<String>,
}
/// A developed DNG: premultiplied, scene-linear Rec. 709 RGBA floats (alpha 1), top row first.
/// A white surface the sensor saw at its clipping level is 1.0 times 2^BaselineExposure.
pub struct Image {
pub info: Info,
pub width: usize,
pub height: usize,
pub rgba: Vec<f32>,
}
/// A parsed DNG file.
pub struct Dng<'a> {
tiff: Tiff<'a>,
ifds: Vec<Ifd>,
/// Index of the main raw IFD in `ifds` (the first is IFD 0).
raw: usize,
}
impl<'a> Dng<'a> {
/// Parse the container and find the main raw image.
pub fn parse(bytes: &'a [u8]) -> Result<Dng<'a>> {
let (tiff, first) = Tiff::new(bytes)?;
let ifds = tiff.all_ifds(first)?;
if ifds.first().and_then(|i| i.get(tags::DNG_VERSION)).is_none() {
return Err(Error::NotDng);
}
let main = raw::find_raw_ifd(&tiff, &ifds)?;
let raw = ifds.iter().position(|i| i.offset == main.offset).ok_or(Error::Malformed("raw IFD"))?;
Ok(Dng { tiff, ifds, raw })
}
fn ifd0(&self) -> &Ifd {
static EMPTY: Ifd = Ifd { offset: 0, entries: Vec::new(), depth: 0 };
self.ifds.first().unwrap_or(&EMPTY)
}
fn raw_ifd(&self) -> &Ifd {
self.ifds.get(self.raw).unwrap_or(self.ifd0())
}
/// A raw-image tag: from the raw IFD, else IFD 0.
fn raw_tag(&self, tag: u16) -> Option<&Entry> {
self.raw_ifd().get(tag).or_else(|| self.ifd0().get(tag))
}
/// A camera-wide tag: from IFD 0, else the raw IFD.
fn main_tag(&self, tag: u16) -> Option<&Entry> {
self.ifd0().get(tag).or_else(|| self.raw_ifd().get(tag))
}
/// A numeric tag's values; a value outside the file (or a zero denominator) is an error, so a
/// damaged file does not silently lose its white balance.
fn floats(&self, e: Option<&Entry>) -> Result<Option<Vec<f64>>> {
e.map(|e| self.tiff.floats(e)).transpose()
}
/// A 3 × 3 matrix tag (`None` when absent or not 3 × 3, e.g. for other colour counts).
fn matrix(&self, tag: u16) -> Result<Option<M3>> {
let Some(v) = self.floats(self.main_tag(tag))? else { return Ok(None) };
let Ok(v) = <[f64; 9]>::try_from(v.as_slice()) else { return Ok(None) };
Ok(Some([[v[0], v[1], v[2]], [v[3], v[4], v[5]], [v[6], v[7], v[8]]]))
}
fn vec3(&self, tag: u16) -> Result<Option<V3>> {
Ok(self.floats(self.main_tag(tag))?.and_then(|v| <[f64; 3]>::try_from(v.as_slice()).ok()))
}
fn text(&self, tag: u16) -> Option<String> {
self.main_tag(tag).and_then(|e| self.tiff.ascii(e).ok()).filter(|s| !s.is_empty())
}
/// The CFA pattern in camera planes (`None` for LinearRaw).
fn cfa(&self) -> Result<Option<Pattern>> {
let l = raw::layout(&self.tiff, self.raw_ifd())?;
if l.photometric != PHOTOMETRIC_CFA {
return Ok(None);
}
let dims = self.raw_tag(tags::CFA_REPEAT_PATTERN_DIM).map(|e| self.tiff.uints(e)).transpose()?.unwrap_or_else(|| vec![2, 2]);
let (rows, cols) = match dims[..] {
[r, c, ..] => (r as usize, c as usize),
_ => return Err(Error::Malformed("CFARepeatPatternDim")),
};
if !(1..=8).contains(&rows) || !(1..=8).contains(&cols) {
return Err(Error::Unsupported(format!("{rows} × {cols} CFA pattern")));
}
let pattern = self.tiff.uints(self.raw_tag(tags::CFA_PATTERN).ok_or(Error::Malformed("no CFAPattern"))?)?;
if pattern.len() < rows * cols {
return Err(Error::Malformed("CFAPattern shorter than its repeat"));
}
let planes = match self.raw_tag(tags::CFA_PLANE_COLOR) {
Some(e) => self.tiff.uints(e)?,
None => vec![0, 1, 2],
};
let mut sorted = planes.clone();
sorted.sort_unstable();
if sorted != [0, 1, 2] {
return Err(Error::Unsupported("CFA colours other than red, green and blue".into()));
}
if let Some(e) = self.raw_tag(tags::CFA_LAYOUT)
&& self.tiff.uint(e)? != 1
{
return Err(Error::Unsupported("a non-rectangular CFA layout".into()));
}
let colors = pattern[..rows * cols]
.iter()
.map(|&v| planes.iter().position(|&p| p == v).map(|i| i as u8).ok_or(Error::Malformed("CFAPattern colour not in CFAPlaneColor")))
.collect::<Result<Vec<u8>>>()?;
Ok(Some(Pattern { rows, cols, colors }))
}
/// The processing tags of the raw image.
pub fn params(&self) -> Result<Params> {
let uint_pair = |tag| -> Result<Option<(usize, usize)>> {
match self.raw_tag(tag) {
Some(e) => match self.tiff.uints(e)?[..] {
[a, b, ..] => Ok(Some((a as usize, b as usize))),
_ => Err(Error::Malformed("short field")),
},
None => Ok(None),
}
};
let active_area = match self.raw_tag(tags::ACTIVE_AREA) {
Some(e) => match self.tiff.uints(e)?[..] {
[t, l, b, r, ..] => Some([t, l, b, r].map(|v| usize::try_from(v).unwrap_or(usize::MAX))),
_ => return Err(Error::Malformed("ActiveArea")),
},
None => None,
};
let linearization = match self.raw_tag(tags::LINEARIZATION_TABLE) {
Some(e) => {
let v = self.tiff.uints(e)?;
if v.len() > 65_536 {
return Err(Error::Malformed("LinearizationTable longer than 65536"));
}
Some(v.into_iter().map(|x| x.min(65_535) as u16).collect())
}
None => None,
};
let pair = |tag| -> Result<Option<(f64, f64)>> {
match self.raw_tag(tag) {
Some(e) => match self.tiff.floats(e)?[..] {
[a, b, ..] => Ok(Some((a, b))),
_ => Err(Error::Malformed("short field")),
},
None => Ok(None),
}
};
let list = |tag| -> Result<Vec<f64>> { self.raw_tag(tag).map(|e| self.tiff.floats(e)).transpose().map(Option::unwrap_or_default) };
Ok(Params {
active_area,
linearization,
black_repeat: uint_pair(tags::BLACK_LEVEL_REPEAT_DIM)?.unwrap_or((1, 1)),
black: list(tags::BLACK_LEVEL)?,
black_delta_h: list(tags::BLACK_LEVEL_DELTA_H)?,
black_delta_v: list(tags::BLACK_LEVEL_DELTA_V)?,
white: list(tags::WHITE_LEVEL)?,
crop_origin: pair(tags::DEFAULT_CROP_ORIGIN)?,
crop_size: pair(tags::DEFAULT_CROP_SIZE)?,
cfa: self.cfa()?,
})
}
/// The colour tags.
pub fn calibration(&self) -> Result<Calibration> {
let illum = |tag| self.main_tag(tag).map(|e| self.tiff.uint(e)).transpose();
let white_xy = self.floats(self.main_tag(tags::AS_SHOT_WHITE_XY))?.and_then(|v| match v[..] {
[x, y] => Some((x, y)),
_ => None,
});
Ok(Calibration {
color_matrix: [self.matrix(tags::COLOR_MATRIX_1)?, self.matrix(tags::COLOR_MATRIX_2)?],
camera_calibration: [self.matrix(tags::CAMERA_CALIBRATION_1)?, self.matrix(tags::CAMERA_CALIBRATION_2)?],
forward_matrix: [self.matrix(tags::FORWARD_MATRIX_1)?, self.matrix(tags::FORWARD_MATRIX_2)?],
illuminant: [illum(tags::CALIBRATION_ILLUMINANT_1)?, illum(tags::CALIBRATION_ILLUMINANT_2)?],
analog_balance: self.vec3(tags::ANALOG_BALANCE)?,
as_shot_neutral: self.vec3(tags::AS_SHOT_NEUTRAL)?,
as_shot_white_xy: white_xy,
baseline_exposure: self.floats(self.main_tag(tags::BASELINE_EXPOSURE))?.and_then(|v| v.first().copied()).unwrap_or(0.0),
})
}
/// Size, depth and CinemaDNG metadata.
pub fn info(&self) -> Result<Info> {
let l = raw::layout(&self.tiff, self.raw_ifd())?;
let p = self.params()?;
let (_, _, aw, ah) = process::active_rect(&p, l.width, l.height)?;
let (_, _, width, height) = process::crop_rect(&p, aw, ah);
let dng_version = self
.main_tag(tags::DNG_VERSION)
.and_then(|e| self.tiff.uints(e).ok())
.and_then(|v| v.get(..4).map(|v| [0, 1, 2, 3].map(|i| v[i] as u8)))
.unwrap_or_default();
let frame_rate = self.main_tag(tags::FRAME_RATE).and_then(|e| {
let b = self.tiff.value_bytes(e).ok()?;
if !matches!(e.ty, tiff::Type::SRational | tiff::Type::Rational) {
return None;
}
let n = i64::from(self.tiff.u32_of(b.get(0..4)?.try_into().ok()?) as i32);
let d = i64::from(self.tiff.u32_of(b.get(4..8)?.try_into().ok()?) as i32);
(n > 0 && d > 0 && n <= 1000 * d && n * 1000 >= d).then_some((n, d))
});
let timecode = self.main_tag(tags::TIME_CODES).and_then(|e| self.tiff.value_bytes(e).ok()).and_then(Timecode::from_bytes);
let camera = self.text(tags::UNIQUE_CAMERA_MODEL).or_else(|| {
let mm = [self.text(tags::MAKE), self.text(tags::MODEL)].into_iter().flatten().collect::<Vec<_>>().join(" ");
(!mm.is_empty()).then_some(mm)
});
Ok(Info {
width,
height,
raw_width: l.width,
raw_height: l.height,
bits: l.bits,
samples_per_pixel: l.samples_per_pixel,
cfa: p.cfa.is_some(),
compression: if l.compression == 7 { "lossless JPEG" } else { "uncompressed" },
dng_version,
frame_rate,
timecode,
camera,
reel: self.text(tags::REEL_NAME),
})
}
/// The raw samples of the main image, as stored (before any processing).
pub fn raw(&self) -> Result<RawImage> {
raw::read(&self.tiff, self.raw_ifd())
}
/// Develop the main image to scene-linear Rec. 709.
pub fn develop(&self) -> Result<Image> {
let info = self.info()?;
let p = self.params()?;
let raw = self.raw()?;
let calibration = self.calibration()?;
let solved = if p.cfa.is_none() && raw.samples_per_pixel == 1 {
// monochrome: no colour, only the exposure
let e = 2f64.powf(calibration.baseline_exposure.clamp(-16.0, 16.0));
color::Solved { neutral: [1.0; 3], matrix: color::scale(&color::IDENTITY, e), white_xy: color::D65 }
} else {
calibration.solve()?
};
let d = process::develop(&raw, &p, &solved.neutral, &solved.matrix)?;
Ok(Image { info, width: d.width, height: d.height, rgba: d.rgba })
}
}
/// Parse and develop a DNG file.
pub fn decode(bytes: &[u8]) -> Result<Image> {
Dng::parse(bytes)?.develop()
}
/// Parse a DNG file's metadata.
pub fn probe(bytes: &[u8]) -> Result<Info> {
Dng::parse(bytes)?.info()
}

395
crates/dng/src/ljpeg.rs Normal file
View File

@ -0,0 +1,395 @@
//! Lossless JPEG decoding: ITU-T T.81 | ISO/IEC 10918-1 Annex H (process 14, Huffman coded,
//! "SOF3"), the compression DNG calls lossless JPEG (Compression = 7).
//!
//! - Frame header (B.2.2): 2–16-bit precision, 1–4 components, each sampled 1×1.
//! - Huffman tables (B.2.4.2, Annex C) decoded with a 9-bit lookup and the F.2.2.3 code-length
//! walk for longer codes.
//! - Scan header (B.2.3): one scan holding every component, predictor `Ss` 1–7 (Table H.1) and
//! point transform `Al`.
//! - Prediction (H.1.2.1): the first line (and the first line after a restart) predicts from the
//! left neighbour with 2^(P − Pt − 1) for its first sample; later lines predict their first
//! sample from the one above and the rest with the selected predictor. Differences are added
//! modulo 2^16 (H.1.2.2) and the SSSS = 16 category is 32768 with no extra bits.
//! - Restart intervals (DRI, RSTn) that span whole lines.
use crate::{Error, Result};
/// A decoded lossless JPEG image.
#[derive(Debug)]
pub struct Decoded {
pub width: usize,
pub height: usize,
pub components: usize,
pub precision: u8,
pub predictor: u8,
/// Samples, interleaved by component, line after line, shifted back up by the point transform.
pub samples: Vec<u16>,
}
const LOOKUP_BITS: u32 = 9;
/// A Huffman table for decoding (T.81 Annex C and F.2.2.3).
#[derive(Clone)]
struct Huffman {
/// For codes of up to LOOKUP_BITS bits: (length, value) indexed by the next LOOKUP_BITS bits;
/// length 0 means "longer code".
fast: Vec<(u8, u8)>,
maxcode: [i32; 17],
valptr: [i32; 17],
mincode: [i32; 17],
values: Vec<u8>,
}
impl Huffman {
fn new(bits: &[u8; 16], values: Vec<u8>) -> Result<Huffman> {
// C.2: code sizes and codes
let mut sizes = Vec::new();
for (l, &n) in bits.iter().enumerate() {
for _ in 0..n {
sizes.push(l as u8 + 1);
}
}
if sizes.len() != values.len() || sizes.is_empty() {
return Err(Error::Malformed("lossless JPEG: Huffman table counts"));
}
let mut codes = Vec::with_capacity(sizes.len());
let mut code: u32 = 0;
let mut si = sizes[0];
let mut k = 0;
while k < sizes.len() {
while k < sizes.len() && sizes[k] == si {
codes.push(code);
code += 1;
k += 1;
}
// the codes of one length must fit in that length
if code > (1u32 << si) {
return Err(Error::Malformed("lossless JPEG: Huffman table over-subscribed"));
}
code <<= 1;
si += 1;
}
// F.2.2.3 decoder tables
let mut maxcode = [-1i32; 17];
let mut valptr = [0i32; 17];
let mut mincode = [0i32; 17];
let mut j = 0usize;
for l in 1..=16usize {
let n = usize::from(bits[l - 1]);
if n == 0 {
maxcode[l] = -1;
} else {
valptr[l] = j as i32;
mincode[l] = codes[j] as i32;
j += n;
maxcode[l] = codes[j - 1] as i32;
}
}
let mut fast = vec![(0u8, 0u8); 1 << LOOKUP_BITS];
for (i, (&c, &s)) in codes.iter().zip(&sizes).enumerate() {
let s = u32::from(s);
if s <= LOOKUP_BITS {
let shift = LOOKUP_BITS - s;
let base = (c << shift) as usize;
for f in fast.iter_mut().skip(base).take(1 << shift) {
*f = (s as u8, values[i]);
}
}
}
Ok(Huffman { fast, maxcode, valptr, mincode, values })
}
}
/// Entropy-coded data reader: removes stuffed zero bytes and stops at markers (feeding zeros).
struct Bits<'a> {
data: &'a [u8],
pos: usize,
acc: u64,
n: u32,
/// The marker that ended the data (after 0xFF), if one was met.
marker: Option<u8>,
}
impl<'a> Bits<'a> {
fn new(data: &'a [u8], pos: usize) -> Self {
Bits { data, pos, acc: 0, n: 0, marker: None }
}
fn fill(&mut self) {
while self.n <= 56 {
let mut byte = 0u8;
if self.marker.is_none() {
match self.data.get(self.pos) {
Some(&0xFF) => match self.data.get(self.pos + 1) {
Some(&0x00) => {
byte = 0xFF;
self.pos += 2;
}
Some(&m) => self.marker = Some(m),
None => self.marker = Some(0xD9),
},
Some(&b) => {
byte = b;
self.pos += 1;
}
None => self.marker = Some(0xD9),
}
}
self.acc |= u64::from(byte) << (56 - self.n);
self.n += 8;
}
}
fn peek(&mut self, k: u32) -> u32 {
if self.n < k {
self.fill();
}
(self.acc >> (64 - k)) as u32
}
fn skip(&mut self, k: u32) {
self.acc <<= k;
self.n -= k;
}
fn bits(&mut self, k: u32) -> u32 {
if k == 0 {
return 0;
}
let v = self.peek(k);
self.skip(k);
v
}
/// Decode one Huffman symbol (F.2.2.3).
fn decode(&mut self, h: &Huffman) -> Result<u8> {
let p = self.peek(LOOKUP_BITS) as usize;
if let Some(&(len, v)) = h.fast.get(p)
&& len > 0
{
self.skip(u32::from(len));
return Ok(v);
}
let mut code = self.bits(1) as i32;
for l in 1..=16usize {
if code <= h.maxcode[l] {
let i = h.valptr[l] + code - h.mincode[l];
return h.values.get(i as usize).copied().ok_or(Error::Malformed("lossless JPEG: bad Huffman code"));
}
code = (code << 1) | self.bits(1) as i32;
}
Err(Error::Malformed("lossless JPEG: bad Huffman code"))
}
/// Skip to the byte after the next RSTn marker, starting a new bit stream there.
fn restart(&mut self) -> Result<()> {
// the decoder may have looked ahead into fill bits; find the marker in the bytes
let mut p = self.pos;
loop {
match (self.data.get(p), self.data.get(p + 1)) {
(Some(0xFF), Some(&m)) if (0xD0..=0xD7).contains(&m) => {
*self = Bits::new(self.data, p + 2);
return Ok(());
}
(Some(_), Some(_)) => p += 1,
_ => return Err(Error::Malformed("lossless JPEG: missing restart marker")),
}
}
}
}
struct Component {
id: u8,
table: usize,
}
/// Decode a lossless JPEG stream. `max_samples` caps the picture (width × height × components).
pub fn decode(data: &[u8], max_samples: usize) -> Result<Decoded> {
if data.get(0..2) != Some(&[0xFF, 0xD8]) {
return Err(Error::Malformed("lossless JPEG: no SOI marker"));
}
let mut pos = 2usize;
let mut tables: [Option<Huffman>; 4] = [None, None, None, None];
let mut frame: Option<(u8, usize, usize, Vec<Component>)> = None;
let mut restart_interval = 0usize;
loop {
// markers may be preceded by fill bytes 0xFF
while data.get(pos) == Some(&0xFF) && data.get(pos + 1) == Some(&0xFF) {
pos += 1;
}
let (Some(&0xFF), Some(&m)) = (data.get(pos), data.get(pos + 1)) else {
return Err(Error::Malformed("lossless JPEG: expected a marker"));
};
pos += 2;
if m == 0xD9 {
return Err(Error::Malformed("lossless JPEG: no scan"));
}
if (0xD0..=0xD8).contains(&m) || m == 0x01 {
continue; // markers without a segment
}
let len_b = data.get(pos..pos + 2).ok_or(Error::Truncated)?;
let len = usize::from(u16::from_be_bytes([len_b[0], len_b[1]]));
if len < 2 {
return Err(Error::Malformed("lossless JPEG: segment length"));
}
let seg = data.get(pos + 2..pos + len).ok_or(Error::Truncated)?;
match m {
0xC3 => {
let [p, y0, y1, x0, x1, nf, rest @ ..] = seg else {
return Err(Error::Malformed("lossless JPEG: short frame header"));
};
let (precision, nf) = (*p, usize::from(*nf));
let height = usize::from(u16::from_be_bytes([*y0, *y1]));
let width = usize::from(u16::from_be_bytes([*x0, *x1]));
if !(2..=16).contains(&precision) {
return Err(Error::Unsupported(format!("lossless JPEG precision {precision}")));
}
if height == 0 {
return Err(Error::Unsupported("lossless JPEG with the height in a DNL marker".into()));
}
if width == 0 || !(1..=4).contains(&nf) || rest.len() < nf * 3 {
return Err(Error::Malformed("lossless JPEG: frame header"));
}
let mut comps = Vec::with_capacity(nf);
for c in rest.as_chunks::<3>().0.iter().take(nf) {
if c[1] != 0x11 {
return Err(Error::Unsupported("subsampled lossless JPEG components".into()));
}
comps.push(Component { id: c[0], table: 0 });
}
frame = Some((precision, width, height, comps));
}
0xC4 => {
let mut s = seg;
while !s.is_empty() {
let [tcth, counts @ ..] = s else { break };
let bits: [u8; 16] = counts.get(..16).and_then(|b| b.try_into().ok()).ok_or(Error::Truncated)?;
let n: usize = bits.iter().map(|&b| usize::from(b)).sum();
let values = counts.get(16..16 + n).ok_or(Error::Truncated)?.to_vec();
let th = usize::from(tcth & 0x0F);
if tcth >> 4 != 0 || th > 3 {
return Err(Error::Malformed("lossless JPEG: Huffman table class or id"));
}
if values.iter().any(|&v| v > 16) {
return Err(Error::Malformed("lossless JPEG: difference category above 16"));
}
tables[th] = Some(Huffman::new(&bits, values)?);
s = counts.get(16 + n..).unwrap_or(&[]);
}
}
0xDD => {
let r = seg.get(0..2).ok_or(Error::Truncated)?;
restart_interval = usize::from(u16::from_be_bytes([r[0], r[1]]));
}
0xDA => {
let Some((precision, width, height, mut comps)) = frame.take() else {
return Err(Error::Malformed("lossless JPEG: scan before the frame header"));
};
let ns = usize::from(*seg.first().ok_or(Error::Truncated)?);
if ns != comps.len() {
return Err(Error::Unsupported("lossless JPEG with more than one scan".into()));
}
for k in 0..ns {
let c = seg.get(1 + 2 * k..3 + 2 * k).ok_or(Error::Truncated)?;
let comp = comps.iter_mut().find(|x| x.id == c[0]).ok_or(Error::Malformed("lossless JPEG: unknown scan component"))?;
comp.table = usize::from(c[1] >> 4);
if comp.table > 3 {
return Err(Error::Malformed("lossless JPEG: Huffman table id"));
}
}
let tail = seg.get(1 + 2 * ns..4 + 2 * ns).ok_or(Error::Truncated)?;
let (predictor, pt) = (tail[0], tail[2] & 0x0F);
if !(1..=7).contains(&predictor) {
return Err(Error::Unsupported(format!("lossless JPEG predictor {predictor}")));
}
if pt >= precision {
return Err(Error::Malformed("lossless JPEG: point transform"));
}
let n = width.checked_mul(height).and_then(|v| v.checked_mul(comps.len())).ok_or(Error::TooLarge)?;
if n > max_samples {
return Err(Error::TooLarge);
}
let mut hufs = Vec::with_capacity(comps.len());
for c in &comps {
hufs.push(tables.get(c.table).and_then(|t| t.as_ref()).ok_or(Error::Malformed("lossless JPEG: missing Huffman table"))?);
}
let samples = scan(data, pos + len, &hufs, width, height, precision, predictor, pt, restart_interval)?;
return Ok(Decoded { width, height, components: comps.len(), precision, predictor, samples });
}
0xC0..=0xC2 | 0xC5..=0xC7 | 0xC9..=0xCB | 0xCD..=0xCF => {
return Err(Error::Unsupported("lossy (DCT) JPEG in a DNG".into()));
}
_ => {} // APPn, COM, DQT … are skipped
}
pos += len;
}
}
/// Decode the entropy-coded data of one scan (H.1.2).
fn scan(data: &[u8], start: usize, hufs: &[&Huffman], width: usize, height: usize, precision: u8, predictor: u8, pt: u8, restart: usize) -> Result<Vec<u16>> {
let nc = hufs.len();
let row_len = width * nc;
if restart != 0 && !restart.is_multiple_of(width) {
return Err(Error::Unsupported("lossless JPEG restart interval that splits a line".into()));
}
let lines_per_interval = if restart == 0 { usize::MAX } else { restart / width };
let mut out = vec![0u16; row_len * height];
let mut bits = Bits::new(data, start);
let initial = 1i32 << (precision - pt - 1);
let mut line_in_interval = 0usize;
for y in 0..height {
if line_in_interval == lines_per_interval {
bits.restart()?;
line_in_interval = 0;
}
let first_line = line_in_interval == 0;
let (above, cur) = if y == 0 {
(None, &mut out[..row_len])
} else {
let (a, c) = out.split_at_mut(y * row_len);
(Some(&a[(y - 1) * row_len..]), &mut c[..row_len])
};
for x in 0..width {
for (c, h) in hufs.iter().enumerate() {
let i = x * nc + c;
let ssss = u32::from(bits.decode(h)?);
let diff = match ssss {
0 => 0,
16 => 32768,
s => {
let v = bits.bits(s) as i32;
if v < (1 << (s - 1)) { v - (1 << s) + 1 } else { v }
}
};
let px = match above {
_ if first_line && x == 0 => initial,
_ if first_line => i32::from(cur[i - nc]),
Some(a) if x == 0 => i32::from(a[i]),
Some(a) => predict(predictor, i32::from(cur[i - nc]), i32::from(a[i]), i32::from(a[i - nc])),
None => initial,
};
cur[i] = ((px + diff) & 0xFFFF) as u16;
}
}
line_in_interval += 1;
}
if pt > 0 {
for s in &mut out {
*s <<= pt;
}
}
Ok(out)
}
/// Table H.1 predictors (`ra` left, `rb` above, `rc` above left).
pub fn predict(p: u8, ra: i32, rb: i32, rc: i32) -> i32 {
match p {
1 => ra,
2 => rb,
3 => rc,
4 => ra + rb - rc,
5 => ra + ((rb - rc) >> 1),
6 => rb + ((ra - rc) >> 1),
_ => (ra + rb) >> 1,
}
}

264
crates/dng/src/process.rs Normal file
View File

@ -0,0 +1,264 @@
//! Raw samples to a linear Rec. 709 picture, in the order of the DNG specification's chapter 5
//! ("Mapping Raw Values to Linear Reference Values") and chapter 6 (colour):
//!
//! 1. ActiveArea: only the active (unmasked) sensor area is used; black levels, the CFA pattern
//! and the default crop are relative to its top left corner.
//! 2. LinearizationTable maps each sample.
//! 3. The black level (BlackLevel over its BlackLevelRepeatDim pattern, plus BlackLevelDeltaH for
//! the column and BlackLevelDeltaV for the row) is subtracted and the range up to WhiteLevel
//! scaled to 0–1 (clipped there).
//! 4. White balance: each sample is divided by the camera neutral of its colour and clipped at
//! 1, so highlights the sensor clipped stay neutral.
//! 5. A CFA is demosaiced ([`crate::demosaic`]).
//! 6. DefaultCropOrigin / DefaultCropSize cut the picture.
//! 7. The colour matrix of [`crate::color`] takes the white-balanced camera values to linear
//! Rec. 709 (including BaselineExposure); negative results are clipped to 0.
#[cfg(feature = "threads")]
use rayon::prelude::*;
use crate::color::{M3, V3};
use crate::demosaic::{Pattern, demosaic};
use crate::raw::RawImage;
use crate::{Error, Result};
/// The processing tags of a raw image.
#[derive(Clone, Debug, Default)]
pub struct Params {
/// top, left, bottom, right
pub active_area: Option<[usize; 4]>,
pub linearization: Option<Vec<u16>>,
/// rows, columns of the BlackLevel pattern
pub black_repeat: (usize, usize),
/// rows × columns × samples per pixel values
pub black: Vec<f64>,
pub black_delta_h: Vec<f64>,
pub black_delta_v: Vec<f64>,
/// per sample
pub white: Vec<f64>,
pub crop_origin: Option<(f64, f64)>,
pub crop_size: Option<(f64, f64)>,
/// CFA pattern in camera planes (`None`: LinearRaw).
pub cfa: Option<Pattern>,
}
/// A rectangle: x, y, width, height.
type Rect = (usize, usize, usize, usize);
/// The active area of a `w` × `h` raw image.
pub fn active_rect(p: &Params, w: usize, h: usize) -> Result<Rect> {
match p.active_area {
None => Ok((0, 0, w, h)),
Some([top, left, bottom, right]) => {
if top >= bottom || left >= right || bottom > h || right > w {
return Err(Error::Malformed("ActiveArea outside the image"));
}
Ok((left, top, right - left, bottom - top))
}
}
}
/// The default crop inside an active area of `aw` × `ah`.
pub fn crop_rect(p: &Params, aw: usize, ah: usize) -> Rect {
let (ox, oy) = p.crop_origin.unwrap_or((0.0, 0.0));
let (sw, sh) = p.crop_size.unwrap_or((aw as f64, ah as f64));
let to = |v: f64, max: usize| if v.is_finite() && v > 0.0 { (v.min(max as f64)) as usize } else { 0 };
let x = to(ox.floor(), aw.saturating_sub(1));
let y = to(oy.floor(), ah.saturating_sub(1));
let w = to(sw.round(), aw - x).max(1);
let h = to(sh.round(), ah - y).max(1);
(x, y, w, h)
}
/// Normalised, white-balanced samples of the active area (steps 1–4), one plane per sample.
fn normalise(raw: &RawImage, p: &Params, (ax, ay, aw, ah): Rect, neutral: &V3, planes: usize) -> Result<Vec<f32>> {
let spp = raw.samples_per_pixel;
let (br, bc) = (p.black_repeat.0.max(1), p.black_repeat.1.max(1));
if br > 64 || bc > 64 {
return Err(Error::Unsupported("a BlackLevelRepeatDim larger than 64".into()));
}
let need = br * bc * spp;
let black: Vec<f64> = match p.black.len() {
0 => vec![0.0; need],
1 => vec![p.black[0]; need],
n if n >= need => p.black[..need].to_vec(),
_ => return Err(Error::Malformed("BlackLevel has too few values")),
};
let max_code = ((1u32 << raw.bits) - 1) as f64;
let white: Vec<f64> = (0..spp).map(|s| p.white.get(s).or(p.white.first()).copied().unwrap_or(max_code)).collect();
let lin = p.linearization.as_deref().filter(|t| !t.is_empty());
let inv_neutral: Vec<f32> = (0..planes).map(|c| (1.0 / neutral.get(c).copied().unwrap_or(1.0)) as f32).collect();
let cfa = p.cfa.as_ref();
let mut out = vec![0f32; aw * ah * spp];
let row = |y: usize, o: &mut [f32]| {
let dv = p.black_delta_v.get(y).copied().unwrap_or(0.0);
let src_row = (ay + y) * raw.width * spp;
for x in 0..aw {
let dh = p.black_delta_h.get(x).copied().unwrap_or(0.0);
for s in 0..spp {
let code = raw.data.get(src_row + (ax + x) * spp + s).copied().unwrap_or(0);
let v = match lin {
Some(t) => f64::from(t.get(usize::from(code)).or(t.last()).copied().unwrap_or(0)),
None => f64::from(code),
};
let b = black[((y % br) * bc + x % bc) * spp + s] + dh + dv;
let range = white[s] - b;
let n = if range > 0.0 { ((v - b) / range).clamp(0.0, 1.0) } else { 0.0 };
let plane = match cfa {
Some(c) => usize::from(c.color(x, y)),
None => s,
};
let wb = inv_neutral.get(plane).copied().unwrap_or(1.0);
if let Some(o) = o.get_mut(x * spp + s) {
*o = (n as f32 * wb).min(1.0);
}
}
}
};
#[cfg(feature = "threads")]
out.par_chunks_mut((aw * spp).max(1)).enumerate().for_each(|(y, o)| row(y, o));
#[cfg(not(feature = "threads"))]
out.chunks_mut((aw * spp).max(1)).enumerate().for_each(|(y, o)| row(y, o));
Ok(out)
}
/// A developed picture: premultiplied linear Rec. 709 RGBA floats (alpha 1).
pub struct Developed {
pub width: usize,
pub height: usize,
pub rgba: Vec<f32>,
}
/// Develop `raw` (steps 1–7). `neutral` and `matrix` come from [`crate::color::Calibration::solve`].
pub fn develop(raw: &RawImage, p: &Params, neutral: &V3, matrix: &M3) -> Result<Developed> {
let spp = raw.samples_per_pixel;
if p.cfa.is_some() && spp != 1 {
return Err(Error::Malformed("a CFA image with more than one sample per pixel"));
}
if p.cfa.is_none() && !matches!(spp, 1 | 3) {
return Err(Error::Unsupported(format!("LinearRaw with {spp} samples per pixel")));
}
let active = active_rect(p, raw.width, raw.height)?;
let (_, _, aw, ah) = active;
let planes = if p.cfa.is_some() { 3 } else { spp };
let norm = normalise(raw, p, active, neutral, planes)?;
// three samples per pixel
let rgb = match (&p.cfa, spp) {
(Some(pat), _) => demosaic(&norm, aw, ah, pat),
(None, 3) => norm,
_ => norm.iter().flat_map(|&v| [v, v, v]).collect(),
};
let (cx, cy, cw, ch) = crop_rect(p, aw, ah);
let m: [[f32; 3]; 3] = matrix.map(|r| r.map(|v| v as f32));
let mono = p.cfa.is_none() && spp == 1;
let mut rgba = vec![0f32; cw * ch * 4];
let row = |y: usize, o: &mut [f32]| {
let src = (cy + y) * aw + cx;
for x in 0..cw {
let Some(c) = rgb.get((src + x) * 3..(src + x) * 3 + 3) else { continue };
let px = if mono {
// grey: the matrix's luminance gain only
let g = c[0] * (m[1][0] + m[1][1] + m[1][2]);
[g, g, g]
} else {
[0, 1, 2].map(|i| m[i][0] * c[0] + m[i][1] * c[1] + m[i][2] * c[2])
};
if let Some(d) = o.get_mut(x * 4..x * 4 + 4) {
d.copy_from_slice(&[px[0].max(0.0), px[1].max(0.0), px[2].max(0.0), 1.0]);
}
}
};
#[cfg(feature = "threads")]
rgba.par_chunks_mut(cw * 4).enumerate().for_each(|(y, o)| row(y, o));
#[cfg(not(feature = "threads"))]
rgba.chunks_mut(cw * 4).enumerate().for_each(|(y, o)| row(y, o));
Ok(Developed { width: cw, height: ch, rgba })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::color::IDENTITY;
fn raw(w: usize, h: usize, spp: usize, data: Vec<u16>) -> RawImage {
RawImage { width: w, height: h, samples_per_pixel: spp, bits: 12, data }
}
#[test]
fn black_white_linearisation_and_white_balance() {
// LinearRaw 2 × 1: black 100, white 1100; neutral (0.5, 1, 0.25)
let r = raw(2, 1, 3, vec![600, 600, 600, 100, 1100, 2000]);
let p = Params { black: vec![100.0], white: vec![1100.0], ..Default::default() };
let d = develop(&r, &p, &[0.5, 1.0, 0.25], &IDENTITY).unwrap();
assert_eq!((d.width, d.height), (2, 1));
// 0.5 / 0.5 = 1, 0.5 / 1, 0.5 / 0.25 = 2 → clipped to 1
assert_eq!(&d.rgba[..4], &[1.0, 0.5, 1.0, 1.0]);
// below black → 0, at white → 1, above white → 1
assert_eq!(&d.rgba[4..], &[0.0, 1.0, 1.0, 1.0]);
// a linearisation table doubling the codes
let t: Vec<u16> = (0..4096u32).map(|v| (v * 2).min(65_535) as u16).collect();
let p = Params { linearization: Some(t), black: vec![200.0], white: vec![2200.0], ..p };
let d = develop(&r, &p, &[1.0; 3], &IDENTITY).unwrap();
assert_eq!(&d.rgba[..3], &[0.5, 0.5, 0.5]);
}
#[test]
fn per_cfa_black_levels_and_deltas() {
// 2 × 2 CFA, black repeat 2 × 2 with a different black per site, plus row/column deltas
let pat = Pattern { rows: 2, cols: 2, colors: vec![0, 1, 1, 2] };
let p = Params {
black_repeat: (2, 2),
black: vec![10.0, 20.0, 30.0, 40.0],
black_delta_h: vec![0.0, 5.0],
black_delta_v: vec![0.0, 1.0],
white: vec![1010.0],
cfa: Some(pat),
..Default::default()
};
let r = raw(2, 2, 1, vec![510, 525, 531, 546]);
let n = normalise(&r, &p, (0, 0, 2, 2), &[1.0; 3], 3).unwrap();
// (v - black) / (white - black) for black 10, 25, 31, 46
let want = [500.0 / 1000.0, 500.0 / 985.0, 500.0 / 979.0, 500.0 / 964.0];
for (a, b) in n.iter().zip(want) {
assert!((a - b).abs() < 1e-6, "{a} vs {b}");
}
}
#[test]
fn active_area_and_default_crop() {
// 6 × 4 LinearRaw grey ramp; active area cuts a 1-pixel border, crop takes 2 × 1 at (1, 1)
let data: Vec<u16> = (0..24u16).flat_map(|i| [i * 10; 3]).collect();
let r = raw(6, 4, 3, data);
let p =
Params { active_area: Some([1, 1, 3, 5]), white: vec![4095.0], crop_origin: Some((1.0, 1.0)), crop_size: Some((2.0, 1.0)), ..Default::default() };
let d = develop(&r, &p, &[1.0; 3], &IDENTITY).unwrap();
assert_eq!((d.width, d.height), (2, 1));
// active (1,1)–(5,3); crop origin (1,1) inside it → raw (2, 2) and (3, 2): samples 14, 15
assert!((d.rgba[0] - 140.0 / 4095.0).abs() < 1e-6);
assert!((d.rgba[4] - 150.0 / 4095.0).abs() < 1e-6);
// hostile active areas and crops
let bad = Params { active_area: Some([3, 0, 1, 6]), ..Default::default() };
assert!(develop(&r, &bad, &[1.0; 3], &IDENTITY).is_err());
let bad = Params { active_area: Some([0, 0, 4, 7]), ..Default::default() };
assert!(develop(&r, &bad, &[1.0; 3], &IDENTITY).is_err());
let huge = Params { crop_origin: Some((1e12, -5.0)), crop_size: Some((f64::NAN, 1e9)), white: vec![4095.0], ..Default::default() };
let d = develop(&r, &huge, &[1.0; 3], &IDENTITY).unwrap();
assert!(d.width >= 1 && d.width <= 6 && d.height >= 1 && d.height <= 4);
}
#[test]
fn matrix_and_clipping_of_negatives() {
let r = raw(1, 1, 3, vec![4095, 0, 0]);
let p = Params { white: vec![4095.0], ..Default::default() };
let m = [[1.0, 0.0, 0.0], [-0.5, 1.0, 0.0], [0.0, 0.0, 2.0]];
let d = develop(&r, &p, &[1.0; 3], &m).unwrap();
assert_eq!(&d.rgba, &[1.0, 0.0, 0.0, 1.0]);
}
#[test]
fn short_black_level_is_an_error() {
let r = raw(2, 2, 1, vec![0; 4]);
let p = Params { black_repeat: (2, 2), black: vec![1.0, 2.0], cfa: Some(Pattern { rows: 2, cols: 2, colors: vec![0, 1, 1, 2] }), ..Default::default() };
assert!(develop(&r, &p, &[1.0; 3], &IDENTITY).is_err());
}
}

254
crates/dng/src/raw.rs Normal file
View File

@ -0,0 +1,254 @@
//! The raw image data of a DNG: finding the main raw IFD, and reading its strips or tiles
//! (uncompressed or lossless JPEG) into one array of 16-bit samples.
#[cfg(feature = "threads")]
use rayon::prelude::*;
use crate::tiff::{Ifd, Tiff, tags};
use crate::{Error, Result, ljpeg};
/// Largest raw image decoded, in samples (width × height × samples per pixel).
pub const MAX_SAMPLES: usize = 1 << 27;
/// Photometric interpretations of DNG raw data.
pub const PHOTOMETRIC_CFA: u64 = 32803;
pub const PHOTOMETRIC_LINEAR_RAW: u64 = 34892;
/// The layout of a raw IFD's image data.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RawLayout {
pub width: usize,
pub height: usize,
pub samples_per_pixel: usize,
pub bits: u32,
/// 1 uncompressed, 7 lossless JPEG.
pub compression: u64,
pub photometric: u64,
}
/// Raw samples: `width` × `height` pixels of `samples_per_pixel` interleaved 16-bit samples.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RawImage {
pub width: usize,
pub height: usize,
pub samples_per_pixel: usize,
pub bits: u32,
pub data: Vec<u16>,
}
fn uint_or(t: &Tiff, ifd: &Ifd, tag: u16, default: u64) -> Result<u64> {
match ifd.get(tag) {
Some(e) => t.uint(e),
None => Ok(default),
}
}
/// The main raw image IFD: the largest full-resolution (NewSubFileType 0) IFD whose photometric
/// interpretation is CFA or LinearRaw.
pub fn find_raw_ifd<'i>(t: &Tiff, ifds: &'i [Ifd]) -> Result<&'i Ifd> {
let mut best: Option<(&Ifd, u64)> = None;
for ifd in ifds {
let Ok(kind) = uint_or(t, ifd, tags::NEW_SUBFILE_TYPE, 0) else { continue };
let Ok(photometric) = uint_or(t, ifd, tags::PHOTOMETRIC, 0) else { continue };
if kind != 0 || !matches!(photometric, PHOTOMETRIC_CFA | PHOTOMETRIC_LINEAR_RAW) {
continue;
}
let w = uint_or(t, ifd, tags::IMAGE_WIDTH, 0).unwrap_or(0);
let h = uint_or(t, ifd, tags::IMAGE_LENGTH, 0).unwrap_or(0);
let area = w.saturating_mul(h);
if best.is_none_or(|(_, a)| area > a) {
best = Some((ifd, area));
}
}
best.map(|(i, _)| i).ok_or(Error::Unsupported("no raw image in this DNG".into()))
}
/// Read and check the layout tags of a raw IFD.
pub fn layout(t: &Tiff, ifd: &Ifd) -> Result<RawLayout> {
let width = uint_or(t, ifd, tags::IMAGE_WIDTH, 0)?;
let height = uint_or(t, ifd, tags::IMAGE_LENGTH, 0)?;
let spp = uint_or(t, ifd, tags::SAMPLES_PER_PIXEL, 1)?;
let photometric = uint_or(t, ifd, tags::PHOTOMETRIC, 0)?;
let compression = uint_or(t, ifd, tags::COMPRESSION, 1)?;
let bits = match ifd.get(tags::BITS_PER_SAMPLE) {
Some(e) => {
let b = t.uints(e)?;
let first = *b.first().ok_or(Error::Malformed("empty BitsPerSample"))?;
if b.iter().any(|&x| x != first) {
return Err(Error::Unsupported("different bit depths per sample".into()));
}
first
}
None => 1,
};
if width == 0 || height == 0 || width > 65_535 || height > 65_535 {
return Err(Error::Malformed("raw image size"));
}
if !(1..=4).contains(&spp) {
return Err(Error::Unsupported(format!("{spp} samples per pixel")));
}
if photometric == PHOTOMETRIC_CFA && spp != 1 {
return Err(Error::Malformed("a CFA image with more than one sample per pixel"));
}
if !(1..=16).contains(&bits) {
return Err(Error::Unsupported(format!("{bits}-bit raw samples")));
}
if uint_or(t, ifd, tags::SAMPLE_FORMAT, 1)? != 1 {
return Err(Error::Unsupported("floating-point raw data".into()));
}
if spp > 1 && uint_or(t, ifd, tags::PLANAR_CONFIGURATION, 1)? != 1 {
return Err(Error::Unsupported("planar raw data".into()));
}
match compression {
1 => {
if uint_or(t, ifd, tags::PREDICTOR, 1)? != 1 {
return Err(Error::Unsupported("a TIFF predictor on uncompressed raw data".into()));
}
}
7 => {}
8 => return Err(Error::Unsupported("deflate-compressed DNG".into())),
34892 => return Err(Error::Unsupported("lossy JPEG DNG".into())),
52546 => return Err(Error::Unsupported("JPEG XL DNG".into())),
c => return Err(Error::Unsupported(format!("DNG compression {c}"))),
}
Ok(RawLayout { width: width as usize, height: height as usize, samples_per_pixel: spp as usize, bits: bits as u32, compression, photometric })
}
/// One strip or tile: where it sits in the image and its bytes.
struct Chunk<'a> {
x: usize,
y: usize,
/// Pixel size of the chunk as coded (tiles are whole even at the image edge).
w: usize,
h: usize,
bytes: &'a [u8],
}
fn chunks<'a>(t: &Tiff<'a>, ifd: &Ifd, l: &RawLayout) -> Result<Vec<Chunk<'a>>> {
let (offs, counts, tw, th) = if let (Some(o), Some(c)) = (ifd.get(tags::TILE_OFFSETS), ifd.get(tags::TILE_BYTE_COUNTS)) {
let tw = uint_or(t, ifd, tags::TILE_WIDTH, 0)? as usize;
let th = uint_or(t, ifd, tags::TILE_LENGTH, 0)? as usize;
if tw == 0 || th == 0 || tw > 65_535 || th > 65_535 {
return Err(Error::Malformed("tile size"));
}
(t.uints(o)?, t.uints(c)?, tw, th)
} else if let (Some(o), Some(c)) = (ifd.get(tags::STRIP_OFFSETS), ifd.get(tags::STRIP_BYTE_COUNTS)) {
let rows = (uint_or(t, ifd, tags::ROWS_PER_STRIP, l.height as u64)? as usize).clamp(1, l.height);
(t.uints(o)?, t.uints(c)?, l.width, rows)
} else {
return Err(Error::Malformed("no strips or tiles"));
};
let across = l.width.div_ceil(tw);
let down = l.height.div_ceil(th);
let n = across * down;
if offs.len() < n || counts.len() < n {
return Err(Error::Malformed("fewer strips or tiles than the image needs"));
}
let mut out = Vec::with_capacity(n);
for i in 0..n {
let (x, y) = ((i % across) * tw, (i / across) * th);
let start = usize::try_from(offs[i]).map_err(|_| Error::Truncated)?;
let len = usize::try_from(counts[i]).map_err(|_| Error::Truncated)?;
let end = start.checked_add(len).ok_or(Error::Truncated)?;
let bytes = t.data.get(start..end).ok_or(Error::Truncated)?;
// strips end at the image; tiles are coded whole
let h = if ifd.get(tags::TILE_OFFSETS).is_some() { th } else { th.min(l.height - y) };
out.push(Chunk { x, y, w: tw, h, bytes });
}
Ok(out)
}
/// Unpack `w` × `h` × `spp` uncompressed samples of `bits` bits: 8 and 16-bit samples as bytes /
/// words in the file's byte order, other depths packed most significant bit first with each row
/// starting on a byte boundary (TIFF 6.0 §7, DNG BitsPerSample).
pub fn unpack(bytes: &[u8], w: usize, h: usize, spp: usize, bits: u32, big_endian: bool) -> Result<Vec<u16>> {
let per_row = w * spp;
let row_bytes = (per_row * bits as usize).div_ceil(8);
let need = row_bytes.checked_mul(h).ok_or(Error::TooLarge)?;
if bytes.len() < need {
return Err(Error::Truncated);
}
let mut out = Vec::with_capacity(per_row * h);
for row in bytes.chunks_exact(row_bytes.max(1)).take(h) {
match bits {
8 => out.extend(row.iter().map(|&b| u16::from(b))),
16 => out.extend(row.as_chunks::<2>().0.iter().map(|&b| if big_endian { u16::from_be_bytes(b) } else { u16::from_le_bytes(b) })),
_ => {
let mut acc: u32 = 0;
let mut n: u32 = 0;
let mut it = row.iter();
for _ in 0..per_row {
while n < bits {
acc = (acc << 8) | u32::from(*it.next().ok_or(Error::Truncated)?);
n += 8;
}
n -= bits;
out.push(((acc >> n) & ((1 << bits) - 1)) as u16);
acc &= (1 << n) - 1;
}
}
}
}
Ok(out)
}
/// Decode one chunk into `w` × `h` × `spp` samples.
fn decode_chunk(c: &Chunk, l: &RawLayout, big_endian: bool) -> Result<Vec<u16>> {
let need = c.w * c.h * l.samples_per_pixel;
match l.compression {
1 => unpack(c.bytes, c.w, c.h, l.samples_per_pixel, l.bits, big_endian),
_ => {
// the JPEG's samples fill the chunk in raster order (its own width × components may
// differ from the chunk's, e.g. two components of half the width)
let d = ljpeg::decode(c.bytes, need.saturating_add(65_536 * 4))?;
if d.samples.len() < need {
return Err(Error::Malformed("lossless JPEG smaller than its tile or strip"));
}
let mut s = d.samples;
s.truncate(need);
Ok(s)
}
}
}
/// Read the raw image of `ifd`.
pub fn read(t: &Tiff, ifd: &Ifd) -> Result<RawImage> {
let l = layout(t, ifd)?;
let spp = l.samples_per_pixel;
let total = l.width.checked_mul(l.height).and_then(|v| v.checked_mul(spp)).ok_or(Error::TooLarge)?;
if total > MAX_SAMPLES {
return Err(Error::TooLarge);
}
let chunks = chunks(t, ifd, &l)?;
// every sample takes at least one bit of file (a Huffman code is at least one bit long)
let coded: usize = chunks.iter().map(|c| c.bytes.len()).fold(0usize, usize::saturating_add);
if total / 8 > coded {
return Err(Error::Truncated);
}
for c in &chunks {
let n = c.w.checked_mul(c.h).and_then(|v| v.checked_mul(spp)).ok_or(Error::TooLarge)?;
if n > MAX_SAMPLES {
return Err(Error::TooLarge);
}
if n / 8 > c.bytes.len() {
return Err(Error::Truncated);
}
}
#[cfg(feature = "threads")]
let decoded: Vec<Result<Vec<u16>>> = chunks.par_iter().map(|c| decode_chunk(c, &l, t.big_endian)).collect();
#[cfg(not(feature = "threads"))]
let decoded: Vec<Result<Vec<u16>>> = chunks.iter().map(|c| decode_chunk(c, &l, t.big_endian)).collect();
let mut data = vec![0u16; total];
let row_len = l.width * spp;
for (c, d) in chunks.iter().zip(decoded) {
let d = d?;
let cw = c.w.min(l.width - c.x);
let ch = c.h.min(l.height - c.y);
for r in 0..ch {
let src = d.get(r * c.w * spp..(r * c.w + cw) * spp).ok_or(Error::Truncated)?;
let at = (c.y + r) * row_len + c.x * spp;
data.get_mut(at..at + cw * spp).ok_or(Error::Truncated)?.copy_from_slice(src);
}
}
Ok(RawImage { width: l.width, height: l.height, samples_per_pixel: spp, bits: l.bits, data })
}

334
crates/dng/src/tiff.rs Normal file
View File

@ -0,0 +1,334 @@
//! The TIFF container (TIFF 6.0 §2, as DNG uses it): byte order, image file directories (IFDs),
//! their entries and values, and the SubIFD tree.
//!
//! Every offset and count comes from the file and is checked before use: an entry whose value lies
//! outside the file is an error when it is read, the walk visits at most [`MAX_IFDS`] directories,
//! never the same one twice (cyclic chains end), and goes at most [`MAX_DEPTH`] SubIFD levels deep.
use crate::{Error, Result};
/// Most IFDs one file may have (main chain and SubIFDs together).
pub const MAX_IFDS: usize = 64;
/// Deepest SubIFD nesting followed.
pub const MAX_DEPTH: usize = 4;
/// TIFF field types (TIFF 6.0 §2, plus type 13 IFD from TIFF Technical Note 1).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Type {
Byte,
Ascii,
Short,
Long,
Rational,
SByte,
Undefined,
SShort,
SLong,
SRational,
Float,
Double,
Ifd,
}
impl Type {
fn from_code(c: u16) -> Option<Type> {
Some(match c {
1 => Type::Byte,
2 => Type::Ascii,
3 => Type::Short,
4 => Type::Long,
5 => Type::Rational,
6 => Type::SByte,
7 => Type::Undefined,
8 => Type::SShort,
9 => Type::SLong,
10 => Type::SRational,
11 => Type::Float,
12 => Type::Double,
13 => Type::Ifd,
_ => return None,
})
}
/// Bytes per value.
pub fn size(self) -> usize {
match self {
Type::Byte | Type::Ascii | Type::SByte | Type::Undefined => 1,
Type::Short | Type::SShort => 2,
Type::Long | Type::SLong | Type::Float | Type::Ifd => 4,
Type::Rational | Type::SRational | Type::Double => 8,
}
}
}
/// One directory entry.
#[derive(Clone, Copy, Debug)]
pub struct Entry {
pub tag: u16,
pub ty: Type,
pub count: u32,
/// Where the value bytes start: inside the entry when they fit in four bytes, else the offset
/// the entry holds (not yet checked against the file length).
value_at: u64,
}
/// One image file directory.
#[derive(Clone, Debug, Default)]
pub struct Ifd {
/// File offset of the directory.
pub offset: u64,
pub entries: Vec<Entry>,
/// Nesting: 0 for the main chain, 1 for its SubIFDs, …
pub depth: usize,
}
impl Ifd {
pub fn get(&self, tag: u16) -> Option<&Entry> {
self.entries.iter().find(|e| e.tag == tag)
}
}
/// A TIFF file in memory.
#[derive(Clone, Copy)]
pub struct Tiff<'a> {
pub data: &'a [u8],
pub big_endian: bool,
}
impl<'a> Tiff<'a> {
/// Read the header. Classic TIFF only (BigTIFF, version 43, is not used by DNG).
pub fn new(data: &'a [u8]) -> Result<(Tiff<'a>, u64)> {
let big_endian = match data.get(0..4) {
Some(b"II*\0") => false,
Some(b"MM\0*") => true,
_ => return Err(Error::NotDng),
};
let t = Tiff { data, big_endian };
let first = u64::from(t.u32_at(4)?);
Ok((t, first))
}
fn bytes_at(&self, off: u64, len: usize) -> Result<&'a [u8]> {
let start = usize::try_from(off).map_err(|_| Error::Malformed("offset outside the file"))?;
let end = start.checked_add(len).ok_or(Error::Malformed("offset outside the file"))?;
self.data.get(start..end).ok_or(Error::Malformed("offset outside the file"))
}
pub fn u16_of(&self, b: [u8; 2]) -> u16 {
if self.big_endian { u16::from_be_bytes(b) } else { u16::from_le_bytes(b) }
}
pub fn u32_of(&self, b: [u8; 4]) -> u32 {
if self.big_endian { u32::from_be_bytes(b) } else { u32::from_le_bytes(b) }
}
fn u16_at(&self, off: u64) -> Result<u16> {
let b = self.bytes_at(off, 2)?;
Ok(self.u16_of([b[0], b[1]]))
}
fn u32_at(&self, off: u64) -> Result<u32> {
let b = self.bytes_at(off, 4)?;
Ok(self.u32_of([b[0], b[1], b[2], b[3]]))
}
/// Read the IFD at `off`: its entries and the offset of the next IFD (0 at the end of a chain).
pub fn read_ifd(&self, off: u64, depth: usize) -> Result<(Ifd, u64)> {
let n = self.u16_at(off)?;
let mut entries = Vec::with_capacity(usize::from(n));
for i in 0..u64::from(n) {
let at = off + 2 + i * 12;
let e = self.bytes_at(at, 12)?;
let tag = self.u16_of([e[0], e[1]]);
let code = self.u16_of([e[2], e[3]]);
let count = self.u32_of([e[4], e[5], e[6], e[7]]);
// unknown field types are skipped (TIFF 6.0: readers ignore them)
let Some(ty) = Type::from_code(code) else { continue };
let len = u64::from(count).saturating_mul(ty.size() as u64);
let value_at = if len <= 4 { at + 8 } else { u64::from(self.u32_of([e[8], e[9], e[10], e[11]])) };
entries.push(Entry { tag, ty, count, value_at });
}
let next = u64::from(self.u32_at(off + 2 + u64::from(n) * 12)?);
Ok((Ifd { offset: off, entries, depth }, next))
}
/// Every IFD of the file: the main chain from `first` and, depth first, the SubIFDs (tag 330) of
/// each. Bounded by [`MAX_IFDS`] and [`MAX_DEPTH`]; a directory reached twice ends that branch.
pub fn all_ifds(&self, first: u64) -> Result<Vec<Ifd>> {
let mut out: Vec<Ifd> = Vec::new();
let mut seen: Vec<u64> = Vec::new();
let mut stack: Vec<(u64, usize)> = vec![(first, 0)];
while let Some((off, depth)) = stack.pop() {
let mut next = off;
while next != 0 {
if seen.contains(&next) {
break;
}
if out.len() >= MAX_IFDS {
return Err(Error::Malformed("too many IFDs"));
}
seen.push(next);
let (ifd, after) = match self.read_ifd(next, depth) {
Ok(x) => x,
// a broken directory ends the chain; the first one must be readable
Err(e) if out.is_empty() => return Err(e),
Err(_) => break,
};
if depth < MAX_DEPTH
&& let Some(sub) = ifd.get(tags::SUB_IFDS)
&& let Ok(offs) = self.uints(sub)
{
for o in offs.into_iter().rev() {
stack.push((o, depth + 1));
}
}
out.push(ifd);
next = after;
}
}
Ok(out)
}
/// The value bytes of an entry.
pub fn value_bytes(&self, e: &Entry) -> Result<&'a [u8]> {
let len = (e.count as usize).checked_mul(e.ty.size()).ok_or(Error::Malformed("entry too large"))?;
self.bytes_at(e.value_at, len)
}
/// The `i`-th value of a numeric entry as f64 (rationals divided; a zero denominator is an
/// error).
fn number(&self, e: &Entry, b: &[u8], i: usize) -> Result<f64> {
let s = e.ty.size();
let v = b.get(i * s..i * s + s).ok_or(Error::Malformed("entry too short"))?;
let u16v = |o: usize| self.u16_of([v[o], v[o + 1]]);
let u32v = |o: usize| self.u32_of([v[o], v[o + 1], v[o + 2], v[o + 3]]);
Ok(match e.ty {
Type::Byte | Type::Undefined => f64::from(v[0]),
Type::SByte => f64::from(v[0] as i8),
Type::Short => f64::from(u16v(0)),
Type::SShort => f64::from(u16v(0) as i16),
Type::Long | Type::Ifd => f64::from(u32v(0)),
Type::SLong => f64::from(u32v(0) as i32),
Type::Rational => {
let d = u32v(4);
if d == 0 {
return Err(Error::Malformed("rational with a zero denominator"));
}
f64::from(u32v(0)) / f64::from(d)
}
Type::SRational => {
let d = u32v(4) as i32;
if d == 0 {
return Err(Error::Malformed("rational with a zero denominator"));
}
f64::from(u32v(0) as i32) / f64::from(d)
}
Type::Float => f64::from(f32::from_bits(u32v(0))),
Type::Double => {
let mut a = [0u8; 8];
a.copy_from_slice(&v[..8]);
if self.big_endian { f64::from_be_bytes(a) } else { f64::from_le_bytes(a) }
}
Type::Ascii => return Err(Error::Malformed("text where a number belongs")),
})
}
/// All values of a numeric entry as f64. Non-finite values are an error.
pub fn floats(&self, e: &Entry) -> Result<Vec<f64>> {
let b = self.value_bytes(e)?;
let mut out = Vec::with_capacity(e.count as usize);
for i in 0..e.count as usize {
let v = self.number(e, b, i)?;
if !v.is_finite() {
return Err(Error::Malformed("non-finite number"));
}
out.push(v);
}
Ok(out)
}
/// All values of an unsigned integer entry (BYTE, SHORT, LONG, IFD).
pub fn uints(&self, e: &Entry) -> Result<Vec<u64>> {
if !matches!(e.ty, Type::Byte | Type::Undefined | Type::Short | Type::Long | Type::Ifd) {
return Err(Error::Malformed("expected an unsigned integer field"));
}
let b = self.value_bytes(e)?;
let s = e.ty.size();
Ok(b.chunks_exact(s)
.map(|v| match s {
1 => u64::from(v[0]),
2 => u64::from(self.u16_of([v[0], v[1]])),
_ => u64::from(self.u32_of([v[0], v[1], v[2], v[3]])),
})
.collect())
}
/// The first value of an unsigned integer entry.
pub fn uint(&self, e: &Entry) -> Result<u64> {
self.uints(e)?.first().copied().ok_or(Error::Malformed("empty field"))
}
/// An ASCII entry up to its first NUL, lossily decoded.
pub fn ascii(&self, e: &Entry) -> Result<String> {
let b = self.value_bytes(e)?;
let end = b.iter().position(|&c| c == 0).unwrap_or(b.len());
Ok(String::from_utf8_lossy(&b[..end]).trim().to_string())
}
}
/// Tag numbers (TIFF 6.0, TIFF/EP, DNG 1.7, CinemaDNG).
pub mod tags {
pub const NEW_SUBFILE_TYPE: u16 = 254;
pub const IMAGE_WIDTH: u16 = 256;
pub const IMAGE_LENGTH: u16 = 257;
pub const BITS_PER_SAMPLE: u16 = 258;
pub const COMPRESSION: u16 = 259;
pub const PHOTOMETRIC: u16 = 262;
pub const MAKE: u16 = 271;
pub const MODEL: u16 = 272;
pub const STRIP_OFFSETS: u16 = 273;
pub const SAMPLES_PER_PIXEL: u16 = 277;
pub const ROWS_PER_STRIP: u16 = 278;
pub const STRIP_BYTE_COUNTS: u16 = 279;
pub const PLANAR_CONFIGURATION: u16 = 284;
pub const PREDICTOR: u16 = 317;
pub const TILE_WIDTH: u16 = 322;
pub const TILE_LENGTH: u16 = 323;
pub const TILE_OFFSETS: u16 = 324;
pub const TILE_BYTE_COUNTS: u16 = 325;
pub const SUB_IFDS: u16 = 330;
pub const SAMPLE_FORMAT: u16 = 339;
pub const CFA_REPEAT_PATTERN_DIM: u16 = 33421;
pub const CFA_PATTERN: u16 = 33422;
pub const DNG_VERSION: u16 = 50706;
pub const UNIQUE_CAMERA_MODEL: u16 = 50708;
pub const CFA_PLANE_COLOR: u16 = 50710;
pub const CFA_LAYOUT: u16 = 50711;
pub const LINEARIZATION_TABLE: u16 = 50712;
pub const BLACK_LEVEL_REPEAT_DIM: u16 = 50713;
pub const BLACK_LEVEL: u16 = 50714;
pub const BLACK_LEVEL_DELTA_H: u16 = 50715;
pub const BLACK_LEVEL_DELTA_V: u16 = 50716;
pub const WHITE_LEVEL: u16 = 50717;
pub const DEFAULT_CROP_ORIGIN: u16 = 50719;
pub const DEFAULT_CROP_SIZE: u16 = 50720;
pub const COLOR_MATRIX_1: u16 = 50721;
pub const COLOR_MATRIX_2: u16 = 50722;
pub const CAMERA_CALIBRATION_1: u16 = 50723;
pub const CAMERA_CALIBRATION_2: u16 = 50724;
pub const ANALOG_BALANCE: u16 = 50727;
pub const AS_SHOT_NEUTRAL: u16 = 50728;
pub const AS_SHOT_WHITE_XY: u16 = 50729;
pub const BASELINE_EXPOSURE: u16 = 50730;
pub const CALIBRATION_ILLUMINANT_1: u16 = 50778;
pub const CALIBRATION_ILLUMINANT_2: u16 = 50779;
pub const ACTIVE_AREA: u16 = 50829;
pub const FORWARD_MATRIX_1: u16 = 50964;
pub const FORWARD_MATRIX_2: u16 = 50965;
/// CinemaDNG: SMPTE ST 12-1 time codes, 8 bytes each.
pub const TIME_CODES: u16 = 51043;
/// CinemaDNG: frames per second (SRATIONAL).
pub const FRAME_RATE: u16 = 51044;
/// CinemaDNG: the camera's reel name.
pub const REEL_NAME: u16 = 51081;
}

449
crates/dng/src/writer.rs Normal file
View File

@ -0,0 +1,449 @@
//! A small DNG writer for synthesising test files (feature `writer`; not used by the app): a
//! TIFF container with the raw image in IFD 0 or in a SubIFD behind an 8-bit preview, strips or
//! tiles, uncompressed samples of any depth or lossless JPEG (T.81 process 14) with any predictor,
//! and whatever extra tags the caller adds.
use crate::ljpeg::predict;
use crate::tiff::tags;
/// A tag value.
#[derive(Clone, Debug)]
pub enum Value {
Byte(Vec<u8>),
Ascii(String),
Short(Vec<u16>),
Long(Vec<u32>),
Rational(Vec<(u32, u32)>),
SRational(Vec<(i32, i32)>),
Undefined(Vec<u8>),
}
impl Value {
fn code(&self) -> u16 {
match self {
Value::Byte(_) => 1,
Value::Ascii(_) => 2,
Value::Short(_) => 3,
Value::Long(_) => 4,
Value::Rational(_) => 5,
Value::Undefined(_) => 7,
Value::SRational(_) => 10,
}
}
fn count(&self) -> usize {
match self {
Value::Byte(v) | Value::Undefined(v) => v.len(),
Value::Ascii(s) => s.len() + 1,
Value::Short(v) => v.len(),
Value::Long(v) => v.len(),
Value::Rational(v) => v.len(),
Value::SRational(v) => v.len(),
}
}
fn bytes(&self, be: bool) -> Vec<u8> {
let w16 = |v: u16| if be { v.to_be_bytes() } else { v.to_le_bytes() };
let w32 = |v: u32| if be { v.to_be_bytes() } else { v.to_le_bytes() };
match self {
Value::Byte(v) | Value::Undefined(v) => v.clone(),
Value::Ascii(s) => s.bytes().chain([0]).collect(),
Value::Short(v) => v.iter().flat_map(|&x| w16(x)).collect(),
Value::Long(v) => v.iter().flat_map(|&x| w32(x)).collect(),
Value::Rational(v) => v.iter().flat_map(|&(n, d)| w32(n).into_iter().chain(w32(d))).collect(),
Value::SRational(v) => v.iter().flat_map(|&(n, d)| w32(n as u32).into_iter().chain(w32(d as u32))).collect(),
}
}
/// Signed rationals with denominator 10000 from floats (colour matrices and the like).
pub fn srationals(v: &[f64]) -> Value {
Value::SRational(v.iter().map(|&x| ((x * 10_000.0).round() as i32, 10_000)).collect())
}
/// Unsigned rationals with denominator 10000.
pub fn rationals(v: &[f64]) -> Value {
Value::Rational(v.iter().map(|&x| ((x * 10_000.0).round().max(0.0) as u32, 10_000)).collect())
}
}
/// How the raw data is stored.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Compression {
None,
/// Lossless JPEG: `predictor` 1–7; each tile / strip is coded as a JPEG of `components`
/// interleaved components (its width × samples per pixel divided by `components`), with a
/// restart marker every `restart_lines` lines (0: none) and point transform `pt`.
LosslessJpeg {
predictor: u8,
components: usize,
restart_lines: usize,
pt: u8,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Layout {
Strips { rows: usize },
Tiles { width: usize, height: usize },
}
/// What to write.
#[derive(Clone, Debug)]
pub struct Options {
pub width: usize,
pub height: usize,
pub samples_per_pixel: usize,
pub bits: u32,
pub big_endian: bool,
pub compression: Compression,
pub layout: Layout,
/// CFA repeat (rows, columns) and pattern colour codes; `None` writes LinearRaw.
pub cfa: Option<(usize, usize, Vec<u8>)>,
/// Put an 8-bit preview in IFD 0 and the raw image in a SubIFD (as cameras do).
pub preview: bool,
/// Extra tags of IFD 0 (colour, CinemaDNG…).
pub main_tags: Vec<(u16, Value)>,
/// Extra tags of the raw IFD (black and white levels, crop…).
pub raw_tags: Vec<(u16, Value)>,
}
impl Default for Options {
fn default() -> Self {
Options {
width: 8,
height: 8,
samples_per_pixel: 1,
bits: 16,
big_endian: false,
compression: Compression::None,
layout: Layout::Strips { rows: 8 },
cfa: Some((2, 2, vec![0, 1, 1, 2])),
preview: false,
main_tags: Vec::new(),
raw_tags: Vec::new(),
}
}
}
/// Pack `rows` rows of `per_row` samples of `bits` bits (8/16-bit in byte order `be`, other
/// depths most significant bit first, each row starting on a byte boundary).
pub fn pack(samples: &[u16], per_row: usize, bits: u32, be: bool) -> Vec<u8> {
let mut out = Vec::new();
for row in samples.chunks(per_row.max(1)) {
match bits {
8 => out.extend(row.iter().map(|&v| v as u8)),
16 => out.extend(row.iter().flat_map(|&v| if be { v.to_be_bytes() } else { v.to_le_bytes() })),
_ => {
let (mut acc, mut n) = (0u32, 0u32);
for &v in row {
acc = (acc << bits) | (u32::from(v) & ((1 << bits) - 1));
n += bits;
while n >= 8 {
n -= 8;
out.push((acc >> n) as u8);
}
acc &= (1 << n) - 1;
}
if n > 0 {
out.push((acc << (8 - n)) as u8);
}
}
}
}
out
}
/// Entropy-coded bit output with 0xFF stuffing.
struct BitOut {
out: Vec<u8>,
acc: u32,
n: u32,
}
impl BitOut {
fn put(&mut self, v: u32, len: u32) {
for i in (0..len).rev() {
self.acc = (self.acc << 1) | ((v >> i) & 1);
self.n += 1;
if self.n == 8 {
let b = self.acc as u8;
self.out.push(b);
if b == 0xFF {
self.out.push(0);
}
self.acc = 0;
self.n = 0;
}
}
}
/// Pad the last byte with 1 bits.
fn flush(&mut self) {
while self.n != 0 {
self.put(1, 1);
}
}
}
/// Encode `samples` (`width` × `height` × `comps`, interleaved) as lossless JPEG.
pub fn lj92(samples: &[u16], width: usize, height: usize, comps: usize, precision: u8, predictor: u8, restart_lines: usize, pt: u8) -> Vec<u8> {
let comps = comps.clamp(1, 4);
let row = width * comps;
let initial = 1i32 << (precision - pt - 1);
// differences, line by line
let mut cats: Vec<Vec<u32>> = vec![vec![0; 17]; comps];
let mut coded: Vec<(usize, u32, u32, i32)> = Vec::with_capacity(samples.len()); // (comp, ssss, extra, line)
let s = |i: usize| i32::from(samples.get(i).copied().unwrap_or(0) >> pt);
for y in 0..height {
let first_line = restart_lines == 0 && y == 0 || restart_lines != 0 && y % restart_lines == 0;
for x in 0..width {
for c in 0..comps {
let i = y * row + x * comps + c;
let px = if first_line && x == 0 {
initial
} else if first_line {
s(i - comps)
} else if x == 0 {
s(i - row)
} else {
predict(predictor, s(i - comps), s(i - row), s(i - row - comps))
};
let d16 = (s(i) - px).rem_euclid(65_536);
let (ssss, extra) = if d16 == 32_768 {
(16, 0)
} else {
let d = if d16 >= 32_768 { d16 - 65_536 } else { d16 };
let ssss = 32 - d.unsigned_abs().leading_zeros();
let extra = if d >= 0 { d as u32 } else { (d + (1 << ssss) - 1) as u32 };
(ssss, extra)
};
cats[c][ssss as usize] += 1;
coded.push((c, ssss, extra, y as i32));
}
}
}
// per component: the used categories by frequency get 2-bit then 6-bit codes
let mut tables = Vec::new();
for counts in &cats {
let mut syms: Vec<u8> = (0..17u8).filter(|&k| counts[usize::from(k)] > 0).collect();
syms.sort_by_key(|&k| std::cmp::Reverse(counts[usize::from(k)]));
if syms.is_empty() {
syms.push(0);
}
let mut code_of = [(0u32, 0u32); 17];
for (i, &k) in syms.iter().enumerate() {
code_of[usize::from(k)] = if i < 3 { (i as u32, 2) } else { (0b110000 + (i as u32 - 3), 6) };
}
let mut bits = [0u8; 16];
bits[1] = syms.len().min(3) as u8;
bits[5] = syms.len().saturating_sub(3) as u8;
tables.push((bits, syms, code_of));
}
let mut out = vec![0xFF, 0xD8];
// SOF3
let mut sof = vec![precision, (height >> 8) as u8, height as u8, (width >> 8) as u8, width as u8, comps as u8];
for c in 0..comps {
sof.extend([c as u8, 0x11, 0]);
}
segment(&mut out, 0xC3, &sof);
let mut dht = Vec::new();
for (c, (bits, syms, _)) in tables.iter().enumerate() {
dht.push(c as u8);
dht.extend(bits);
dht.extend(syms);
}
segment(&mut out, 0xC4, &dht);
if restart_lines != 0 {
let ri = (restart_lines * width) as u16;
segment(&mut out, 0xDD, &ri.to_be_bytes());
}
let mut sos = vec![comps as u8];
for c in 0..comps {
sos.extend([c as u8, (c as u8) << 4]);
}
sos.extend([predictor, 0, pt]);
segment(&mut out, 0xDA, &sos);
let mut bo = BitOut { out, acc: 0, n: 0 };
let mut line = 0;
let mut rst = 0u8;
for (c, ssss, extra, y) in coded {
if restart_lines != 0 && y != line && (y as usize).is_multiple_of(restart_lines) {
bo.flush();
bo.out.extend([0xFF, 0xD0 + rst]);
rst = (rst + 1) % 8;
}
line = y;
let (code, len) = tables[c].2[ssss as usize];
bo.put(code, len);
if ssss != 16 {
bo.put(extra, ssss);
}
}
bo.flush();
let mut out = bo.out;
out.extend([0xFF, 0xD9]);
out
}
fn segment(out: &mut Vec<u8>, marker: u8, body: &[u8]) {
out.extend([0xFF, marker]);
out.extend(((body.len() + 2) as u16).to_be_bytes());
out.extend(body);
}
/// Append an IFD at the end of `buf` (word aligned) and return its offset.
fn put_ifd(buf: &mut Vec<u8>, mut entries: Vec<(u16, Value)>, be: bool) -> u32 {
if buf.len() % 2 == 1 {
buf.push(0);
}
entries.sort_by_key(|e| e.0);
let at = buf.len();
let n = entries.len();
let w16 = |v: u16| if be { v.to_be_bytes() } else { v.to_le_bytes() };
let w32 = |v: u32| if be { v.to_be_bytes() } else { v.to_le_bytes() };
let mut ext_at = at + 2 + 12 * n + 4;
let mut head = Vec::new();
let mut ext = Vec::new();
head.extend(w16(n as u16));
for (tag, v) in &entries {
let b = v.bytes(be);
head.extend(w16(*tag));
head.extend(w16(v.code()));
head.extend(w32(v.count() as u32));
if b.len() <= 4 {
let mut inline = b.clone();
inline.resize(4, 0);
head.extend(inline);
} else {
head.extend(w32(ext_at as u32));
ext.extend(&b);
if b.len() % 2 == 1 {
ext.push(0);
}
ext_at += b.len().div_ceil(2) * 2;
}
}
head.extend(w32(0));
buf.extend(head);
buf.extend(ext);
at as u32
}
/// Encode the strips or tiles of `samples`, appending them to `buf`; returns (offsets, counts).
fn put_chunks(buf: &mut Vec<u8>, samples: &[u16], o: &Options) -> (Vec<u32>, Vec<u32>) {
let spp = o.samples_per_pixel;
let (cw, ch) = match o.layout {
Layout::Strips { rows } => (o.width, rows.max(1)),
Layout::Tiles { width, height } => (width.max(1), height.max(1)),
};
let tiles = matches!(o.layout, Layout::Tiles { .. });
let (mut offs, mut counts) = (Vec::new(), Vec::new());
for ty in (0..o.height).step_by(ch) {
for tx in (0..o.width).step_by(cw) {
let rows = if tiles { ch } else { ch.min(o.height - ty) };
// the chunk's samples; tiles past the image edge repeat the edge pixel
let mut chunk = Vec::with_capacity(cw * rows * spp);
for r in 0..rows {
let y = (ty + r).min(o.height - 1);
for c in 0..cw {
let x = (tx + c).min(o.width - 1);
for s in 0..spp {
chunk.push(samples.get((y * o.width + x) * spp + s).copied().unwrap_or(0));
}
}
}
let data = match o.compression {
Compression::None => pack(&chunk, cw * spp, o.bits, o.big_endian),
Compression::LosslessJpeg { predictor, components, restart_lines, pt } => {
let comps = components.max(1);
lj92(&chunk, cw * spp / comps, rows, comps, o.bits as u8, predictor, restart_lines, pt)
}
};
if buf.len() % 2 == 1 {
buf.push(0);
}
offs.push(buf.len() as u32);
counts.push(data.len() as u32);
buf.extend(data);
}
}
(offs, counts)
}
/// Write a DNG holding `samples` (`width` × `height` × `samples_per_pixel`, interleaved).
pub fn write(samples: &[u16], o: &Options) -> Vec<u8> {
let be = o.big_endian;
let mut buf = if be { b"MM\0*\0\0\0\0".to_vec() } else { b"II*\0\0\0\0\0".to_vec() };
let (offs, counts) = put_chunks(&mut buf, samples, o);
let mut raw = vec![
(tags::NEW_SUBFILE_TYPE, Value::Long(vec![0])),
(tags::IMAGE_WIDTH, Value::Long(vec![o.width as u32])),
(tags::IMAGE_LENGTH, Value::Long(vec![o.height as u32])),
(tags::BITS_PER_SAMPLE, Value::Short(vec![o.bits as u16; o.samples_per_pixel])),
(tags::COMPRESSION, Value::Short(vec![if o.compression == Compression::None { 1 } else { 7 }])),
(tags::SAMPLES_PER_PIXEL, Value::Short(vec![o.samples_per_pixel as u16])),
(tags::PLANAR_CONFIGURATION, Value::Short(vec![1])),
];
match o.layout {
Layout::Strips { rows } => {
raw.push((tags::ROWS_PER_STRIP, Value::Long(vec![rows as u32])));
raw.push((tags::STRIP_OFFSETS, Value::Long(offs)));
raw.push((tags::STRIP_BYTE_COUNTS, Value::Long(counts)));
}
Layout::Tiles { width, height } => {
raw.push((tags::TILE_WIDTH, Value::Long(vec![width as u32])));
raw.push((tags::TILE_LENGTH, Value::Long(vec![height as u32])));
raw.push((tags::TILE_OFFSETS, Value::Long(offs)));
raw.push((tags::TILE_BYTE_COUNTS, Value::Long(counts)));
}
}
match &o.cfa {
Some((r, c, p)) => {
raw.push((tags::PHOTOMETRIC, Value::Short(vec![32803])));
raw.push((tags::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![*r as u16, *c as u16])));
raw.push((tags::CFA_PATTERN, Value::Byte(p.clone())));
}
None => raw.push((tags::PHOTOMETRIC, Value::Short(vec![34892]))),
}
// later entries replace earlier ones with the same tag
let merge = |base: &mut Vec<(u16, Value)>, extra: &[(u16, Value)]| {
for (t, v) in extra {
base.retain(|e| e.0 != *t);
base.push((*t, v.clone()));
}
};
merge(&mut raw, &o.raw_tags);
let mut main = vec![(tags::DNG_VERSION, Value::Byte(vec![1, 4, 0, 0])), (tags::UNIQUE_CAMERA_MODEL, Value::Ascii("FilmCraft Synthetic".into()))];
let ifd0 = if o.preview {
let raw_at = put_ifd(&mut buf, raw, be);
// a 2 × 2 RGB preview
if buf.len() % 2 == 1 {
buf.push(0);
}
let px_at = buf.len() as u32;
buf.extend([128u8; 12]);
let mut p = vec![
(tags::NEW_SUBFILE_TYPE, Value::Long(vec![1])),
(tags::IMAGE_WIDTH, Value::Long(vec![2])),
(tags::IMAGE_LENGTH, Value::Long(vec![2])),
(tags::BITS_PER_SAMPLE, Value::Short(vec![8, 8, 8])),
(tags::COMPRESSION, Value::Short(vec![1])),
(tags::PHOTOMETRIC, Value::Short(vec![2])),
(tags::STRIP_OFFSETS, Value::Long(vec![px_at])),
(tags::SAMPLES_PER_PIXEL, Value::Short(vec![3])),
(tags::ROWS_PER_STRIP, Value::Long(vec![2])),
(tags::STRIP_BYTE_COUNTS, Value::Long(vec![12])),
(tags::SUB_IFDS, Value::Long(vec![raw_at])),
];
p.append(&mut main);
merge(&mut p, &o.main_tags);
put_ifd(&mut buf, p, be)
} else {
raw.append(&mut main);
merge(&mut raw, &o.main_tags);
put_ifd(&mut buf, raw, be)
};
let off = if be { ifd0.to_be_bytes() } else { ifd0.to_le_bytes() };
buf[4..8].copy_from_slice(&off);
buf
}

97
crates/dng/tests/oracle_libraw.rs Normal file
View File

@ -0,0 +1,97 @@
//! External oracle: LibRaw's `unprocessed_raw` sample program (an independent DNG reader) must
//! read the raw samples of our synthetic DNGs exactly as written and as we decode them: packed
//! uncompressed data at every depth in both byte orders, and lossless JPEG with every predictor.
//! Skipped when `unprocessed_raw` is not installed (or `FILMCRAFT_UNPROCESSED_RAW` names it).
//! LibRaw is only run, never linked or read.
use std::path::PathBuf;
use std::process::Command;
use filmcraft_dng::Dng;
use filmcraft_dng::writer::{Compression, Layout, Options, write};
fn tool() -> Option<String> {
let t = std::env::var("FILMCRAFT_UNPROCESSED_RAW").unwrap_or_else(|_| "unprocessed_raw".into());
Command::new(&t).arg("-q").output().ok().map(|_| t)
}
fn picture(w: usize, h: usize, bits: u32, seed: u32) -> Vec<u16> {
let mut s = seed.wrapping_mul(2_654_435_761).wrapping_add(7);
let max = (1u32 << bits) - 1;
(0..w * h)
.map(|i| {
s ^= s << 13;
s ^= s >> 17;
s ^= s << 5;
let (x, y) = (i % w, i / w);
(((x * 97 + y * 41) as u32 % (max + 1)) ^ (s & 15)).min(max) as u16
})
.collect()
}
/// The 16-bit samples of a binary PGM.
fn pgm(b: &[u8]) -> Option<(usize, usize, Vec<u16>)> {
let mut fields = Vec::new();
let mut i = 0;
while fields.len() < 4 {
while b.get(i)?.is_ascii_whitespace() {
i += 1;
}
let start = i;
while !b.get(i)?.is_ascii_whitespace() {
i += 1;
}
fields.push(std::str::from_utf8(&b[start..i]).ok()?.to_string());
}
let (w, h): (usize, usize) = (fields[1].parse().ok()?, fields[2].parse().ok()?);
let data = b.get(i + 1..)?;
Some((w, h, data.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect()))
}
#[test]
fn libraw_reads_our_raw_samples() {
let Some(tool) = tool() else {
eprintln!("SKIPPED: unprocessed_raw (LibRaw samples) not installed");
return;
};
let dir: PathBuf = std::env::temp_dir().join(format!("filmcraft-dng-oracle-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let (w, h) = (40, 26);
let mut cases: Vec<(String, Options, u32)> = Vec::new();
for bits in [8, 10, 12, 14, 16] {
for be in [false, true] {
cases.push((
format!("u{bits}{}", if be { "be" } else { "le" }),
Options { width: w, height: h, bits, big_endian: be, layout: Layout::Strips { rows: 7 }, ..Default::default() },
bits,
));
}
}
for predictor in 1..=7 {
for components in [1, 2] {
let o = Options {
width: w,
height: h,
bits: 14,
layout: Layout::Tiles { width: 16, height: 16 },
compression: Compression::LosslessJpeg { predictor, components, restart_lines: 0, pt: 0 },
..Default::default()
};
cases.push((format!("lj{predictor}c{components}"), o, 14));
}
}
for (i, (name, o, bits)) in cases.iter().enumerate() {
let px = picture(w, h, *bits, i as u32);
let f = write(&px, o);
assert_eq!(Dng::parse(&f).unwrap().raw().unwrap().data, px, "{name}: ours");
let path = dir.join(format!("{name}.dng"));
std::fs::write(&path, &f).unwrap();
let out = Command::new(&tool).arg("-q").arg(&path).output().unwrap();
assert!(out.status.success(), "{name}: {}", String::from_utf8_lossy(&out.stderr));
let (pw, ph, theirs) = pgm(&std::fs::read(dir.join(format!("{name}.dng.pgm"))).unwrap()).unwrap();
assert_eq!((pw, ph), (w, h), "{name}");
assert_eq!(theirs, px, "{name}: LibRaw");
eprintln!("{name}: LibRaw agrees");
}
let _ = std::fs::remove_dir_all(&dir);
}

366
crates/dng/tests/roundtrip.rs Normal file
View File

@ -0,0 +1,366 @@
//! Round trips through the test writer: container parsing, bit-exact raw decoding (uncompressed at
//! every depth, lossless JPEG with every predictor), the processing maths against hand-computed
//! values, CinemaDNG metadata, and hostile files.
use filmcraft_dng::color::{D50, D65, bradford, inverse, mul, rec709_to_xyz, xy_to_xyz};
use filmcraft_dng::tiff::tags;
use filmcraft_dng::writer::{Compression, Layout, Options, Value, lj92, write};
use filmcraft_dng::{Dng, Error, Timecode, decode, ljpeg, probe, sniff};
/// A deterministic pseudo-random sequence.
fn noise(n: usize, bits: u32, seed: u32) -> Vec<u16> {
let mut s = seed.wrapping_mul(2_654_435_761).wrapping_add(1);
(0..n)
.map(|_| {
s ^= s << 13;
s ^= s >> 17;
s ^= s << 5;
(s & ((1u32 << bits) - 1)) as u16
})
.collect()
}
/// A smooth picture with some noise (like a real sensor), `bits` deep.
fn picture(w: usize, h: usize, spp: usize, bits: u32, seed: u32) -> Vec<u16> {
let max = (1u32 << bits) - 1;
let n = noise(w * h * spp, 4.min(bits), seed);
(0..w * h * spp)
.map(|i| {
let (x, y) = ((i / spp) % w, (i / spp) / w);
let v = (x * 37 + y * 23 + (i % spp) * 101) as u32 % (max + 1);
(v ^ u32::from(n[i])).min(max) as u16
})
.collect()
}
#[test]
fn sniff_and_container() {
let px = picture(16, 8, 1, 12, 1);
for be in [false, true] {
for preview in [false, true] {
let o = Options { width: 16, height: 8, bits: 12, big_endian: be, preview, ..Default::default() };
let f = write(&px, &o);
assert!(sniff(&f), "be {be} preview {preview}");
let d = Dng::parse(&f).unwrap();
let info = d.info().unwrap();
assert_eq!((info.raw_width, info.raw_height, info.width, info.height, info.bits), (16, 8, 16, 8, 12));
assert!(info.cfa);
assert_eq!(info.dng_version, [1, 4, 0, 0]);
assert_eq!(info.camera.as_deref(), Some("FilmCraft Synthetic"));
assert_eq!(d.raw().unwrap().data, px, "be {be} preview {preview}");
}
}
// a plain TIFF is not a DNG
let mut plain = write(&px, &Options::default());
// remove the DNGVersion tag by renaming it to an unknown tag number
let pos = find_tag(&plain, tags::DNG_VERSION).unwrap();
plain[pos..pos + 2].copy_from_slice(&40000u16.to_le_bytes());
assert!(!sniff(&plain));
assert_eq!(Dng::parse(&plain).err(), Some(Error::NotDng));
assert!(!sniff(b"\x89PNG\r\n\x1a\n"));
assert!(!sniff(b""));
}
/// Offset of the directory entry for `tag` in a little-endian file's first IFD.
fn find_tag(f: &[u8], tag: u16) -> Option<usize> {
let ifd = u32::from_le_bytes(f[4..8].try_into().unwrap()) as usize;
let n = u16::from_le_bytes(f[ifd..ifd + 2].try_into().unwrap()) as usize;
(0..n).map(|i| ifd + 2 + 12 * i).find(|&e| u16::from_le_bytes(f[e..e + 2].try_into().unwrap()) == tag)
}
#[test]
fn uncompressed_every_depth_strips_and_tiles() {
for bits in [8, 9, 10, 12, 14, 16] {
for (spp, cfa) in [(1, true), (3, false)] {
for layout in [Layout::Strips { rows: 3 }, Layout::Strips { rows: 100 }, Layout::Tiles { width: 16, height: 8 }] {
for be in [false, true] {
let (w, h) = (21, 13);
let px = picture(w, h, spp, bits, bits + spp as u32);
let o = Options {
width: w,
height: h,
samples_per_pixel: spp,
bits,
big_endian: be,
layout,
cfa: cfa.then(|| (2, 2, vec![0, 1, 1, 2])),
..Default::default()
};
let raw = Dng::parse(&write(&px, &o)).unwrap().raw().unwrap();
assert_eq!(raw.data, px, "bits {bits} spp {spp} {layout:?} be {be}");
assert_eq!(raw.bits, bits);
}
}
}
}
}
#[test]
fn lossless_jpeg_every_predictor_bit_exact() {
for predictor in 1..=7u8 {
for bits in [8u32, 12, 14, 16] {
for (components, layout) in
[(1, Layout::Strips { rows: 5 }), (2, Layout::Tiles { width: 16, height: 16 }), (4, Layout::Tiles { width: 8, height: 4 })]
{
let (w, h) = (36, 22);
let px = picture(w, h, 1, bits, u32::from(predictor) * 7 + bits);
let o = Options {
width: w,
height: h,
bits,
layout,
compression: Compression::LosslessJpeg { predictor, components, restart_lines: 0, pt: 0 },
..Default::default()
};
let f = write(&px, &o);
assert_eq!(probe(&f).unwrap().compression, "lossless JPEG");
let raw = Dng::parse(&f).unwrap().raw().unwrap();
assert_eq!(raw.data, px, "predictor {predictor} bits {bits} components {components}");
}
}
}
}
#[test]
fn lossless_jpeg_extremes_restarts_and_point_transform() {
// full-swing 16-bit noise hits every difference category including SSSS = 16
let (w, h) = (17, 9);
let mut px = noise(w * h * 2, 16, 3);
px[0] = 0;
px[2] = 32_768; // 0 after the 2^15 initial prediction is a difference of 32768 (SSSS 16)
for predictor in 1..=7 {
let j = lj92(&px, w, h, 2, 16, predictor, 0, 0);
let d = ljpeg::decode(&j, 1 << 20).unwrap();
assert_eq!((d.width, d.height, d.components, d.precision, d.predictor), (w, h, 2, 16, predictor));
assert_eq!(d.samples, px, "predictor {predictor}");
// restart markers every 2 and every 3 lines
for r in [2, 3] {
let j = lj92(&px, w, h, 2, 16, predictor, r, 0);
assert!(j.windows(2).any(|b| b == [0xFF, 0xD0]), "has RST0");
assert_eq!(ljpeg::decode(&j, 1 << 20).unwrap().samples, px, "predictor {predictor} restart {r}");
}
}
// point transform: the low bits are dropped and the rest shifted back
let px = picture(12, 6, 1, 14, 9);
let j = lj92(&px, 12, 6, 1, 14, 4, 0, 2);
let want: Vec<u16> = px.iter().map(|v| v >> 2 << 2).collect();
assert_eq!(ljpeg::decode(&j, 1 << 20).unwrap().samples, want);
// the sample cap
assert_eq!(ljpeg::decode(&lj92(&px, 12, 6, 1, 14, 1, 0, 0), 71).err(), Some(Error::TooLarge));
}
/// The colour tags of a camera whose white-balanced values are Rec. 709 already.
fn rec709_camera_tags(neutral: [f64; 3]) -> Vec<(u16, Value)> {
let cm = inverse(&rec709_to_xyz().unwrap()).unwrap();
let fm = mul(&bradford(&xy_to_xyz(D65), &xy_to_xyz(D50)).unwrap(), &rec709_to_xyz().unwrap());
let flat = |m: [[f64; 3]; 3]| m.iter().flatten().copied().collect::<Vec<f64>>();
vec![
(tags::COLOR_MATRIX_1, Value::srationals(&flat(cm))),
(tags::FORWARD_MATRIX_1, Value::srationals(&flat(fm))),
(tags::CALIBRATION_ILLUMINANT_1, Value::Short(vec![21])),
(tags::AS_SHOT_NEUTRAL, Value::rationals(&neutral)),
]
}
#[test]
fn develop_matches_hand_computed_values() {
// 12-bit RGGB, black 64, white 4064 (range 4000); four colour quadrants seen through a camera
// with neutral (0.5, 1, 0.8). White-balanced values come out as Rec. 709 (the ForwardMatrix
// is Rec. 709 → XYZ D50), so a quadrant of colour c reads back c.
let (w, h) = (32, 24);
let neutral: [f64; 3] = [0.5, 1.0, 0.8];
let quad = |x: usize, y: usize| -> [f64; 3] {
match (x < w / 2, y < h / 2) {
(true, true) => [0.5, 0.5, 0.5],
(false, true) => [0.8, 0.2, 0.1],
(true, false) => [0.05, 0.6, 0.3],
(false, false) => [0.25, 0.25, 0.9],
}
};
let pat = [0u8, 1, 1, 2];
let px: Vec<u16> = (0..w * h)
.map(|i| {
let (x, y) = (i % w, i / w);
let c = usize::from(pat[(y % 2) * 2 + x % 2]);
let camera = quad(x, y)[c] * neutral[c];
(64.0 + camera * 4000.0).round() as u16
})
.collect();
let mut main = rec709_camera_tags(neutral);
main.push((tags::BASELINE_EXPOSURE, Value::SRational(vec![(0, 1)])));
let raw_tags = vec![(tags::BLACK_LEVEL, Value::Short(vec![64])), (tags::WHITE_LEVEL, Value::Short(vec![4064]))];
for compression in [Compression::None, Compression::LosslessJpeg { predictor: 6, components: 2, restart_lines: 0, pt: 0 }] {
let o = Options { width: w, height: h, bits: 12, compression, main_tags: main.clone(), raw_tags: raw_tags.clone(), ..Default::default() };
let img = decode(&write(&px, &o)).unwrap();
assert_eq!((img.width, img.height), (w, h));
for y in 0..h {
for x in 0..w {
// away from the quadrant edges (the demosaic reaches two pixels)
if (x as isize - w as isize / 2).abs() < 3 || (y as isize - h as isize / 2).abs() < 3 {
continue;
}
let got = &img.rgba[(y * w + x) * 4..][..4];
let want = quad(x, y);
for k in 0..3 {
assert!((got[k] - want[k] as f32).abs() < 2e-3, "({x},{y}) ch {k}: {} vs {}", got[k], want[k]);
}
assert_eq!(got[3], 1.0);
}
}
}
// BaselineExposure +1: twice as bright
let mut main1 = rec709_camera_tags(neutral);
main1.push((tags::BASELINE_EXPOSURE, Value::SRational(vec![(1, 1)])));
let o = Options { width: w, height: h, bits: 12, main_tags: main1, raw_tags, ..Default::default() };
let img = decode(&write(&px, &o)).unwrap();
assert!((img.rgba[(4 * w + 4) * 4] - 1.0).abs() < 4e-3, "{}", img.rgba[(4 * w + 4) * 4]);
}
#[test]
fn active_area_default_crop_and_linear_raw() {
// LinearRaw 3 samples, 10 × 6, active area cuts one pixel off each side, crop 4 × 2 at (2, 1)
let (w, h) = (10, 6);
let px: Vec<u16> = (0..w * h).flat_map(|i| [i as u16 * 100, 50, 7]).collect();
let o = Options {
width: w,
height: h,
samples_per_pixel: 3,
bits: 16,
cfa: None,
raw_tags: vec![
(tags::ACTIVE_AREA, Value::Long(vec![1, 1, 5, 9])),
(tags::DEFAULT_CROP_ORIGIN, Value::Long(vec![2, 1])),
(tags::DEFAULT_CROP_SIZE, Value::Rational(vec![(8, 2), (2, 1)])),
(tags::WHITE_LEVEL, Value::Long(vec![65_535])),
],
..Default::default()
};
let f = write(&px, &o);
let info = probe(&f).unwrap();
assert_eq!((info.width, info.height, info.cfa, info.samples_per_pixel), (4, 2, false, 3));
let img = decode(&f).unwrap();
assert_eq!((img.width, img.height), (4, 2));
// output (0, 0) is raw (1 + 2, 1 + 1) = sample 23
assert!((img.rgba[0] - 2300.0 / 65_535.0).abs() < 1e-6);
assert!((img.rgba[1] - 50.0 / 65_535.0).abs() < 1e-6);
// output (3, 1) is raw (6, 3) = sample 36
assert!((img.rgba[(4 + 3) * 4] - 3600.0 / 65_535.0).abs() < 1e-6);
}
#[test]
fn cinemadng_frame_rate_time_code_and_reel() {
let px = picture(8, 8, 1, 12, 2);
let o = Options {
bits: 12,
main_tags: vec![
(tags::FRAME_RATE, Value::SRational(vec![(24_000, 1001)])),
// 01:02:03:04, drop frame
(tags::TIME_CODES, Value::Byte(vec![0x44, 0x03, 0x02, 0x01, 0, 0, 0, 0])),
(tags::REEL_NAME, Value::Ascii("A001".into())),
],
..Default::default()
};
let info = probe(&write(&px, &o)).unwrap();
assert_eq!(info.frame_rate, Some((24_000, 1001)));
assert_eq!(info.timecode, Some(Timecode { hours: 1, minutes: 2, seconds: 3, frames: 4, drop_frame: true }));
assert_eq!(info.reel.as_deref(), Some("A001"));
// nonsense rates and time codes are ignored
let o = Options {
bits: 12,
main_tags: vec![(tags::FRAME_RATE, Value::SRational(vec![(-25, 1)])), (tags::TIME_CODES, Value::Byte(vec![0x0A, 0x99, 0, 0, 0, 0, 0, 0]))],
..Default::default()
};
let info = probe(&write(&px, &o)).unwrap();
assert_eq!((info.frame_rate, info.timecode), (None, None));
}
/// Every hostile variant must give an error or a picture, never a panic.
fn survives(f: &[u8]) -> bool {
std::panic::catch_unwind(|| {
let _ = probe(f);
let _ = decode(f);
})
.is_ok()
}
#[test]
fn hostile_files_are_errors_not_panics() {
let px = picture(24, 16, 1, 12, 5);
let files: Vec<Vec<u8>> = [
Options { width: 24, height: 16, bits: 12, ..Default::default() },
Options { width: 24, height: 16, bits: 12, preview: true, layout: Layout::Tiles { width: 16, height: 16 }, ..Default::default() },
Options {
width: 24,
height: 16,
bits: 12,
compression: Compression::LosslessJpeg { predictor: 4, components: 2, restart_lines: 2, pt: 0 },
main_tags: rec709_camera_tags([0.5, 1.0, 0.7]),
..Default::default()
},
]
.iter()
.map(|o| write(&px, o))
.collect();
for f in &files {
assert!(decode(f).is_ok());
// every truncation
for n in 0..f.len() {
assert!(survives(&f[..n]), "truncated to {n}");
if n < f.len() / 2 {
assert!(decode(&f[..n]).is_err(), "truncated to {n}");
}
}
// bit flips and corrupt bytes everywhere
let mut s = 12345u32;
for i in 0..4000 {
let mut g = f.clone();
for _ in 0..1 + i % 4 {
s ^= s << 13;
s ^= s >> 17;
s ^= s << 5;
let at = s as usize % g.len();
g[at] = if i % 3 == 0 { 0xFF } else { g[at] ^ (1 << (s >> 29)) };
}
assert!(survives(&g), "mutation {i}");
}
}
}
#[test]
fn cyclic_ifds_huge_sizes_and_bad_offsets() {
let px = picture(8, 8, 1, 16, 4);
let f = write(&px, &Options { preview: true, ..Default::default() });
let ifd0 = u32::from_le_bytes(f[4..8].try_into().unwrap()) as usize;
let n = u16::from_le_bytes(f[ifd0..ifd0 + 2].try_into().unwrap()) as usize;
// IFD 0's next pointer to itself
let mut g = f.clone();
g[ifd0 + 2 + 12 * n..ifd0 + 6 + 12 * n].copy_from_slice(&(ifd0 as u32).to_le_bytes());
assert!(decode(&g).is_ok(), "a self-loop ends the chain");
// the SubIFD pointing back at IFD 0
let mut g = f.clone();
let sub = find_tag(&g, tags::SUB_IFDS).unwrap();
g[sub + 8..sub + 12].copy_from_slice(&(ifd0 as u32).to_le_bytes());
assert!(matches!(decode(&g), Err(Error::Unsupported(_))), "no raw image left");
// huge dimensions with a tiny file
for (wtag, v) in [(tags::IMAGE_WIDTH, 65_535u32), (tags::IMAGE_LENGTH, 65_535), (tags::IMAGE_WIDTH, 0), (tags::IMAGE_WIDTH, u32::MAX)] {
let mut g = write(&px, &Options::default());
let e = find_tag(&g, wtag).unwrap();
g[e + 8..e + 12].copy_from_slice(&v.to_le_bytes());
assert!(decode(&g).is_err(), "{wtag} = {v}");
}
// a strip offset past the end of the file
let mut g = write(&px, &Options::default());
let e = find_tag(&g, tags::STRIP_OFFSETS).unwrap();
g[e + 8..e + 12].copy_from_slice(&0xFFFF_FF00u32.to_le_bytes());
assert_eq!(decode(&g).err(), Some(Error::Truncated));
// an IFD offset past the end
let mut g = write(&px, &Options::default());
g[4..8].copy_from_slice(&0x7FFF_FFFFu32.to_le_bytes());
assert!(decode(&g).is_err());
// a rational with a zero denominator in a raw tag
let o = Options { raw_tags: vec![(tags::DEFAULT_CROP_SIZE, Value::Rational(vec![(8, 0), (8, 1)]))], ..Default::default() };
assert!(matches!(decode(&write(&px, &o)), Err(Error::Malformed(_))));
// unsupported compressions and depths are named
let o = Options { raw_tags: vec![(tags::COMPRESSION, Value::Short(vec![34892]))], ..Default::default() };
assert_eq!(decode(&write(&px, &o)).err(), Some(Error::Unsupported("lossy JPEG DNG".into())));
}

97
crates/edit/src/lib.rs
View File

@ -247,6 +247,15 @@ fn track_mut(seq: &mut Sequence, id: TrackId) -> Result<&mut Track> {
seq.track_mut(id).ok_or(EditError::NoTrack(id))
}
/// The clip's track, or [`EditError::Locked`] when that track may not change.
fn unlocked_track(seq: &Sequence, clip: ClipId) -> Result<TrackId> {
let (tid, _) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
if seq.track(tid).ok_or(EditError::NoTrack(tid))?.locked {
return Err(EditError::Locked);
}
Ok(tid)
}
/// Overwrite edits: each item replaces material on its track.
pub fn overwrite(seq: &mut Sequence, placements: Vec<(TrackId, TrackItem)>, ctx: &mut EditCtx) -> Result<Vec<ClipId>> {
let mut links = HashMap::new();
@ -503,7 +512,10 @@ pub fn move_items(seq: &mut Sequence, moves: &[(ClipId, TrackId, Tick)], insert_
let mut work = seq.clone();
let mut placed = Vec::new();
for (c, dest, start) in moves {
let (_, it) = work.find_item(*c).ok_or(EditError::NoItem(*c))?;
let (src, it) = work.find_item(*c).ok_or(EditError::NoItem(*c))?;
if work.track(src).is_some_and(|t| t.locked) {
return Err(EditError::Locked);
}
let mut it = it.clone();
if work.track(*dest).ok_or(EditError::NoTrack(*dest))?.locked {
return Err(EditError::Locked);
@ -633,7 +645,8 @@ fn neighbours(track: &Track, id: ClipId) -> (Tick, Tick) {
/// Compute the clamped delta a trim may apply (for UI feedback while dragging).
pub fn clamp_trim(seq: &Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta: Tick, ctx: &EditCtx) -> Result<Tick> {
let (tid, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let tid = unlocked_track(seq, clip)?;
let (_, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let tr = seq.track(tid).ok_or(EditError::NoTrack(tid))?;
let (prev_end, next_start) = neighbours(tr, clip);
let media = media_len(ctx, it);
@ -703,6 +716,7 @@ pub fn move_edge(it: &mut TrackItem, edge: Edge, d: Tick, move_start: bool) {
/// Trim one edge of an item (Selection-tool edge drag = Regular; Ripple tool = Ripple).
/// Linked partners should be trimmed by the caller with the same delta.
pub fn trim(seq: &mut Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, clip)?;
let d = clamp_trim(seq, clip, edge, mode, delta, ctx)?;
if d == Tick::ZERO {
return Ok(d);
@ -759,6 +773,9 @@ pub fn trim(seq: &mut Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta:
/// overwrite material on a sync-locked track, or when such a linked clip has no room to follow.
pub fn ripple_trim_group(seq: &mut Sequence, clips: &[ClipId], edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
let Some(&first) = clips.first() else { return Ok(Tick::ZERO) };
for c in clips {
unlocked_track(seq, *c)?;
}
let mut d = delta;
for c in clips {
let x = clamp_trim(seq, *c, edge, TrimMode::Ripple, d, ctx)?;
@ -861,6 +878,8 @@ pub fn ripple_trim_group(seq: &mut Sequence, clips: &[ClipId], edge: Edge, delta
/// Rolling edit between two adjacent items on one track: moves the cut by `delta`.
pub fn roll(seq: &mut Sequence, left: ClipId, right: ClipId, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, left)?;
unlocked_track(seq, right)?;
let (_, l) = seq.find_item(left).ok_or(EditError::NoItem(left))?;
let (_, r) = seq.find_item(right).ok_or(EditError::NoItem(right))?;
if l.end() != r.start {
@ -908,6 +927,7 @@ pub fn roll(seq: &mut Sequence, left: ClipId, right: ClipId, delta: Tick, ctx: &
/// Slip: change which part of the media an item shows without moving it.
pub fn slip(seq: &mut Sequence, clip: ClipId, delta_media: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, clip)?;
let (_, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let used = src_of(it.duration, it.speed);
let max_in = media_len(ctx, it).map(|m| m - used).unwrap_or(Tick::MAX);
@ -921,6 +941,7 @@ pub fn slip(seq: &mut Sequence, clip: ClipId, delta_media: Tick, ctx: &mut EditC
/// Slide: move an item between its neighbours, trimming them to keep the gapless span.
pub fn slide(seq: &mut Sequence, clip: ClipId, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, clip)?;
let (tid, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let it = it.clone();
let tr = seq.track(tid).ok_or(EditError::NoTrack(tid))?;
@ -999,27 +1020,65 @@ pub fn slide_items(seq: &mut Sequence, clips: &[ClipId], delta: Tick, ctx: &mut
Ok(d)
}
/// Rate stretch: change an item's duration by dragging an edge, adjusting speed to keep the same media.
pub fn rate_stretch(seq: &mut Sequence, clip: ClipId, edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<f64> {
/// How far `clip` can rate-stretch on `edge` without overlapping a neighbour or going shorter than
/// one frame. Linked partners use this so they all take the tightest room.
pub fn clamp_rate_stretch(seq: &Sequence, clip: ClipId, edge: Edge, delta: Tick, ctx: &EditCtx) -> Result<Tick> {
let (tid, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let tr = seq.track(tid).ok_or(EditError::NoTrack(tid))?;
let (prev_end, next_start) = neighbours(tr, clip);
let src_len = it.source_out() - it.source_in;
let d = match edge {
Ok(match edge {
Edge::Out => delta.clamp(-(it.duration - ctx.min_duration), next_start - it.end()),
Edge::In => delta.clamp(-(it.start - prev_end), it.duration - ctx.min_duration),
};
let new_dur = match edge {
Edge::Out => it.duration + d,
Edge::In => it.duration - d,
};
let speed = src_len.0 as f64 / new_dur.0 as f64;
let (_, it) = seq.find_item_mut(clip).ok_or(EditError::NoItem(clip))?;
if edge == Edge::In {
it.start += d;
})
}
/// Rate stretch: change an item's duration by dragging an edge, adjusting speed to keep the same media.
pub fn rate_stretch(seq: &mut Sequence, clip: ClipId, edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<f64> {
rate_stretch_items(seq, &[clip], edge, delta, ctx)
}
/// Rate stretch a clip and its linked partners as one edit: the tightest neighbour on any member
/// clamps them all, so they keep one speed. A neighbour on the sound track cannot leave the picture
/// stretching further (and the pair still linked at different rates).
pub fn rate_stretch_items(seq: &mut Sequence, clips: &[ClipId], edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<f64> {
// clips on locked tracks are left as they are; with every clip locked the edit is refused
let mut open = Vec::with_capacity(clips.len());
for &c in clips {
match unlocked_track(seq, c) {
Ok(_) => open.push(c),
Err(EditError::Locked) => {}
Err(e) => return Err(e),
}
}
if open.is_empty() {
return Err(EditError::Locked);
}
let clips = open.as_slice();
let mut d = delta;
for c in clips {
let x = clamp_rate_stretch(seq, *c, edge, d, ctx)?;
if x.0.signum() != d.0.signum() {
d = Tick::ZERO;
} else if x.abs() < d.abs() {
d = x;
}
}
let mut speed = 1.0;
for c in clips {
let (_, it) = seq.find_item(*c).ok_or(EditError::NoItem(*c))?;
let src_len = it.source_out() - it.source_in;
let new_dur = match edge {
Edge::Out => it.duration + d,
Edge::In => it.duration - d,
};
speed = src_len.0 as f64 / new_dur.0 as f64;
let (_, it) = seq.find_item_mut(*c).ok_or(EditError::NoItem(*c))?;
if edge == Edge::In {
it.start += d;
}
it.duration = new_dur;
it.speed = speed;
}
it.duration = new_dur;
it.speed = speed;
Ok(speed)
}
@ -1038,6 +1097,9 @@ pub fn set_speed_group(seq: &mut Sequence, clips: &[ClipId], speed: f64, reverse
if !(speed.is_finite() && speed > 0.0) {
return Err(EditError::Other("speed must be positive".into()));
}
for clip in clips {
unlocked_track(seq, *clip)?;
}
let mut work = seq.clone();
// per member track: where its later material starts, and by how much the member's length changed
let mut origins: Vec<(TrackId, Tick, Tick)> = Vec::new();
@ -1061,6 +1123,7 @@ pub fn set_speed_group(seq: &mut Sequence, clips: &[ClipId], speed: f64, reverse
it.speed = speed;
it.reverse = reverse;
it.duration = new_dur;
fades_follow_edges(work.track_mut(tid).ok_or(EditError::NoTrack(tid))?, *clip);
origins.push((tid, old_end, delta));
}
let main_from = origins.first().map(|o| o.1).unwrap_or_default();

58
crates/edit/src/tests.rs
View File

@ -274,6 +274,17 @@ fn regular_trim_respects_neighbours_and_handles() {
fx.seq.check().unwrap();
}
#[test]
fn trim_on_a_locked_track_is_refused() {
let mut fx = Fx::new();
let v1 = fx.v(0);
let a = fx.put(v1, 0, 100, 0);
fx.seq.track_mut(v1).unwrap().locked = true;
let mut n = fx.next;
assert_eq!(trim(&mut fx.seq, a, Edge::Out, TrimMode::Regular, -f(20), &mut Fx::ctx(&mut n)), Err(EditError::Locked));
assert_eq!(fx.spans(v1), vec![(0, 100)]);
}
#[test]
fn held_regular_in_trim_stops_at_previous_clip() {
let mut fx = Fx::new();
@ -492,6 +503,30 @@ fn fades_follow_a_regular_trim() {
fx.seq.check().unwrap();
}
#[test]
fn fades_follow_a_non_ripple_speed_change() {
// #918: faster without ripple shortens the clip; the fade-out stays on the new out point.
let mut fx = Fx::new();
let v1 = fx.v(0);
let a = fx.put(v1, 0, 100, 0);
fx.seq.track_mut(v1).unwrap().transitions = vec![Transition {
id: TransitionId(1),
effect: filmcraft_project::find_effect("cross_dissolve").unwrap().instance(),
start: f(90),
duration: f(10),
from: Some(a),
to: None,
align: Default::default(),
reverse: false,
}];
let mut n = fx.next;
set_speed(&mut fx.seq, a, 2.0, false, false, &mut Fx::ctx(&mut n)).unwrap();
let tr = &fx.seq.track(v1).unwrap().transitions[0];
assert_eq!(fx.spans(v1), vec![(0, 50)]);
assert_eq!((tr.start, tr.duration), (f(40), f(10)));
fx.seq.check().unwrap();
}
#[test]
fn transitions_travel_with_moved_clips() {
// V1: a [0,10) fading in, b [10,20), c [20,30); b→c crossfade, c fading out
@ -537,6 +572,29 @@ fn rate_stretch_and_speed() {
assert_eq!(fx.spans(v1), vec![(0, 5)]);
}
#[test]
fn linked_rate_stretch_uses_the_tighter_neighbour() {
// picture and sound are 60 frames; a later clip on A1 starts at 70, so the sound has only 10
// frames of room. Stretching the pair +20 must not leave the picture at 80f / 0.75x and the
// sound at 70f / 6/7x while still linked.
let mut fx = Fx::new();
let (v1, a1) = (fx.v(0), fx.a(0));
let pic = fx.put(v1, 0, 60, 0);
let snd = fx.put(a1, 0, 60, 0);
fx.put(a1, 70, 10, 0);
for c in [pic, snd] {
fx.seq.find_item_mut(c).unwrap().1.link = Some(1);
}
let mut n = fx.next;
let speed = rate_stretch_items(&mut fx.seq, &[pic, snd], Edge::Out, f(20), &mut Fx::ctx(&mut n)).unwrap();
assert!((speed - 60.0 / 70.0).abs() < 1e-9);
let (p, s) = (fx.seq.find_item(pic).unwrap().1, fx.seq.find_item(snd).unwrap().1);
assert_eq!((R.frame_at(p.duration), R.frame_at(s.duration)), (70, 70));
assert!((p.speed - s.speed).abs() < 1e-12);
assert_eq!(fx.spans(a1), vec![(0, 70), (70, 10)]);
fx.seq.check().unwrap();
}
#[test]
fn speed_on_a_linked_group_ripples_once() {
// V1: a [0,10), b [10,20). A1: a's sound [0,10), b's sound [10,20).

4
crates/engine/Cargo.toml
View File

@ -46,4 +46,6 @@ web-time = "1"
[dev-dependencies]
filmcraft-testkit = { workspace = true }
image = { version = "0.25", default-features = false, features = ["png"] }
filmcraft-dng = { workspace = true, features = ["writer"] }
filmcraft-isobmff = { workspace = true }
image = { version = "0.25", default-features = false, features = ["png", "gif"] }

62
crates/engine/src/animated_image_tests.rs Normal file
View File

@ -0,0 +1,62 @@
//! Animated GIF and APNG files import as movie clips that play every frame with its transparency
//! (#402), and reopen that way from a saved project.
use filmcraft_media::{FrameRequest, MediaKind};
use filmcraft_project::ItemId;
use filmcraft_time::{TICKS_PER_SECOND, Tick};
use serde_json::{Value, json};
use crate::Session;
use crate::media_test_util::tmp_dir;
/// A 16×8 animated GIF: frame `i` is red `40 * i` with its left half transparent, 100 ms each.
fn write_gif(path: &std::path::Path, frames: u8) {
let mut out = Vec::new();
{
let mut e = image::codecs::gif::GifEncoder::new(&mut out);
let frames = (0..frames).map(|i| {
let img = image::RgbaImage::from_fn(16, 8, |x, _| image::Rgba([40 * i, 0, 0, if x < 8 { 0 } else { 255 }]));
image::Frame::from_parts(img, 0, 0, image::Delay::from_numer_denom_ms(100, 1))
});
e.encode_frames(frames).unwrap();
}
std::fs::write(path, out).unwrap();
}
fn item_of(r: &Value) -> ItemId {
ItemId(r["items"][0].as_u64().unwrap_or_else(|| panic!("{r}")))
}
/// The pixel at (`x`, 0) at media time `ms`.
fn pixel_at(s: &Session, item: ItemId, ms: i64, x: usize) -> [u8; 4] {
let src = s.media.source_for(&s.project, item, &*s.services).unwrap();
let f = src.video_frame(FrameRequest::full(Tick(ms * TICKS_PER_SECOND / 1000))).unwrap();
match &f.data {
filmcraft_frame::PixelData::Rgba8(p) => [p[4 * x], p[4 * x + 1], p[4 * x + 2], p[4 * x + 3]],
_ => panic!("rgba frames"),
}
}
#[test]
fn animated_gif_imports_as_a_clip_and_reopens() {
let dir = tmp_dir("animated-gif");
let gif = dir.join("loop.gif");
write_gif(&gif, 4);
let mut s = Session::default();
let id = item_of(&s.execute("file.import", json!({"paths": [gif.to_string_lossy()]})).unwrap());
let m = s.project.item(id).unwrap().as_media().unwrap();
assert_eq!(m.info.kind, MediaKind::Movie, "not a still showing the first frame");
assert_eq!(m.info.duration, Tick(4 * TICKS_PER_SECOND / 10));
assert!(m.info.video.as_ref().unwrap().has_alpha);
assert_eq!(pixel_at(&s, id, 0, 12), [0, 0, 0, 255]);
assert_eq!(pixel_at(&s, id, 250, 12), [80, 0, 0, 255], "the third frame plays");
assert_eq!(pixel_at(&s, id, 250, 2)[3], 0, "transparent pixels stay transparent, not black");
let proj = dir.join("anim.fcproj");
s.execute("file.saveAs", json!({"path": proj.to_string_lossy()})).unwrap();
let mut t = Session::default();
t.execute("file.open", json!({"path": proj.to_string_lossy()})).unwrap();
assert_eq!(t.project.item(id).unwrap().as_media().unwrap().info.kind, MediaKind::Movie);
assert_eq!(pixel_at(&t, id, 350, 12), [120, 0, 0, 255], "frames reopen through the media pool");
let _ = std::fs::remove_dir_all(&dir);
}

16
crates/engine/src/audio_effects_tests.rs
View File

@ -111,3 +111,19 @@ fn shared_names_resolve_by_track_kind() {
let it = s.active_sequence().unwrap().find_item(ClipId(v)).unwrap().1.clone();
assert!(it.effects.iter().any(|e| e.effect == "invert"));
}
/// #696: applying an audio effect to the video side of an A/V pair lands on the linked audio.
#[test]
fn audio_effect_on_video_clip_applies_to_linked_audio() {
let mut s = demo();
let seq = s.active_sequence().unwrap();
let v = seq.video_tracks[0].items[0].clone();
let a = seq.audio_tracks[0].items.iter().find(|i| i.link == v.link).expect("linked audio").clone();
let n0 = a.effects.len();
s.execute("effects.apply", json!({"clips": [v.id.0], "effect": "parametric_eq"})).unwrap();
let a = s.active_sequence().unwrap().find_item(a.id).unwrap().1.clone();
let v = s.active_sequence().unwrap().find_item(v.id).unwrap().1.clone();
assert_eq!(a.effects.len(), n0 + 1);
assert_eq!(a.effects.last().unwrap().effect, "parametric_eq");
assert!(v.effects.iter().all(|e| e.effect != "parametric_eq"), "EQ stays off the picture clip");
}

26
crates/engine/src/audio_placement_tests.rs
View File

@ -168,6 +168,32 @@ fn missing_audio_tracks_are_cloned_from_the_destination() {
assert_eq!(again, ids);
}
/// #823: a container with two PCM tracks imports both streams, and overwrite places linked clips.
#[test]
fn import_keeps_every_audio_stream_and_places_linked_clips() {
use filmcraft_isobmff::{Brand, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions};
let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
for (rate, value) in [(48_000u32, 8192i16), (24_000, -16384)] {
let entry = SampleEntry::pcm(PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: rate as f64 });
let track = mux.add_track(TrackConfig::new(entry, rate)).unwrap();
let data: Vec<u8> = (0..rate / 10).flat_map(|_| value.to_le_bytes()).collect();
mux.write_sample(track, WriteSample { data: &data, duration: rate / 10, composition_offset: 0, is_sync: true }).unwrap();
}
let bytes = mux.finish().unwrap().into_inner();
let mut s = Session::default();
let item = crate::commands::import_bytes(&mut s, "/two-audio.mov", bytes.into(), None).unwrap();
let streams = s.project.item(item).unwrap().as_media().unwrap().info.audio_streams.len();
assert_eq!(streams, 2);
s.execute("file.newSequence", json!({"name": "S", "fromItem": item.0, "video": 1, "audio": 1})).unwrap();
let seq = s.state.active_sequence.unwrap();
let q = s.project.sequence(seq).unwrap();
assert_eq!(q.audio_tracks.len(), 2);
let (a, b) = (&q.audio_tracks[0].items[0], &q.audio_tracks[1].items[0]);
assert_eq!((a.audio_stream, b.audio_stream), (0, 1));
assert_eq!(a.link, b.link);
assert!(a.link.is_some());
}
#[test]
fn hostile_track_counts_are_an_error_not_a_crash() {
let s = demo();

17
crates/engine/src/autosave.rs
View File

@ -572,15 +572,16 @@ pub fn scan_candidates(data_dir: &Path, skip: Option<&Path>) -> Vec<RecoveryCand
if !dir.is_dir() || Some(dir.as_path()) == skip || !lock_is_stale(&dir) {
continue;
}
let meta = fs::read(dir.join(META)).ok().and_then(|b| serde_json::from_slice::<JournalMeta>(&b).ok());
match meta {
Some(meta) if dir.join(SNAPSHOT).is_file() => {
out.push(RecoveryCandidate { id: e.file_name().to_string_lossy().into_owned(), dir, meta });
}
_ => {
let _ = fs::remove_dir_all(&dir);
}
let snap = dir.join(SNAPSHOT);
if !snap.is_file() {
// Empty leftover dirs (clean exit, or a crash before the first snapshot).
let _ = fs::remove_dir_all(&dir);
continue;
}
// Keep the snapshot even when session.json is missing or unreadable: the project
// is still recoverable. Meta fields fall back to defaults.
let meta = fs::read(dir.join(META)).ok().and_then(|b| serde_json::from_slice::<JournalMeta>(&b).ok()).unwrap_or_default();
out.push(RecoveryCandidate { id: e.file_name().to_string_lossy().into_owned(), dir, meta });
}
out.sort_by(|a, b| b.meta.saved_unix.cmp(&a.meta.saved_unix).then(b.id.cmp(&a.id)));
out

22
crates/engine/src/autosave_tests.rs
View File

@ -176,6 +176,28 @@ fn auto_save_ring_rotates_and_can_save_the_project() {
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn scan_keeps_snapshot_when_session_metadata_is_unreadable() {
// #894: a stale session with a snapshot must not be deleted because session.json is missing
// or corrupt — the recovered project would vanish from the list and the file would be gone.
let d = temp_dir("recover-nometa");
let root = autosave::recovery_root(&d);
let sess = root.join("123-1");
std::fs::create_dir_all(&sess).unwrap();
let snap = filmcraft_format::encode(&filmcraft_project::Project::default(), false);
std::fs::write(sess.join("snapshot.fcproj"), snap).unwrap();
std::fs::write(sess.join("session.json"), b"not json {").unwrap();
let found = autosave::scan_candidates(&d, None);
assert_eq!(found.len(), 1);
assert_eq!(found[0].id, "123-1");
assert!(sess.join("snapshot.fcproj").is_file(), "snapshot must survive the scan");
std::fs::remove_file(sess.join("session.json")).unwrap();
let found = autosave::scan_candidates(&d, None);
assert_eq!(found.len(), 1, "missing metadata still lists the snapshot");
assert!(sess.join("snapshot.fcproj").is_file());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn portable_marker_keeps_data_next_to_the_executable() {
// #139: a portable install must not write to %APPDATA%.

58
crates/engine/src/avi_import_tests.rs Normal file
View File

@ -0,0 +1,58 @@
//! #598: AVI files import (ffmpeg-made, skipped without it): the Project item describes the
//! streams, a sequence made from it plays picture and sound, and the Media Browser's streaming
//! probe opens it without reading it whole.
use serde_json::json;
use crate::Session;
use filmcraft_project::ItemId;
fn fixture() -> Option<std::path::PathBuf> {
let ffmpeg = filmcraft_testkit::oracle::ffmpeg_or_skip("avi import")?;
let path = filmcraft_testkit::fixtures_dir("engine/avi").join("h264_mp3.avi");
filmcraft_testkit::fixtures::generate(&path, |tmp| {
std::process::Command::new(&ffmpeg)
.args([
"-y",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=size=160x120:rate=25:duration=2",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=48000:duration=2",
])
.args(["-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p", "-c:a", "libmp3lame", "-f", "avi"])
.arg(tmp)
.status()
.is_ok_and(|s| s.success())
})
}
#[test]
fn avi_imports_plays_and_probes() {
let Some(path) = fixture() else { return };
let mut s = Session::default();
let r = s.execute("file.import", json!({"paths": [path.to_string_lossy()]})).unwrap();
assert!(r["errors"].as_array().is_none_or(|e| e.is_empty()), "{r}");
let item = ItemId(r["items"][0].as_u64().unwrap());
let info = s.project.item(item).unwrap().as_media().unwrap().info.clone();
assert_eq!(info.container, "AVI");
let v = info.video.as_ref().unwrap();
assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, "H.264"));
assert_eq!(info.audio_streams.first().map(|a| a.codec.as_str()), Some("MP3"));
assert!((info.duration.seconds() - 2.0).abs() < 0.1, "{}", info.duration.seconds());
// a sequence from the clip has its picture and sound
s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap();
let q = s.active_sequence().unwrap().clone();
assert!(!q.video_tracks[0].items.is_empty() && !q.audio_tracks[0].items.is_empty());
let src = s.source(item).unwrap();
assert!(src.video_frame(filmcraft_media::FrameRequest::full(filmcraft_time::Tick(filmcraft_time::TICKS_PER_SECOND))).is_ok());
assert!(src.audio(48_000, 4_800, 48_000).unwrap().channels[0].iter().any(|v| v.abs() > 0.05));
// the Media Browser's probe streams it (index now, frames on demand)
let probed = s.media.probe_file(&path.to_string_lossy(), &*s.services).unwrap();
assert_eq!(probed.info().container, "AVI");
}

4
crates/engine/src/clip_ops.rs
View File

@ -1715,8 +1715,8 @@ fn replace_with(s: &mut Session, p: &Value, how: Replace) -> Result<Value> {
}
let Some((_, it)) = q.find_item_mut(*c) else { continue };
let src_in = match how {
// the Source monitor frame lines up with the sequence playhead
Replace::MatchFrame => src_ph - Tick(((ph - it.start).0 as f64 * it.speed.abs()).round() as i64),
// the Source monitor frame lines up with the sequence playhead (reverse-aware)
Replace::MatchFrame => src_ph - (it.moving_source_time_at(ph) - it.source_in),
_ if keep_in => {
let kept = keep_within.map(|(first, last)| it.source_in.clamp(first, last)).unwrap_or(it.source_in);
if kept != it.source_in {

71
crates/engine/src/clip_ops_tests.rs
View File

@ -606,6 +606,32 @@ fn replace_with_clip_from_source_match_frame_and_bin() {
assert!(s.execute("clip.replaceFromBin", json!({"item": music.0})).is_err());
}
/// Match Frame on a reversed clip maps the Source monitor frame through reverse playback, not as
/// if the clip still ran forward (source 50 at sequence 20 used to become source 69).
#[test]
fn replace_from_source_match_frame_on_a_reversed_clip() {
let mut s = demo();
let rate = s.sequence_rate();
s.execute("sequence.linkedSelection", json!({"on": false})).unwrap();
let forest = item_named(&s, "Misty_Forest.mp4");
s.execute("file.newSequence", json!({"name": "MatchFrameReverse"})).unwrap();
s.execute("timeline.place", json!({"item": forest.0, "time": 0})).unwrap();
let c = s.active_sequence().unwrap().video_tracks[0].items[0].clone();
s.execute("timeline.trim", json!({"clip": c.id.0, "edge": "out", "mode": "regular", "delta": (rate.tick_of(60) - c.duration).0})).unwrap();
select(&mut s, &[c.id.0]);
s.execute("clip.speedDuration", json!({"speed": 100, "reverse": true, "ripple": false})).unwrap();
let got = clip(&s, c.id.0);
assert_eq!(rate.frame_at(got.duration), 60);
assert!(got.reverse);
s.execute("source.open", json!({"item": forest.0})).unwrap();
s.execute("playhead.set", json!({"time": rate.tick_of(20).0})).unwrap();
s.execute("source.setPlayhead", json!({"frame": 50})).unwrap();
s.execute("clip.replaceFromSourceMatchFrame", json!({})).unwrap();
let got = clip(&s, c.id.0);
assert_eq!(got.source_time_at(rate.tick_of(20)), rate.tick_of(50));
assert_ne!(got.source_time_at(rate.tick_of(20)), rate.tick_of(69));
}
/// Replace With Clip ▸ From Bin restarts the clip at the new item's In point (as Premiere does);
/// `keepSourceIn` keeps the clip's place in the media instead.
#[test]
@ -868,6 +894,29 @@ fn color_matte_color_changes_and_undoes() {
assert!(s.execute("project.matteColor", json!({"item": matte, "color": "green"})).is_err());
}
/// #918: non-ripple Speed/Duration 200% halves a 100-frame clip; a 10-frame fade-out must
/// move from frames 90–100 to 40–50 (the new clip end), not stay past the clip.
#[test]
fn speed_duration_moves_fade_out_to_the_new_clip_end() {
let mut s = Session::default();
s.execute("file.newSequence", json!({"name": "Speed fade", "video": 1, "audio": 0})).unwrap();
let rate = s.sequence_rate();
let seconds = rate.tick_of(100).seconds();
let matte = s.execute("file.newColorMatte", json!({"color": "#ff0000", "seconds": seconds})).unwrap()["item"].as_u64().unwrap();
let c = s.execute("timeline.place", json!({"item": matte, "track": "V1", "seconds": 0.0})).unwrap()["clips"][0].as_u64().unwrap();
select(&mut s, &[c]);
s.execute("sequence.applyVideoTransition", json!({"edge": "out", "frames": 10})).unwrap();
let before = s.active_sequence().unwrap().video_tracks[0].transitions[0].clone();
assert_eq!((rate.frame_at(before.start), rate.frame_at(before.end())), (90, 100));
s.execute("clip.speedDuration", json!({"speed": 200})).unwrap();
let it = clip(&s, c);
let fade = &s.active_sequence().unwrap().video_tracks[0].transitions[0];
assert_eq!((rate.frame_at(it.start), rate.frame_at(it.end())), (0, 50));
assert_eq!((rate.frame_at(fade.start), rate.frame_at(fade.end())), (40, 50));
assert_eq!(fade.from, Some(filmcraft_project::ClipId(c)));
assert_eq!(fade.to, None);
}
/// A damaged project whose subclip is its own parent (#66): `media_duration` followed the chain
/// without a bound and overflowed the stack. It gives up after a few hops now.
#[test]
@ -988,3 +1037,25 @@ fn keyboard_slip_keeps_linked_clips_in_sync_past_the_shorter_handle() {
s.execute("timeline.slipLeft5", json!({})).unwrap();
assert_eq!((clip(&s, v).source_in, clip(&s, a).source_in), (Tick(fr.0 * 30), Tick(fr.0 * 35)));
}
#[test]
fn linked_rate_stretch_keeps_one_speed_when_a_neighbour_blocks_the_sound() {
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Stretch"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 1, "width": 16, "height": 16})).unwrap();
let r = s.execute("file.newOfflineFile", json!({"name": "A", "seconds": 8, "fps": 25, "video": true, "audio": true})).unwrap();
let a = r["item"].as_u64().unwrap();
let r = s.execute("file.newOfflineFile", json!({"name": "B", "seconds": 2, "fps": 25, "video": true, "audio": true})).unwrap();
let b = r["item"].as_u64().unwrap();
let fr = s.sequence_rate().tick_of(1);
let r = s.execute("timeline.place", json!({"item": a, "track": "V1", "audioTrack": "A1", "time": 0, "duration": fr.0 * 60})).unwrap();
let (v, snd) = (r["clips"][0].as_u64().unwrap(), r["clips"][1].as_u64().unwrap());
s.execute("sequence.linkedSelection", json!({"on": false})).unwrap();
s.execute("timeline.place", json!({"item": b, "track": "A1", "time": fr.0 * 70, "duration": fr.0 * 10})).unwrap();
s.execute("sequence.linkedSelection", json!({"on": true})).unwrap();
let r = s.execute("timeline.rateStretch", json!({"clip": v, "edge": "out", "deltaFrames": 20})).unwrap();
let (pic, sound) = (clip(&s, v), clip(&s, snd));
assert_eq!((pic.duration, sound.duration), (Tick(fr.0 * 70), Tick(fr.0 * 70)));
assert!((pic.speed - sound.speed).abs() < 1e-12);
assert!((r["speed"].as_f64().unwrap() - 60.0 / 70.0).abs() < 1e-9);
}

585
crates/engine/src/commands.rs
View File

@ -185,6 +185,18 @@ pub(crate) fn named_items(s: &Session, spec: &CommandSpec, p: &Value) -> Option<
pub(crate) fn bad(cmd: &str, msg: impl Into<String>) -> EngineError {
EngineError::BadParams { cmd: cmd.into(), msg: msg.into() }
}
/// A marker's `color` param: absent (or null) is `None`; anything but a label name is an error
/// that lists the accepted names, instead of silently becoming Green (#857).
pub(crate) fn marker_color_p(p: &Value, cmd: &str) -> Result<Option<Label>> {
let v = match p.get("color") {
None | Some(Value::Null) => return Ok(None),
Some(v) => v,
};
v.as_str().and_then(Label::from_name).map(Some).ok_or_else(|| {
let names: Vec<&str> = Label::ALL.iter().map(|l| l.name()).collect();
bad(cmd, format!("unknown marker color {v}; use one of: {}", names.join(", ")))
})
}
pub(crate) fn str_p<'a>(p: &'a Value, k: &str) -> Option<&'a str> {
p.get(k).and_then(Value::as_str)
}
@ -221,6 +233,12 @@ pub(crate) fn checked_u32_p(p: &Value, key: &str, cmd: &str) -> Result<Option<u3
/// Parse a time from params: `time` (ticks), `frame`, `seconds` or `timecode`, with `prefix`.
pub(crate) fn time_p(s: &Session, p: &Value, prefix: &str) -> Option<Tick> {
let rate = s.sequence_rate();
let df = s.active_sequence().map(|q| q.settings.drop_frame).unwrap_or(false);
time_p_at(p, prefix, rate, df, s.playhead())
}
/// Same formats as [`time_p`], against an explicit rate (Source monitor, remix duration, …).
pub(crate) fn time_p_at(p: &Value, prefix: &str, rate: FrameRate, drop_frame: bool, current: Tick) -> Option<Tick> {
let k = |n: &str| if prefix.is_empty() { n.to_string() } else { format!("{prefix}{}{}", n[..1].to_uppercase(), &n[1..]) };
if let Some(t) = p.get(k("time")).and_then(Value::as_i64) {
return Some(Tick(t));
@ -232,13 +250,28 @@ pub(crate) fn time_p(s: &Session, p: &Value, prefix: &str) -> Option<Tick> {
return Some(Tick::from_seconds_f64(sec));
}
if let Some(tc) = p.get(k("timecode")).and_then(Value::as_str) {
let df = s.active_sequence().map(|q| q.settings.drop_frame).unwrap_or(false);
let cur = rate.frame_at(s.playhead());
return parse_timecode(tc, rate, df, cur).ok().map(|f| rate.tick_of(f));
return parse_timecode(tc, rate, drop_frame, rate.frame_at(current)).ok().map(|f| rate.tick_of(f));
}
None
}
/// In/Out / split time on the Source monitor: explicit `time`/`frame`/`seconds`/`timecode`, else the Source playhead.
pub(crate) fn source_mark_time(s: &Session, p: &Value, item: ItemId) -> Tick {
let view = crate::clip_ops::source_view(s, item);
let rate = view.as_ref().map(|v| v.rate).unwrap_or_else(|| s.project.item(item).map(|i| i.frame_rate()).unwrap_or_else(|| s.sequence_rate()));
let df = s.project.sequence(item).map(|q| q.settings.drop_frame).unwrap_or(false);
time_p_at(p, "", rate, df, s.state.source_playhead).unwrap_or(s.state.source_playhead)
}
/// Sequence time from params. Times before 0 are unreachable in the GUI and cut off on export.
pub(crate) fn sequence_time_p(s: &Session, p: &Value, prefix: &str, cmd: &str) -> Result<Option<Tick>> {
let Some(t) = time_p(s, p, prefix) else { return Ok(None) };
if t < Tick::ZERO {
return Err(bad(cmd, "time must not be negative"));
}
Ok(Some(t))
}
pub(crate) fn clip_p(p: &Value, k: &str) -> Option<ClipId> {
u64_p(p, k).map(ClipId)
}
@ -397,20 +430,104 @@ fn move_clips(s: &mut Session, p: &Value) -> Result<Value> {
/// most constrained clip (media end or start) still moves, and no linked partner slips further.
pub(crate) fn slip_together(q: &mut filmcraft_project::Sequence, ctx: &mut edit::EditCtx<'_>, deltas: &[(ClipId, Tick)]) -> Result<()> {
let mut scale = 1.0_f64;
let mut any = false;
for (c, d) in deltas {
if d.0 == 0 {
continue;
}
let x = edit::slip(&mut q.clone(), *c, *d, ctx)?;
scale = scale.min(x.0.abs() as f64 / d.0.abs() as f64);
match edit::slip(&mut q.clone(), *c, *d, ctx) {
Err(edit::EditError::Locked) => continue,
Err(e) => return Err(e.into()),
Ok(x) => {
any = true;
scale = scale.min(x.0.abs() as f64 / d.0.abs() as f64);
}
}
}
if !any && deltas.iter().any(|(_, d)| d.0 != 0) {
return Err(edit::EditError::Locked.into());
}
for (c, d) in deltas {
let dd = if scale < 1.0 { Tick((d.0 as f64 * scale).round() as i64) } else { *d };
edit::slip(q, *c, dd, ctx)?;
match edit::slip(q, *c, dd, ctx) {
Err(edit::EditError::Locked) => continue,
Err(e) => return Err(e.into()),
Ok(_) => {}
}
}
Ok(())
}
/// Clips an audio effect should land on: audio ids as given, plus the linked audio of any video
/// clip in the list (so applying Parametric EQ to an A/V pair hits the sound).
pub fn audio_effect_targets(s: &Session, clips: &[ClipId]) -> Vec<ClipId> {
let Some(seq) = s.active_sequence() else { return clips.to_vec() };
let mut out = Vec::new();
for c in clips {
let Some((tid, it)) = seq.find_item(*c) else { continue };
let kind = seq.track(tid).map(|t| t.kind);
if kind == Some(TrackKind::Audio) {
if !out.contains(c) {
out.push(*c);
}
continue;
}
if kind != Some(TrackKind::Video) {
continue;
}
let Some(link) = it.link else { continue };
for t in &seq.audio_tracks {
for i in &t.items {
if i.link == Some(link) && !out.contains(&i.id) {
out.push(i.id);
}
}
}
}
out
}
/// Adjacent roll pairs that follow `left`/`right` when linked selection is on: the given cut, plus
/// through-edits between the same pieces' linked partners at that same time (Premiere linked
/// selection). Split edits (audio cut at another time) and unlinked clips are not added.
pub fn linked_roll_pairs(seq: &filmcraft_project::Sequence, left: ClipId, right: ClipId, follow_links: bool) -> Vec<(ClipId, ClipId)> {
let mut pairs = vec![(left, right)];
if !follow_links {
return pairs;
}
let Some((_, l)) = seq.find_item(left) else {
return pairs;
};
let Some((_, r)) = seq.find_item(right) else {
return pairs;
};
let cut = l.end();
let left_link = l.link;
let right_link = r.link;
if left_link.is_none() && right_link.is_none() {
return pairs;
}
for tr in seq.all_tracks() {
if tr.locked {
continue;
}
for w in tr.items.windows(2) {
if w[0].id == left && w[1].id == right {
continue;
}
if w[0].end() != w[1].start || w[0].end() != cut {
continue;
}
let left_ok = left_link.is_some_and(|g| w[0].link == Some(g));
let right_ok = right_link.is_some_and(|g| w[1].link == Some(g));
if left_ok && right_ok {
pairs.push((w[0].id, w[1].id));
}
}
}
pairs
}
/// Expand a clip selection with linked partners (when linked selection is on).
pub fn with_links(s: &Session, clips: &[ClipId]) -> Vec<ClipId> {
let Some(seq) = s.active_sequence() else { return clips.to_vec() };
@ -432,12 +549,22 @@ pub fn with_links(s: &Session, clips: &[ClipId]) -> Vec<ClipId> {
/// clamped by each one's media and neighbours.
fn clamp_trim_linked(q: &filmcraft_project::Sequence, clips: &[ClipId], edge: Edge, mode: TrimMode, delta: Tick, ctx: &edit::EditCtx) -> Result<Tick> {
let mut d = delta;
let mut any = false;
for c in clips {
let x = edit::clamp_trim(q, *c, edge, mode, d, ctx)?;
if x.abs() < d.abs() {
d = x;
match edit::clamp_trim(q, *c, edge, mode, d, ctx) {
Err(edit::EditError::Locked) => continue,
Err(e) => return Err(e.into()),
Ok(x) => {
any = true;
if x.abs() < d.abs() {
d = x;
}
}
}
}
if !any {
return Err(edit::EditError::Locked.into());
}
Ok(d)
}
@ -664,6 +791,126 @@ pub(crate) fn ensure_audio_tracks(p: &mut Project, seq_id: ItemId, first: usize,
Ok(p.sequence(seq_id).map(|q| q.audio_tracks.iter().map(|t| t.id).collect()).unwrap_or_default())
}
/// `timeline.place` for one item (`item`).
fn place_one(s: &mut Session, p: &Value) -> Result<Value> {
if crate::sequence_tools::wants_new_track(p) {
return crate::sequence_tools::place_on_new_tracks(s, p);
}
let stream_flag = |key: &str| match p.get(key) {
None => Ok(true),
Some(Value::Bool(v)) => Ok(*v),
_ => Err(bad("timeline.place", format!("`{key}` must be a boolean"))),
};
let (video, audio) = (stream_flag("video")?, stream_flag("audio")?);
if !video && !audio {
return Err(bad("timeline.place", "enable video, audio, or both"));
}
let item = item_p(p, "item").ok_or_else(|| bad("timeline.place", "need `item`"))?;
let at = sequence_time_p(s, p, "", "timeline.place")?.unwrap_or(s.playhead());
let tg = s.targeting();
let pi = s.project.item(item).ok_or_else(|| bad("timeline.place", "no such item"))?;
let (has_video, has_audio) = s.project.source_streams(item);
let is_audio = |t: TrackId| s.active_sequence().is_some_and(|q| q.audio_tracks.iter().any(|x| x.id == t));
// `track` is where the picture goes and `audioTrack` where the sound goes. A caller that
// names an audio track as `track` asks for the sound alone, on that track.
let (v, a) = match (track_p(s, p, "track", "timeline.place")?, track_p(s, p, "audioTrack", "timeline.place")?) {
(Some(t), Some(_)) if is_audio(t) => {
return Err(bad("timeline.place", "`track` names an audio track and `audioTrack` is given as well: name the sound's track once"));
}
(_, Some(t)) if !is_audio(t) => return Err(bad("timeline.place", "`audioTrack` names a video track")),
(Some(t), None) if is_audio(t) && !has_audio => {
return Err(bad("timeline.place", "`track` names an audio track and this item has no sound"));
}
(Some(t), None) if is_audio(t) => (None, Some(t)),
(Some(_), None) if !has_video => {
return Err(bad("timeline.place", "`track` names a video track and this item has no picture: name an audio track"));
}
(v, a) => (v.or(tg.video_dest), a.or(tg.audio_dest)),
};
let v = v.filter(|_| video);
let a = a.filter(|_| audio);
if p.get("video").is_some() && video && has_video && v.is_none() {
return Err(bad("timeline.place", "no video destination: add or enable a video track"));
}
if p.get("audio").is_some() && audio && has_audio && a.is_none() {
return Err(bad("timeline.place", "no audio destination: add or enable an audio track"));
}
let full = full_duration(s, pi);
let (mi, mo) = match &pi.kind {
ItemKind::Media(m) => (m.mark_in, m.mark_out.map(|o| o + pi.frame_rate().frame_duration())),
ItemKind::Subclip { range, .. } => (Some(range.start), Some(range.end())),
_ => (None, None),
};
let sin = p.get("sourceIn").and_then(Value::as_i64).map(Tick).or(mi).unwrap_or_default();
let dur = p.get("duration").and_then(Value::as_i64).map(Tick).unwrap_or_else(|| mo.unwrap_or(full) - sin);
// The clip goes on the frame grid: its start and its end (`time + duration`) each snap to the
// nearest frame, so a sub-frame `time` never pushes the end past the one asked for and the
// sequence never ends between frames (#1101).
let rate = s.sequence_rate();
let end = rate.snap_nearest(Tick(at.0.saturating_add(dur.0)));
let at = rate.snap_nearest(at);
let dur = end - at;
let ids = place_item(s, item, TimeRange::new(sin, dur), at, v, a, bool_p(p, "insert").unwrap_or(false), "Place Clip", None)?;
Ok(json!({"clips": ids.iter().map(|c| c.0).collect::<Vec<_>>()}))
}
/// Most items one `timeline.place` with `items` takes: a larger list is refused, not cut.
const MAX_PLACE_ITEMS: usize = 10_000;
/// `timeline.place` for `items`: the first at the given time, each next one where the previous one
/// ends, all as one undo step. `track`, `audioTrack`, `insert`, `video` and `audio` apply to every
/// item; `sourceIn` and `duration` name one item's range, so they need `item`.
fn place_items(s: &mut Session, p: &Value, items: &Value) -> Result<Value> {
let ids: Vec<ItemId> = items
.as_array()
.ok_or_else(|| bad("timeline.place", "`items` must be a list of item ids"))?
.iter()
.map(|v| v.as_u64().map(ItemId).ok_or_else(|| bad("timeline.place", "`items` must be a list of item ids")))
.collect::<Result<_>>()?;
if p.get("item").is_some() || p.get("sourceIn").is_some() || p.get("duration").is_some() {
return Err(bad("timeline.place", "give `item` or `items`, and no `sourceIn` or `duration` with `items`"));
}
if ids.is_empty() || ids.len() > MAX_PLACE_ITEMS {
return Err(bad("timeline.place", format!("`items` needs 1 to {MAX_PLACE_ITEMS} item ids")));
}
let mut at = sequence_time_p(s, p, "", "timeline.place")?.unwrap_or(s.playhead());
s.edit_group("Place Clips", |s| {
let mut placed = Vec::new();
// "new" adds a track for the first item only; the rest follow it on that track
let mut base = p.clone();
for item in ids {
let mut one = base.clone();
if let Some(o) = one.as_object_mut() {
o.remove("items");
for k in ["time", "frame", "seconds"] {
o.remove(k);
}
o.insert("item".into(), json!(item.0));
o.insert("time".into(), json!(at.0));
}
let r = place_one(s, &one)?;
let clips: Vec<ClipId> = r["clips"].as_array().map(|a| a.iter().filter_map(|c| c.as_u64().map(ClipId)).collect()).unwrap_or_default();
if crate::sequence_tools::wants_new_track(&base)
&& let Some(q) = s.active_sequence()
{
for (key, audio) in [("track", false), ("audioTrack", true)] {
if base.get(key).and_then(Value::as_str).is_some_and(|v| v.eq_ignore_ascii_case("new"))
&& let Some(t) =
clips.iter().find_map(|c| q.find_item(*c).map(|(t, _)| t).filter(|t| q.audio_tracks.iter().any(|x| x.id == *t) == audio))
{
base[key] = json!(t.0);
}
}
}
if let Some(end) = s.active_sequence().and_then(|q| clips.iter().filter_map(|c| q.find_item(*c).map(|(_, i)| i.end())).max()) {
at = end;
}
placed.push(json!(clips.iter().map(|c| c.0).collect::<Vec<_>>()));
}
Ok(json!({"clips": placed.iter().flat_map(|c| c.as_array().cloned().unwrap_or_default()).collect::<Vec<_>>(), "placed": placed}))
})
}
/// Place a project item on the timeline (drag from Project, or Insert/Overwrite from source).
pub(crate) fn place_item(
s: &mut Session,
@ -681,8 +928,9 @@ pub(crate) fn place_item(
if s.state.sequences_as_clips && pi.as_sequence().is_some_and(|q| q.multicam.is_none()) {
return crate::sequence_tools::place_sequence_clips(s, item, range, at, vdest, adest, insert, label);
}
let has_v = pi.has_video() && vdest.is_some();
let has_a = pi.has_audio() && adest.is_some();
let (video, audio) = s.project.source_streams(item);
let has_v = video && vdest.is_some();
let has_a = audio && adest.is_some();
if !has_v && !has_a {
return Err(EngineError::Other("no destination track for this media (check source patching)".into()));
}
@ -724,7 +972,7 @@ pub(crate) fn place_item(
let mut a = p.make_track_item(item, TrackKind::Audio, aat, arange, rate).ok_or_else(|| bad(label, "bad item"))?;
// Modify ▸ Audio Channels with several audio clips: one per clip, on the tracks below
let clips: Vec<Vec<u16>> =
p.item(item).and_then(|i| i.as_media()).and_then(|m| m.interpret.audio_channels.as_ref()).map(|m| m.clips.clone()).unwrap_or_default();
p.resolve_media(item).and_then(|(_, m, _)| m.interpret.audio_channels.as_ref()).map(|m| m.clips.clone()).unwrap_or_default();
let streams = p.resolve_media(item).map_or(1, |(_, m, _)| m.info.audio_streams.len());
let specs = audio_placement_specs(&clips, streams);
let first = p.sequence(seq_id).and_then(|q| q.audio_tracks.iter().position(|t| t.id == adest)).unwrap_or(0);
@ -864,6 +1112,9 @@ fn build() -> Vec<CommandSpec> {
s.state.active_sequence = Some(seq);
s.state.open_sequences = vec![seq];
s.state.playheads.insert(seq, FrameRate::FPS_23_976.tick_of(4 * 24 + 6));
if s.path.take().is_some() {
s.previews.reset_temp();
}
s.revision += 1;
s.saved_revision = s.revision;
s.events.push(crate::Event::ProjectChanged { revision: s.revision });
@ -875,11 +1126,22 @@ fn build() -> Vec<CommandSpec> {
"Sequence…",
["File", "New"],
Some("Cmd+N"),
r#"{"name":str,"width":u32=1920,"height":u32=1080,"fps":f64=23.976,"sampleRate":u32=48000,"video":n=3,"audio":n=3,"mix":"Stereo|Mono|5.1|Adaptive"?,"trackType":"Standard|Mono|5.1|Adaptive"?,"fromItem":itemId?}"#,
r#"{"name":str,"width":u32=1920,"height":u32=1080,"fps":f64=23.976,"sampleRate":u32=48000,"video":n=3,"audio":n=3,"mix":"Stereo|Mono|5.1|Adaptive"?,"trackType":"Standard|Mono|5.1|Adaptive"?,"fromItem":itemId?,"fromItems":[itemId]?}"#,
always,
|s, p| {
|s, p| s.edit_group("New Sequence", |s| {
// `fromItems` is `fromItem` for a list: the first sets the sequence, the rest follow it back to back
let mut from_ids: Vec<ItemId> = match p.get("fromItems") {
None => Vec::new(),
Some(l) => l
.as_array()
.and_then(|a| a.iter().map(|v| v.as_u64().map(ItemId)).collect())
.ok_or_else(|| bad("file.newSequence", "`fromItems` must be a list of item ids"))?,
};
if let Some(one) = item_p(p, "fromItem") {
from_ids.insert(0, one);
}
let mut st = SequenceSettings::default();
if let Some(from) = item_p(p, "fromItem").and_then(|i| s.project.item(i)).and_then(|i| i.as_media()) {
if let Some(from) = from_ids.first().and_then(|i| s.project.item(*i)).and_then(|i| i.as_media()) {
st = default_seq_settings_for(from);
}
if let Some(w) = checked_u32_p(p, "width", "file.newSequence")? {
@ -929,14 +1191,19 @@ fn build() -> Vec<CommandSpec> {
}
Ok(id)
})?;
if let Some(from) = item_p(p, "fromItem") {
if let Some((&from, rest)) = from_ids.split_first() {
let dur = s.project.item(from).map(|i| i.duration()).unwrap_or_default();
let tg = s.targeting();
place_item(s, from, TimeRange::new(Tick::ZERO, dur), Tick::ZERO, tg.video_dest, tg.audio_dest, false, "New Sequence From Clip", None)?;
let ids =
place_item(s, from, TimeRange::new(Tick::ZERO, dur), Tick::ZERO, tg.video_dest, tg.audio_dest, false, "New Sequence From Clip", None)?;
if !rest.is_empty() {
let end = s.active_sequence().and_then(|q| ids.iter().filter_map(|c| q.find_item(*c).map(|(_, i)| i.end())).max()).unwrap_or_default();
place_items(s, &json!({"time": end.0}), &json!(rest.iter().map(|i| i.0).collect::<Vec<_>>()))?;
}
}
s.events.push(crate::Event::OpenSequence(id));
Ok(json!({"sequence": id.0}))
}
})
),
cmd!("file.newBin", "Bin", ["File", "New"], Some("Cmd+B"), r#"{"name":str,"parent":binId?}"#, always, |s, p| {
let n = str_p(p, "name").unwrap_or("New Bin").to_string();
@ -1285,11 +1552,11 @@ fn build() -> Vec<CommandSpec> {
"Media…",
["File", "Export"],
None,
r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|jpg|tga|dpx|exr|gif|wav|aiff|aac"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass|crf"?,"crf":0..51=23,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq|4444|4444xq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#,
r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|jpg|tga|dpx|exr|gif|wav|aiff|aac|flac"?,"audioCodec":"auto|aac|pcm|flac"?,"flacLevel":0..8=5,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass|crf"?,"crf":0..51=23,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"exrCompression":"none|rle|zips|zip"?,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq|4444|4444xq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#,
has_seq,
crate::export_tools::export_media
),
query!("jobs.list", "List Jobs", "{}", |s, _| Ok(Value::Array(s.jobs.iter().map(crate::Job::to_json).collect()))),
query!("jobs.list", "List Jobs", "{}", |s, _| Ok(Value::Array(s.jobs.iter().map(|j| crate::export_tools::job_json(s, j)).collect()))),
query!("perf.stats", "Performance Statistics", "{}", |s, _| Ok(crate::perf::stats(s))),
cmd!("jobs.cancel", "Cancel Job", [], None, r#"{"job":id}"#, always, |s, p| {
let id = u64_p(p, "job").ok_or_else(|| bad("jobs.cancel", "need `job`"))?;
@ -1462,7 +1729,12 @@ fn build() -> Vec<CommandSpec> {
}
}
for group in &groups {
edit::set_speed_group(q, group, speed, reverse, ripple, ctx)?;
let unlocked: Vec<ClipId> =
group.iter().copied().filter(|c| q.find_item(*c).is_some_and(|(t, _)| q.track(t).is_some_and(|tr| !tr.locked))).collect();
if unlocked.is_empty() {
return Err(edit::EditError::Locked.into());
}
edit::set_speed_group(q, &unlocked, speed, reverse, ripple, ctx)?;
}
for c in &sel {
if let (Some(m), Some((_, it))) = (interp, q.find_item_mut(*c)) {
@ -1740,7 +2012,7 @@ fn build() -> Vec<CommandSpec> {
"Apply Audio Transition",
["Sequence"],
Some("Cmd+Shift+D"),
r#"{"clip":id?,"effect":str?,"frames":i64?}"#,
r#"{"clip":id?,"effect":"constant_power|Constant Power|…"?,"frames":i64?,"edge":"in"|"out"?,"params":{param:value}?,"reverse":bool?}"#,
has_seq,
|s, p| apply_transition(s, p, TrackKind::Audio)
),
@ -1897,8 +2169,24 @@ fn build() -> Vec<CommandSpec> {
Ok(json!({"item": item.0}))
}),
// ================= Markers =================
cmd!("markers.markIn", "Mark In", ["Markers"], Some("I"), r#"{"time":ticks?,"target":"program|source"}"#, always, |s, p| mark(s, p, true)),
cmd!("markers.markOut", "Mark Out", ["Markers"], Some("O"), r#"{"time":ticks?,"target":"program|source"}"#, always, |s, p| mark(s, p, false)),
cmd!(
"markers.markIn",
"Mark In",
["Markers"],
Some("I"),
r#"{"time":ticks?,"frame":i64?,"seconds":f64?,"timecode":str?,"target":"program|source"}"#,
always,
|s, p| mark(s, p, true)
),
cmd!(
"markers.markOut",
"Mark Out",
["Markers"],
Some("O"),
r#"{"time":ticks?,"frame":i64?,"seconds":f64?,"timecode":str?,"target":"program|source"}"#,
always,
|s, p| mark(s, p, false)
),
cmd!("markers.markClip", "Mark Clip", ["Markers"], Some("X"), "{}", has_seq, |s, _| {
let t = s.playhead();
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
@ -1999,10 +2287,10 @@ fn build() -> Vec<CommandSpec> {
return Ok(v);
}
let (_seq, ph) = marker_list_mut(s)?;
let t = time_p(s, p, "").unwrap_or(ph);
let t = sequence_time_p(s, p, "", "markers.add")?.unwrap_or(ph);
let name = str_p(p, "name").unwrap_or("").to_string();
let comment = str_p(p, "comment").unwrap_or("").to_string();
let color = str_p(p, "color").and_then(Label::from_name).unwrap_or(Label::Green);
let color = marker_color_p(p, "markers.add")?.unwrap_or(Label::Green);
let dur = p.get("durationFrames").and_then(Value::as_i64).map(|f| s.sequence_rate().tick_of(f)).unwrap_or_default();
let id = s.edit_sequence("Add Marker", |q, ctx, _| {
let id = MarkerId(ctx.alloc());
@ -2078,6 +2366,7 @@ fn build() -> Vec<CommandSpec> {
}
let id = u64_p(p, "marker").map(MarkerId).ok_or_else(|| bad("markers.edit", "need `marker`"))?;
let rate = s.sequence_rate();
let color = marker_color_p(p, "markers.edit")?;
let p = p.clone();
s.edit_sequence("Edit Marker", |q, _, _| {
let m = q.markers.iter_mut().find(|m| m.id == id).ok_or_else(|| bad("markers.edit", "no such marker"))?;
@ -2087,7 +2376,7 @@ fn build() -> Vec<CommandSpec> {
if let Some(c) = str_p(&p, "comment") {
m.comment = c.into();
}
if let Some(c) = str_p(&p, "color").and_then(Label::from_name) {
if let Some(c) = color {
m.color = c;
}
if let Some(f) = p.get("durationFrames").and_then(Value::as_i64) {
@ -2184,60 +2473,11 @@ fn build() -> Vec<CommandSpec> {
"Place Clip",
[],
None,
r#"{"item":id,"track":"V1"|id|"A1" (sound only)|"new" (a video track after the last)?,"audioTrack":"A1"|id|"new"?,"time":ticks|"frame":i64|"seconds":f64,"insert":bool,"sourceIn":ticks?,"duration":ticks?,"video":bool=true,"audio":bool=true}"#,
r#"{"item":id|"items":[id] (back to back, one undo step),"track":"V1"|id|"A1" (sound only)|"new" (a video track after the last)?,"audioTrack":"A1"|id|"new"?,"time":ticks|"frame":i64|"seconds":f64,"insert":bool,"sourceIn":ticks?,"duration":ticks?,"video":bool=true,"audio":bool=true}"#,
has_seq,
|s, p| {
if crate::sequence_tools::wants_new_track(p) {
return crate::sequence_tools::place_on_new_tracks(s, p);
}
let stream_flag = |key: &str| match p.get(key) {
None => Ok(true),
Some(Value::Bool(v)) => Ok(*v),
_ => Err(bad("timeline.place", format!("`{key}` must be a boolean"))),
};
let (video, audio) = (stream_flag("video")?, stream_flag("audio")?);
if !video && !audio {
return Err(bad("timeline.place", "enable video, audio, or both"));
}
let item = item_p(p, "item").ok_or_else(|| bad("timeline.place", "need `item`"))?;
let at = time_p(s, p, "").unwrap_or(s.playhead());
let tg = s.targeting();
let pi = s.project.item(item).ok_or_else(|| bad("timeline.place", "no such item"))?;
let is_audio = |t: TrackId| s.active_sequence().is_some_and(|q| q.audio_tracks.iter().any(|x| x.id == t));
// `track` is where the picture goes and `audioTrack` where the sound goes. A caller that
// names an audio track as `track` asks for the sound alone, on that track.
let (v, a) = match (track_p(s, p, "track", "timeline.place")?, track_p(s, p, "audioTrack", "timeline.place")?) {
(Some(t), Some(_)) if is_audio(t) => {
return Err(bad("timeline.place", "`track` names an audio track and `audioTrack` is given as well: name the sound's track once"));
}
(_, Some(t)) if !is_audio(t) => return Err(bad("timeline.place", "`audioTrack` names a video track")),
(Some(t), None) if is_audio(t) && !pi.has_audio() => {
return Err(bad("timeline.place", "`track` names an audio track and this item has no sound"));
}
(Some(t), None) if is_audio(t) => (None, Some(t)),
(Some(_), None) if !pi.has_video() => {
return Err(bad("timeline.place", "`track` names a video track and this item has no picture: name an audio track"));
}
(v, a) => (v.or(tg.video_dest), a.or(tg.audio_dest)),
};
let v = v.filter(|_| video);
let a = a.filter(|_| audio);
if p.get("video").is_some() && video && pi.has_video() && v.is_none() {
return Err(bad("timeline.place", "no video destination: add or enable a video track"));
}
if p.get("audio").is_some() && audio && pi.has_audio() && a.is_none() {
return Err(bad("timeline.place", "no audio destination: add or enable an audio track"));
}
let full = full_duration(s, pi);
let (mi, mo) = match &pi.kind {
ItemKind::Media(m) => (m.mark_in, m.mark_out.map(|o| o + pi.frame_rate().frame_duration())),
ItemKind::Subclip { range, .. } => (Some(range.start), Some(range.end())),
_ => (None, None),
};
let sin = p.get("sourceIn").and_then(Value::as_i64).map(Tick).or(mi).unwrap_or_default();
let dur = p.get("duration").and_then(Value::as_i64).map(Tick).unwrap_or_else(|| mo.unwrap_or(full) - sin);
let ids = place_item(s, item, TimeRange::new(sin, dur), at, v, a, bool_p(p, "insert").unwrap_or(false), "Place Clip", None)?;
Ok(json!({"clips": ids.iter().map(|c| c.0).collect::<Vec<_>>()}))
|s, p| match p.get("items") {
Some(items) => place_items(s, p, items),
None => place_one(s, p),
}
),
cmd!("timeline.select", "Select Clips", [], None, r#"{"clips":[id],"transitions":[id]?,"add":bool,"toggle":bool}"#, has_seq, |s, p| {
@ -2279,9 +2519,14 @@ fn build() -> Vec<CommandSpec> {
"Move Clips",
[],
None,
r#"{"moves":[{"clip":id,"track":id|"V2","time":ticks}],"insert":bool,"linked":bool?,"copy":bool?}"#,
r#"{"moves":[{"clip":id,"track":id|"V2"|"new" (after the last of its kind),"time":ticks}],"insert":bool,"linked":bool?,"copy":bool?}"#,
has_seq,
move_clips
|s, p| {
if crate::sequence_tools::move_wants_new_track(p) {
return crate::sequence_tools::move_to_new_tracks(s, p);
}
move_clips(s, p)
}
),
cmd!(
"timeline.trim",
@ -2302,6 +2547,9 @@ fn build() -> Vec<CommandSpec> {
.unwrap_or_default();
let clips = with_links(s, &[c]);
let applied = s.edit_sequence(if mode == TrimMode::Ripple { "Ripple Trim" } else { "Trim" }, |q, ctx, st| {
if q.find_item(c).is_some_and(|(t, _)| q.track(t).is_some_and(|tr| tr.locked)) {
return Err(edit::EditError::Locked.into());
}
let before = q.find_item(c).map(|(_, i)| i.range());
// clamp across all linked partners, then apply the common delta
let mut dd = clamp_trim_linked(q, &clips, edge, mode, d, ctx)?;
@ -2309,6 +2557,10 @@ fn build() -> Vec<CommandSpec> {
// the clip and its linked partners ripple as one edit
let mut group = vec![c];
group.extend(clips.iter().copied().filter(|o| *o != c));
group.retain(|id| q.find_item(*id).is_some_and(|(t, _)| q.track(t).is_some_and(|tr| !tr.locked)));
if group.is_empty() {
return Err(edit::EditError::Locked.into());
}
dd = edit::ripple_trim_group(q, &group, edge, dd, ctx)?;
if st.ripple_sequence_markers
&& let Some(r) = before
@ -2322,7 +2574,11 @@ fn build() -> Vec<CommandSpec> {
}
} else {
for c in &clips {
edit::trim(q, *c, edge, mode, dd, ctx)?;
match edit::trim(q, *c, edge, mode, dd, ctx) {
Err(edit::EditError::Locked) => continue,
Err(e) => return Err(e.into()),
Ok(_) => {}
}
}
}
Ok(dd)
@ -2401,7 +2657,7 @@ fn build() -> Vec<CommandSpec> {
s,
&json!({"side": "next", "ripple": false})
)),
cmd!("timeline.roll", "Rolling Edit", [], None, r#"{"left":id,"right":id,"delta":ticks|"deltaFrames":i64}"#, has_seq, |s, p| {
cmd!("timeline.roll", "Rolling Edit", [], None, r#"{"left":id,"right":id,"delta":ticks|"deltaFrames":i64,"followLinks"?:bool}"#, has_seq, |s, p| {
let (l, r) = (
clip_p(p, "left").ok_or_else(|| bad("timeline.roll", "need `left`"))?,
clip_p(p, "right").ok_or_else(|| bad("timeline.roll", "need `right`"))?,
@ -2412,8 +2668,27 @@ fn build() -> Vec<CommandSpec> {
.map(Tick)
.or_else(|| p.get("deltaFrames").and_then(Value::as_i64).map(|f| s.sequence_rate().tick_of(f)))
.unwrap_or_default();
let x = s.edit_sequence("Rolling Edit", |q, ctx, _| Ok(edit::roll(q, l, r, d, ctx)?))?;
Ok(json!({"delta": x.0}))
// `followLinks: false` rolls only this pair: Trim mode already lists linked partners as edit points of their own
let follow = s.state.linked_selection && p.get("followLinks").and_then(Value::as_bool).unwrap_or(true);
let (x, n, linked) = s.edit_sequence("Rolling Edit", |q, ctx, _| {
let pairs = linked_roll_pairs(q, l, r, follow);
let linked = pairs.len() > 1;
let mut d = d;
for &(pl, pr) in &pairs {
let mut probe = q.clone();
let x = edit::roll(&mut probe, pl, pr, d, ctx)?;
if x.abs() < d.abs() {
d = x;
}
}
if d != Tick::ZERO {
for &(pl, pr) in &pairs {
edit::roll(q, pl, pr, d, ctx)?;
}
}
Ok((d, pairs.len(), linked))
})?;
Ok(json!({"delta": x.0, "pairs": n, "linked": linked}))
}),
cmd!("timeline.slip", "Slip", [], None, r#"{"clip":id,"delta":ticks|"deltaFrames":i64}"#, has_seq, |s, p| {
let c = clip_p(p, "clip").ok_or_else(|| bad("timeline.slip", "need `clip`"))?;
@ -2441,7 +2716,14 @@ fn build() -> Vec<CommandSpec> {
.unwrap_or_default();
let clips = with_links(s, &[c]);
// clamp across all linked partners, then slide them by the common delta
s.edit_sequence("Slide", |q, ctx, _| Ok(edit::slide_items(q, &clips, d, ctx)?))?;
s.edit_sequence("Slide", |q, ctx, _| {
let unlocked: Vec<ClipId> =
clips.iter().copied().filter(|id| q.find_item(*id).is_some_and(|(t, _)| q.track(t).is_some_and(|tr| !tr.locked))).collect();
if unlocked.is_empty() {
return Err(edit::EditError::Locked.into());
}
Ok(edit::slide_items(q, &unlocked, d, ctx)?)
})?;
Ok(Value::Null)
}),
cmd!("timeline.rateStretch", "Rate Stretch", [], None, r#"{"clip":id,"edge":"in|out","delta":ticks}"#, has_seq, |s, p| {
@ -2454,13 +2736,7 @@ fn build() -> Vec<CommandSpec> {
.or_else(|| p.get("deltaFrames").and_then(Value::as_i64).map(|f| s.sequence_rate().tick_of(f)))
.unwrap_or_default();
let clips = with_links(s, &[c]);
let sp = s.edit_sequence("Rate Stretch", |q, ctx, _| {
let mut sp = 1.0;
for c in &clips {
sp = edit::rate_stretch(q, *c, edge, d, ctx)?;
}
Ok(sp)
})?;
let sp = s.edit_sequence("Rate Stretch", |q, ctx, _| Ok(edit::rate_stretch_items(q, &clips, edge, d, ctx)?))?;
Ok(json!({"speed": sp}))
}),
cmd!("timeline.razor", "Razor", [], None, r#"{"time":ticks,"track":"V1"|id?,"clip":id?}"#, has_seq, |s, p| {
@ -2558,6 +2834,12 @@ fn build() -> Vec<CommandSpec> {
return Err(bad("effects.apply", "select clips first"));
}
let is_audio = def.kind == filmcraft_project::EffectKind::Audio;
// Audio effects belong on audio clips. A video clip in an A/V pair is retargeted to
// its linked audio so double-click / drop on the picture still applies Parametric EQ.
let clips = if is_audio { audio_effect_targets(s, &clips) } else { clips };
if clips.is_empty() {
return Err(bad("effects.apply", "audio effects need an audio clip (or a video clip with linked audio)"));
}
let (fw, fh) = s.active_sequence().map(|q| (q.settings.width, q.settings.height)).unwrap_or((1920, 1080));
let label = format!("Apply {}", def.name);
s.edit_sequence(&label, |q, _, _| {
@ -2583,6 +2865,16 @@ fn build() -> Vec<CommandSpec> {
cmd!("effects.remove", "Remove Effect", [], None, r#"{"clip":id,"index":n}"#, has_seq, |s, p| {
let c = clip_p(p, "clip").ok_or_else(|| bad("effects.remove", "need `clip`"))?;
let idx = u64_p(p, "index").ok_or_else(|| bad("effects.remove", "need `index`"))? as usize;
// the selected effect: where it is, and its selection once the effect at `idx` is gone
let effects = s.active_sequence().and_then(|q| q.find_item(c)).map(|(_, it)| it.effects.clone()).unwrap_or_default();
let sel = s.state.selected_effect.clone().filter(|e| e.clip == c).and_then(|e| e.index(&effects));
let after = sel.filter(|i| *i != idx).and_then(|i| {
let mut rest = effects;
if idx < rest.len() {
rest.remove(idx);
}
crate::EffectSel::of(c, &rest, if i > idx { i - 1 } else { i })
});
s.edit_sequence("Remove Effect", |q, _, _| {
let (_, it) = q.find_item_mut(c).ok_or(filmcraft_edit::EditError::NoItem(c))?;
if it.effects.get(idx).and_then(|e| e.def()).is_some_and(|d| d.intrinsic) {
@ -2593,14 +2885,27 @@ fn build() -> Vec<CommandSpec> {
}
Ok(())
})?;
if sel.is_some() {
s.state.selected_effect = after;
}
Ok(Value::Null)
}),
CommandSpec {
id: "effects.select",
label: "Select Effect",
menu: &[],
shortcut: None,
params: r#"{"clip":id,"effect":index|id}|{"none":true}"#,
enabled: has_seq,
run: select_effect,
journal: false,
},
cmd!(
"effects.setParam",
"Set Effect Parameter",
[],
None,
r##"{"clip":id,"effect":"motion"|index,"param":str,"mask":n?,"value":num|[x,y]|"#rrggbb"|bool|path|"option name" (a choice takes its index or its name),"time":ticks? (where the keyframe goes when the parameter is animated; a parameter that is not animated gets a new static value whatever the time),"keyframe":bool? (true: write a keyframe at `time` and animate the parameter),"merge":bool?,"begin":bool?}"##,
r##"{"clip":id,"effect":"motion"|index,"param":str,"mask":n?,"value":num|[x,y]|"#rrggbb"|bool|path|"option name" (a choice takes its index or its name),"time":ticks? (where the keyframe goes when the parameter is animated, kept there even outside the clip; default: the playhead, pinned to the clip; a parameter that is not animated gets a new static value whatever the time),"keyframe":bool? (true: write a keyframe at `time` and animate the parameter),"merge":bool|key?,"begin":bool?}"##,
has_seq,
|s, p| {
let c = clip_p(p, "clip").ok_or_else(|| bad("effects.setParam", "need `clip`"))?;
@ -2608,17 +2913,23 @@ fn build() -> Vec<CommandSpec> {
let val = p.get("value").cloned().ok_or_else(|| bad("effects.setParam", "need `value`"))?;
let eff = p.get("effect").cloned().unwrap_or(json!("motion"));
let keyframe = bool_p(p, "keyframe").unwrap_or(false);
// an explicit `time` outside the clip keeps its keyframe there, before the start as after
// the end, so a ramp across a cut keeps its slope (#1055); the playhead is pinned to the clip
let at = time_p(s, p, "");
let ph = s.playhead();
let tl = time_p(s, p, "").unwrap_or(ph);
let pq = p.clone();
// `merge`: a drag of one parameter is one undo step (#201); `begin` starts a new one
let merge = bool_p(p, "merge").unwrap_or(false).then(|| format!("setParam:{}:{eff}:{pid}:{}", c.0, p.get("mask").unwrap_or(&Value::Null)));
// `merge`: a drag of one parameter is one undo step (#201), and a key makes one of a
// drag that changes several (the Program monitor's handles); `begin` starts a new one
let merge = match p.get("merge") {
Some(Value::String(key)) => Some(format!("setParam:{}:{key}", c.0)),
_ => bool_p(p, "merge").unwrap_or(false).then(|| format!("setParam:{}:{eff}:{pid}:{}", c.0, p.get("mask").unwrap_or(&Value::Null))),
};
if bool_p(p, "begin").unwrap_or(false) {
s.history.merge_key = None;
}
let keyframes = s.edit_sequence_as("Change Effect Parameter", merge.as_deref(), |q, _, _| {
let (_, it) = q.find_item_mut(c).ok_or(filmcraft_edit::EditError::NoItem(c))?;
let mt = it.source_time_at(tl.clamp(it.start, (it.end() - Tick(1)).max(it.start)));
let mt = it.source_time_at(at.unwrap_or_else(|| ph.clamp(it.start, (it.end() - Tick(1)).max(it.start))));
let e = match &eff {
Value::Number(n) => it.effects.get_mut(n.as_u64().unwrap_or(0) as usize),
Value::String(sid) => it.effects.iter_mut().find(|e| &e.effect == sid),
@ -2772,6 +3083,15 @@ fn build() -> Vec<CommandSpec> {
has_seq,
|s, p| keyframe_op(s, p, "move")
),
cmd!(
"effects.moveKeyframes",
"Move Keyframes",
[],
None,
r#"{"keyframes":[{"clip":id,"effect":index,"param":str,"mask":n?,"time":mediaTicks}],"delta":timelineTicks}"#,
has_seq,
crate::keyframe_move::move_many
),
cmd!(
"effects.setInterpolation",
"Keyframe Interpolation",
@ -2820,20 +3140,48 @@ fn build() -> Vec<CommandSpec> {
}
items.sort_by_key(|i| i.0);
items.dedup();
let mut used = std::collections::HashSet::new();
for it in s.project.sequences() {
if let Some(q) = it.as_sequence() {
for t in q.all_tracks() {
used.extend(t.items.iter().map(|x| x.item));
}
}
}
let mut gone = Vec::new();
let mut offlined = Vec::new();
for i in items {
let keep = used.contains(&i)
&& s.project.item(i).and_then(filmcraft_project::ProjectItem::as_media).is_some_and(|m| matches!(m.media, MediaRef::File { .. }));
if keep {
offlined.push(i);
} else {
gone.push(i);
}
}
s.edit("Clear", |pr, st| {
for i in &offlined {
if let Some(m) = pr.item_mut(*i).and_then(filmcraft_project::ProjectItem::as_media_mut) {
m.offline = true;
}
}
for b in &bins {
pr.root.remove_bin(*b);
}
for i in &items {
for i in &offlined {
if pr.root.parent_of(*i).is_none() {
pr.root.children.push(filmcraft_project::BinEntry::Item(*i));
}
}
for i in &gone {
pr.items.remove(i);
pr.root.remove_item(*i);
}
// remove track items that referenced deleted media
let seq_ids: Vec<ItemId> = pr.sequences().map(|q| q.id).collect();
for sid in seq_ids {
if let Some(q) = pr.sequence_mut(sid) {
for t in q.all_tracks_mut() {
t.items.retain(|x| !items.contains(&x.item));
t.items.retain(|x| !gone.contains(&x.item));
filmcraft_edit::remove_orphan_transitions(t);
}
}
@ -2841,11 +3189,12 @@ fn build() -> Vec<CommandSpec> {
st.project_selection.clear();
Ok(())
})?;
for i in &items {
for i in gone.iter().chain(&offlined) {
s.media.remove(*i);
}
crate::relink::refresh(s);
s.fix_state();
Ok(json!({"items": items.len(), "bins": bins.len()}))
Ok(json!({"items": gone.len(), "bins": bins.len(), "offline": offlined.len()}))
}),
cmd!("project.moveToBin", "Move to Bin", [], None, r#"{"items":[id|binId]?,"bin":binId|null}"#, always, |s, p| {
let items: Vec<ItemId> = match p.get("items").and_then(Value::as_array) {
@ -3088,7 +3437,7 @@ fn mark(s: &mut Session, p: &Value, is_in: bool) -> Result<Value> {
let target_source = str_p(p, "target") == Some("source");
if target_source {
let item = s.state.source_item.ok_or_else(|| EngineError::Other("no source clip".into()))?;
let t = p.get("time").and_then(Value::as_i64).map(Tick).unwrap_or(s.state.source_playhead);
let t = source_mark_time(s, p, item);
let key = if is_in { "in" } else { "out" };
return s.execute("project.setMarks", json!({"item": item.0, key: t.0}));
}
@ -3464,6 +3813,28 @@ fn transition_ids_p(v: &Value) -> Option<Vec<TransitionId>> {
}
}
/// `effects.select`: the effect selected in Effect Controls (`effect`: index, or the id of the
/// clip's first effect of that kind). It replaces the mask selection.
fn select_effect(s: &mut Session, p: &Value) -> Result<Value> {
const CMD: &str = "effects.select";
if bool_p(p, "none") == Some(true) {
s.state.selected_effect = None;
return Ok(Value::Null);
}
let clip = clip_p(p, "clip").ok_or_else(|| bad(CMD, "need `clip` (or `none`)"))?;
let q = s.active_sequence().ok_or(EngineError::NoSequence)?;
let (_, it) = q.find_item(clip).ok_or_else(|| bad(CMD, "no such clip"))?;
let effect = match p.get("effect") {
Some(Value::Number(n)) => n.as_u64().and_then(|i| usize::try_from(i).ok()).filter(|i| *i < it.effects.len()),
Some(Value::String(id)) => it.effects.iter().position(|e| &e.effect == id),
_ => None,
}
.ok_or_else(|| bad(CMD, "no such effect on clip"))?;
s.state.selected_effect = crate::EffectSel::of(clip, &it.effects, effect);
s.state.selected_mask = None;
Ok(Value::Null)
}
/// `timeline.select {"transitions": […]}`: select transitions (clicking one in the Timeline). It
/// replaces the clip, edit point and caption selections, as a clip selection replaces it.
fn select_transitions(s: &mut Session, ids: &Value, add: bool, toggle: bool) -> Result<Value> {
@ -3775,12 +4146,14 @@ pub fn inspect_sequence(s: &Session, id: ItemId, q: &filmcraft_project::Sequence
"transitions": t.transitions.iter().map(|x| json!({"id": x.id.0, "effect": x.effect.effect, "start": x.start.0, "duration": x.duration.0, "from": x.from.map(|c| c.0), "to": x.to.map(|c| c.0), "reverse": x.reverse, "align": filmcraft_edit::transitions::bounds(t, x).map(|b| filmcraft_edit::transitions::alignment(x, b.1, rate.frame_duration()).id()), "params": x.effect.params.iter().map(|(k, v)| (k.clone(), serde_json::to_value(&v.value).unwrap_or_default())).collect::<serde_json::Map<_, _>>()})).collect::<Vec<_>>(),
})
};
// the frames export renders, a partial last frame included
let (f0, f1) = filmcraft_export::frame_span(rate, TimeRange::new(Tick::ZERO, q.duration()));
json!({
"id": id.0,
"name": s.project.item(id).map(|i| i.name.clone()),
"settings": q.settings,
"duration": q.duration().0,
"durationFrames": rate.frame_at(q.duration()),
"durationFrames": f1 - f0,
"playhead": s.state.playheads.get(&id).map(|t| t.0),
"in": q.mark_in.map(|t| t.0),
"out": q.mark_out.map(|t| t.0),

6
crates/engine/src/explicit_targets_tests.rs
View File

@ -234,15 +234,15 @@ fn project_delete_takes_explicit_bins_and_items_without_a_selection() {
assert_eq!(disabled(s.execute("project.delete", json!({}))), "select an item in the Project panel");
// an empty bin, by its id alone
let r = s.execute("project.delete", json!({"items": [empty]})).unwrap();
assert_eq!(r, json!({"items": 0, "bins": 1}));
assert_eq!(r, json!({"items": 0, "bins": 1, "offline": 0}));
assert!(s.project.root.find_bin(filmcraft_project::BinId(empty)).is_none());
// a bin with a clip in it, and an item, each by id alone
let r = s.execute("project.delete", json!({"items": [outer]})).unwrap();
assert_eq!(r, json!({"items": 1, "bins": 1}));
assert_eq!(r, json!({"items": 1, "bins": 1, "offline": 0}));
assert!(s.project.item(ocean).is_none());
let dunes = item_named(&s, "Desert_Dunes.mp4");
let r = s.execute("project.delete", json!({"items": [dunes.0]})).unwrap();
assert_eq!(r, json!({"items": 1, "bins": 0}));
assert_eq!(r, json!({"items": 1, "bins": 0, "offline": 0}));
assert_eq!(s.project.items.len(), before - 2);
assert!(s.state.project_selection.is_empty(), "the selection is left as it was");
assert!(!s.is_enabled("project.delete"), "menu enablement still follows the selection");

41
crates/engine/src/export_tests.rs
View File

@ -163,10 +163,11 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() {
let Some(j) = probe(&out) else { continue };
let fmt_name = j["format"]["format_name"].as_str().unwrap();
match fmt.as_str() {
"wav" | "aiff" => {
"wav" | "aiff" | "flac" => {
let a = stream(&j, "audio").unwrap();
let bits = settings["audio"]["bits"].as_u64().unwrap();
let codec = match (fmt.as_str(), bits) {
("flac", _) => "flac",
("wav", 24) => "pcm_s24le",
("wav", _) => "pcm_s16le",
(_, 24) => "pcm_s24be",
@ -550,6 +551,21 @@ fn crf_param_picks_constant_quality() {
assert!(e.contains("crf"), "{e}");
}
#[test]
fn exr_compression_param_is_parsed_and_defaults_to_none() {
use filmcraft_export::ExrCompression::{None as Plain, Rle, Zip, Zips};
let s = demo();
let setting = |p: Value| crate::export_tools::settings_from_params(&s, &p, "file.exportMedia").map(|(_, st)| st.exr_compression);
assert_eq!(setting(json!({"path": "x.exr", "format": "exr"})).unwrap(), Plain);
for (name, want) in [("none", Plain), ("rle", Rle), ("zips", Zips), ("zip", Zip)] {
assert_eq!(setting(json!({"path": "x.exr", "format": "exr", "exrCompression": name})).unwrap(), want);
}
let e = setting(json!({"path": "x.exr", "format": "exr", "exrCompression": "piz"})).unwrap_err().to_string();
assert!(e.contains("none | rle | zips | zip"), "{e}");
let old: filmcraft_export::ExportSettings = serde_json::from_value(json!({"format": "exr"})).unwrap();
assert_eq!(old.exr_compression, Plain);
}
#[test]
fn queue_exports_several_sequences_and_ranges() {
let mut s = demo();
@ -667,3 +683,26 @@ fn default_export_dir_never_needs_a_real_filesystem() {
let s = Session { path: Some("/projects/show/edit.fcproj".into()), ..Default::default() };
assert_eq!(crate::export_tools::default_export_dir(&s), std::path::PathBuf::from("/projects/show"));
}
#[test]
fn unknown_settings_keys_are_errors_not_ignored() {
// A flat parameter put inside `settings` used to be dropped without a word: the export ran
// without loudness normalization. A misspelt nested key was dropped the same way.
let s = demo();
let err = crate::export_tools::settings_from_params(&s, &json!({"format": "h264", "settings": {"loudnessLufs": -14}}), "file.exportMedia").unwrap_err();
assert!(err.to_string().contains("`loudnessLufs`"), "{err}");
let err = crate::export_tools::settings_from_params(&s, &json!({"settings": {"effects": {"loudness": {"targetLoudness": -14}}}}), "file.exportMedia")
.unwrap_err();
assert!(err.to_string().contains("`effects.loudness.targetLoudness`"), "{err}");
// Both spellings of a snake_case field are known (#531), and null sets nothing.
for key in ["target_lufs", "targetLufs"] {
let (_, ok) = crate::export_tools::settings_from_params(
&s,
&json!({"settings": {"effects": {"loudness": {"enabled": true, key: -14}}, "captionSidecar": null}}),
"file.exportMedia",
)
.unwrap();
assert!(ok.effects.loudness.enabled);
assert_eq!(ok.effects.loudness.target_lufs, -14.0, "{key}");
}
}

57
crates/engine/src/export_tools.rs
View File

@ -159,6 +159,26 @@ fn merge(base: &mut Value, patch: &Value) {
}
}
/// The first key of `patch` (as a dotted path) that the resolved settings `known` don't have,
/// matched as [`merge`] matches them (the camelCase spelling of a snake_case field counts).
/// A null value is accepted anywhere: it sets nothing.
fn unknown_key(known: &Value, patch: &Value, prefix: &str) -> Option<String> {
let (Value::Object(k), Value::Object(p)) = (known, patch) else { return None };
for (key, v) in p {
let path = if prefix.is_empty() { key.clone() } else { format!("{prefix}.{key}") };
match k.get(key).or_else(|| k.get(&camel_to_snake(key))) {
None if !v.is_null() => return Some(path),
Some(kv) => {
if let Some(found) = unknown_key(kv, v, &path) {
return Some(found);
}
}
None => {}
}
}
None
}
/// A frame rate from a number of frames per second (29.97 → 30000/1001) or "num/den".
pub fn parse_rate(v: &Value) -> Option<FrameRate> {
if let Some(s) = v.as_str() {
@ -191,6 +211,12 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
let mut v = serde_json::to_value(&settings).map_err(|e| EngineError::Other(e.to_string()))?;
merge(&mut v, patch);
settings = serde_json::from_value(v).map_err(|e| bad(cmd, format!("settings: {e}")))?;
// serde ignores keys it doesn't know, so a misspelt or misplaced key (a flat parameter
// such as `loudnessLufs` put inside `settings`) would silently do nothing.
let known = serde_json::to_value(&settings).map_err(|e| EngineError::Other(e.to_string()))?;
if let Some(path) = unknown_key(&known, patch, "") {
return Err(bad(cmd, format!("settings: unknown key `{path}` (see export.presets.get for the settings' keys)")));
}
}
// flat overrides (the original file.exportMedia parameters and a few common ones)
if let Some(f) = str_p(p, "format") {
@ -260,6 +286,12 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
if let Some(v) = str_p(p, "apvProfile") {
settings.apv_profile = v.to_string();
}
if let Some(v) = str_p(p, "audioCodec") {
settings.audio.codec = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "audioCodec: auto | aac | pcm | flac"))?;
}
if let Some(v) = u64_p(p, "flacLevel") {
settings.audio.flac_level = v.min(8) as u8;
}
if let Some(v) = str_p(p, "mxfVideoCodec") {
settings.mxf_video_codec = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "mxfVideoCodec: dnxhr | proRes | h264"))?;
}
@ -269,6 +301,9 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
if let Some(v) = bool_p(p, "alpha") {
settings.alpha = v;
}
if let Some(v) = str_p(p, "exrCompression") {
settings.exr_compression = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "exrCompression: none | rle | zips | zip"))?;
}
match (crate::commands::checked_u32_p(p, "width", cmd)?, crate::commands::checked_u32_p(p, "height", cmd)?) {
(Some(w), Some(h)) => settings.frame_size = Some((w, h)),
(None, None) => {}
@ -440,8 +475,9 @@ pub fn spawn_export(s: &mut Session, project: Arc<Project>, seq: ItemId, mut set
let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1;
let job = crate::Job { id, label, progress: Default::default(), result: Default::default() };
// Export always renders full-resolution media, whatever the proxy toggle says.
let provider = s.media.full_res_provider(project.clone(), s.services.clone());
let provider = s.media.full_res_provider(project.clone(), s.services.clone()).recording();
let (prog, res) = (job.progress.clone(), job.result.clone());
let offline = s.export_offline.entry(id).or_default().clone();
if cfg!(target_arch = "wasm32") && !wait && filmcraft_export::stepped(format) {
// No threads: the host advances the export between UI frames (`Session::pump_jobs`).
let mut exporter = filmcraft_export::Exporter::new(project, seq, &settings, &prog).map_err(|e| EngineError::Other(e.to_string()))?;
@ -452,6 +488,7 @@ pub fn spawn_export(s: &mut Session, project: Arc<Project>, seq: ItemId, mut set
}
let run = move || {
let r = filmcraft_export::export(&project, seq, &settings, &provider, &prog).map_err(|e| e.to_string());
*offline.lock().unwrap_or_else(|x| x.into_inner()) = provider.offline_used();
if let Err(e) = &r {
*prog.error.lock().unwrap_or_else(|x| x.into_inner()) = Some(e.clone());
prog.finished.store(true, Ordering::Relaxed);
@ -514,6 +551,7 @@ pub fn export_media(s: &mut Session, p: &Value) -> Result<Value> {
return Err(EngineError::Other(format!("export failed: {e}")));
}
out["result"] = v["result"].clone();
out["offlineMedia"] = offline_media_json(s, id);
}
Ok(out)
}
@ -568,6 +606,22 @@ impl ExportQueue {
}
}
/// A `jobs.list` entry: [`crate::Job::to_json`], plus `offlineMedia` for an export job, so callers
/// that poll an export (the bridge, MCP long exports) can report it like `file.exportMedia` does.
pub(crate) fn job_json(s: &Session, j: &crate::Job) -> Value {
let mut v = j.to_json();
if s.export_offline.contains_key(&j.id) {
v["offlineMedia"] = offline_media_json(s, j.id);
}
v
}
/// The offline items export job `job` drew the slate for, as JSON (`[]` while it runs or when none).
fn offline_media_json(s: &Session, job: u64) -> Value {
let list = s.export_offline.get(&job).map(|o| o.lock().unwrap_or_else(|e| e.into_inner()).clone()).unwrap_or_default();
serde_json::to_value(list).unwrap_or_else(|_| json!([]))
}
fn item_json(s: &Session, it: &QueueItem) -> Value {
let job = it.job.and_then(|id| s.jobs.iter().find(|j| j.id == id));
let (progress, status_text) = match job {
@ -588,6 +642,7 @@ fn item_json(s: &Session, it: &QueueItem) -> Value {
"statusText": status_text,
"job": it.job,
"error": it.error,
"offlineMedia": it.job.map_or_else(|| json!([]), |j| offline_media_json(s, j)),
})
}

22
crates/engine/src/file_tests.rs
View File

@ -93,3 +93,25 @@ fn project_is_named_after_its_file() {
assert_eq!(s.project.name, "Apollo 11 - Tranquility");
let _ = std::fs::remove_dir_all(&d);
}
/// Opening the demo after a saved project must not keep that path: Save would overwrite it.
#[test]
fn open_demo_project_clears_saved_path_so_save_does_not_overwrite() {
let d = temp_dir("demo-path");
let original = d.join("original.fcproj").to_string_lossy().to_string();
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Original"})).unwrap();
s.execute("file.newBin", json!({"name": "ORIGINAL"})).unwrap();
s.execute("file.saveAs", json!({"path": original})).unwrap();
s.execute("file.openDemoProject", json!({})).unwrap();
assert!(s.path.is_none(), "demo is unsaved, like New Project");
let err = s.execute("file.save", json!({})).unwrap_err().to_string();
assert!(err.contains("no path") || err.contains("Save As"), "{err}");
let mut reopen = Session::default();
reopen.execute("file.open", json!({"path": original})).unwrap();
assert!(
reopen.project.root.children.iter().any(|e| matches!(e, filmcraft_project::BinEntry::Bin(b) if b.name == "ORIGINAL")),
"original.fcproj still has the ORIGINAL bin"
);
let _ = std::fs::remove_dir_all(&d);
}

65
crates/engine/src/image_sequence_tests.rs
View File

@ -138,6 +138,71 @@ fn import_image_sequences_setting_detects_a_numbered_still() {
let _ = std::fs::remove_dir_all(&dir);
}
/// A CinemaDNG frame: 24 × 16 RGGB, 12-bit lossless JPEG behind a preview, a flat grey at `level`
/// (white balanced by the as-shot neutral), 23.976 fps, time code 01:00:00:00 + `frame`.
fn cinemadng_frame(level: f64, frame: u8) -> Vec<u8> {
use filmcraft_dng::tiff::tags;
use filmcraft_dng::writer::{Compression, Options, Value, write};
let (w, h) = (24, 16);
let neutral = [0.6, 1.0, 0.7];
let px: Vec<u16> =
(0..w * h).map(|i| (level * [neutral[0], neutral[1], neutral[1], neutral[2]][(i / w % 2) * 2 + i % 2] * 4000.0 + 95.0).round() as u16).collect();
let o = Options {
width: w,
height: h,
bits: 12,
preview: true,
compression: Compression::LosslessJpeg { predictor: 1, components: 2, restart_lines: 0, pt: 0 },
raw_tags: vec![(tags::BLACK_LEVEL, Value::Short(vec![95])), (tags::WHITE_LEVEL, Value::Short(vec![4095]))],
main_tags: vec![
(tags::AS_SHOT_NEUTRAL, Value::rationals(&neutral)),
(tags::FRAME_RATE, Value::SRational(vec![(24_000, 1001)])),
(tags::TIME_CODES, Value::Byte(vec![frame, 0, 0, 0x01, 0, 0, 0, 0])),
],
..Default::default()
};
write(&px, &o)
}
#[test]
fn import_cinemadng_folder_as_one_clip() {
let dir = tmp_dir("cinemadng-import");
let mut s = demo();
s.execute("prefs.set", json!({"key": "media.indeterminateTimebase", "value": "25"})).unwrap();
let paths: Vec<String> = (0..4u8)
.map(|i| {
let p = dir.join(format!("A001_C002_0101AB_{:06}.dng", i));
std::fs::write(&p, cinemadng_frame(0.1 + 0.2 * f64::from(i), i)).unwrap();
p.to_string_lossy().to_string()
})
.collect();
let r = s.execute("file.import", json!({"paths": [paths[0]], "imageSequence": true})).unwrap();
let id = item_of(&r);
assert_eq!(r["imageSequences"][0]["frames"], 4);
let m = s.project.item(id).unwrap().as_media().unwrap().clone();
assert_eq!(m.info.kind, MediaKind::ImageSequence);
assert_eq!(m.info.frame_rate(), FrameRate::FPS_23_976, "the CinemaDNG FrameRate tag, not the timebase setting");
assert_eq!(m.info.duration, FrameRate::FPS_23_976.tick_of(4));
assert_eq!(m.info.start_timecode, Some(3600 * 24), "01:00:00:00 at 24 nominal");
let v = m.info.video.as_ref().unwrap();
assert_eq!((v.width, v.height), (24, 16));
let src = s.media.source_for(&s.project, id, &*s.services).unwrap();
for i in 0..4 {
let f = src.video_frame(FrameRequest::full(FrameRate::FPS_23_976.tick_of(i))).unwrap();
let filmcraft_frame::PixelData::RgbaF32(px) = &f.data else { panic!("linear float frames") };
let want = 0.1 + 0.2 * i as f32;
for k in 0..3 {
assert!((px[(8 * 24 + 8) * 4 + k] - want).abs() < 2e-3, "frame {i} channel {k}: {}", px[(8 * 24 + 8) * 4 + k]);
}
}
// one frame on its own is a still
let r = s.execute("file.import", json!({"paths": [paths[3]]})).unwrap();
let still = s.project.item(item_of(&r)).unwrap().as_media().unwrap();
assert_eq!(still.info.kind, MediaKind::Still);
assert_eq!(still.info.video.as_ref().unwrap().codec, "DNG");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn image_sequence_survives_save_and_reopen() {
let dir = tmp_dir("imgseq-reopen");

219
crates/engine/src/keyframe_move.rs Normal file
View File

@ -0,0 +1,219 @@
//! Atomic movement of keys across effect/mask parameters; the common timeline delta keeps
//! relative timing intact, including at the clip boundaries. Selection itself is a UI concern.
use std::collections::{BTreeMap, HashSet};
use filmcraft_project::{ClipId, Keyframe, Param, Sequence, TrackItem};
use filmcraft_time::Tick;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::{Result, Session, commands::bad};
const COMMAND: &str = "effects.moveKeyframes";
const MAX_KEYS: usize = 4096;
#[derive(Clone, Deserialize, Serialize, PartialEq, Eq, Hash)]
#[serde(deny_unknown_fields)]
struct Key {
clip: u64,
effect: usize,
param: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
mask: Option<usize>,
time: Tick,
}
type Lane = (u64, usize, Option<usize>, String);
fn param_mut<'a>(sequence: &'a mut Sequence, lane: &Lane) -> Result<&'a mut Param> {
let (_, it) = sequence.find_item_mut(ClipId(lane.0)).ok_or_else(|| bad(COMMAND, "clip does not exist"))?;
let effect = it.effects.get_mut(lane.1).ok_or_else(|| bad(COMMAND, "effect does not exist"))?;
match lane.2 {
Some(mask) => effect.masks.get_mut(mask).and_then(|m| m.param_mut(&lane.3)),
None => effect.params.get_mut(&lane.3),
}
.ok_or_else(|| bad(COMMAND, "parameter does not exist"))
}
fn direction(it: &TrackItem) -> f64 {
it.speed.abs() * if it.reverse { -1.0 } else { 1.0 }
}
pub(crate) fn move_many(s: &mut Session, p: &Value) -> Result<Value> {
let raw = p.get("keyframes").and_then(Value::as_array).ok_or_else(|| bad(COMMAND, "need keyframes"))?;
if raw.is_empty() || raw.len() > MAX_KEYS {
return Err(bad(COMMAND, "keyframe count must be between 1 and 4096"));
}
let requested = p.get("delta").and_then(Value::as_i64).ok_or_else(|| bad(COMMAND, "need an integer timeline delta"))?;
let mut keys: Vec<Key> = serde_json::from_value(Value::Array(raw.clone())).map_err(|_| bad(COMMAND, "invalid keyframe reference"))?;
let mut unique = HashSet::new();
let mut saved = Vec::with_capacity(keys.len());
let q = s.active_sequence().ok_or(crate::EngineError::NoSequence)?;
let frame = q.settings.frame_rate.frame_duration();
let (mut low, mut high) = (i64::MIN, i64::MAX);
for key in &keys {
if key.param.len() > 256 || !unique.insert(key.clone()) {
return Err(bad(COMMAND, "duplicate or invalid keyframe reference"));
}
let (_, it) = q.find_item(ClipId(key.clip)).ok_or_else(|| bad(COMMAND, "clip does not exist"))?;
let effect = it.effects.get(key.effect).ok_or_else(|| bad(COMMAND, "effect does not exist"))?;
let param = match key.mask {
Some(mask) => effect.masks.get(mask).and_then(|m| m.param(&key.param)),
None => effect.params.get(&key.param),
}
.ok_or_else(|| bad(COMMAND, "parameter does not exist"))?;
let k = param.keyframes.iter().find(|k| k.time == key.time).ok_or_else(|| bad(COMMAND, "keyframe does not exist"))?;
let speed = direction(it);
if !speed.is_finite() || speed == 0.0 || it.duration < frame {
return Err(bad(COMMAND, "clip has no movable keyframe range"));
}
let a = it.moving_source_time_at(it.start);
let b = it.moving_source_time_at(it.end() - frame);
if key.time < a.min(b) || key.time > a.max(b) {
return Err(bad(COMMAND, "keyframe is outside the visible clip range"));
}
let da = (a - key.time).0 as f64 / speed;
let db = (b - key.time).0 as f64 / speed;
low = low.max(da.min(db).ceil() as i64);
high = high.min(da.max(db).floor() as i64);
saved.push((k.clone(), speed));
}
if low > high {
return Err(bad(COMMAND, "keyframes have no common movement range"));
}
let delta = requested.clamp(low, high);
if delta == 0 {
return Ok(json!({"delta":0,"keyframes":keys}));
}
let mut lanes: BTreeMap<Lane, Vec<(Tick, Keyframe)>> = BTreeMap::new();
for (key, (mut k, speed)) in keys.iter_mut().zip(saved) {
let from = key.time;
k.time += Tick((delta as f64 * speed).round() as i64);
key.time = k.time;
lanes.entry((key.clip, key.effect, key.mask, key.param.clone())).or_default().push((from, k));
}
s.edit_sequence("Move Keyframes", |sequence, _, _| {
for (lane, moved) in lanes {
let param = param_mut(sequence, &lane)?;
let from: HashSet<_> = moved.iter().map(|(t, _)| *t).collect();
let to: HashSet<_> = moved.iter().map(|(_, k)| k.time).collect();
// Remove all originals before inserting any destination: selected keys can move into
// each other's old positions without overwriting a key still waiting to be moved.
param.keyframes.retain(|k| !from.contains(&k.time) && !to.contains(&k.time));
param.keyframes.extend(moved.into_iter().map(|(_, k)| k));
param.keyframes.sort_by_key(|k| k.time);
}
Ok(())
})?;
Ok(json!({"delta":delta,"keyframes":keys}))
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
fn fixture() -> (Session, Vec<Value>) {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();
let clip = s.active_sequence().unwrap().video_tracks[0].items[0].id;
let mut keys = Vec::new();
for (param, seconds) in [("scale", 0.5), ("scale", 1.0), ("rotation", 1.5)] {
s.execute("playhead.set", json!({"seconds":seconds})).unwrap();
s.execute("effects.addKeyframe", json!({"clip":clip.0,"effect":"motion","param":param})).unwrap();
let it = s.active_sequence().unwrap().find_item(clip).unwrap().1;
let effect = it.effects.iter().position(|e| e.effect == "motion").unwrap();
keys.push(json!({"clip":clip.0,"effect":effect,"param":param,"time":it.source_time_at(s.playhead()).0}));
}
(s, keys)
}
fn values(s: &Session, key: &Value) -> Vec<Keyframe> {
let it = s.active_sequence().unwrap().find_item(ClipId(key["clip"].as_u64().unwrap())).unwrap().1;
it.effects[key["effect"].as_u64().unwrap() as usize].params[key["param"].as_str().unwrap()].keyframes.clone()
}
#[test]
fn movement_across_lanes_is_atomic_and_one_undo_step() {
let (mut s, keys) = fixture();
let project = s.project.clone();
let before = values(&s, &keys[0]);
let undo = s.history.undo.len();
// Scale's first key moves into the second one's old position. Both must survive.
let delta = before[1].time - before[0].time;
let result = s.execute(COMMAND, json!({"keyframes":keys,"delta":delta.0})).unwrap();
let after = values(&s, &keys[0]);
assert_eq!(after.len(), 2);
for (old, new) in before.iter().zip(&after) {
assert_eq!(new.time, old.time + delta);
assert_eq!(serde_json::to_value(&new.value).unwrap(), serde_json::to_value(&old.value).unwrap());
assert_eq!(new.interp, old.interp);
assert_eq!(new.in_influence, old.in_influence);
assert_eq!(new.out_influence, old.out_influence);
}
assert_eq!(result["keyframes"][2]["time"].as_i64().unwrap(), keys[2]["time"].as_i64().unwrap() + delta.0);
assert_eq!(s.history.undo.len(), undo + 1);
let moved = serde_json::to_value(&s.project).unwrap();
s.undo();
assert_eq!(serde_json::to_value(&s.project).unwrap(), serde_json::to_value(project).unwrap());
s.redo();
assert_eq!(serde_json::to_value(&s.project).unwrap(), moved);
}
#[test]
fn bounds_clamp_the_group_without_changing_its_spacing() {
let (mut s, keys) = fixture();
let undo = s.history.undo.len();
let result = s.execute(COMMAND, json!({"keyframes":keys,"delta":i64::MAX})).unwrap();
let delta = result["delta"].as_i64().unwrap();
assert!(delta > 0 && delta < i64::MAX);
for (old, new) in keys.iter().zip(result["keyframes"].as_array().unwrap()) {
assert_eq!(new["time"].as_i64().unwrap() - old["time"].as_i64().unwrap(), delta);
}
assert_eq!(s.history.undo.len(), undo + 1);
s.execute(COMMAND, json!({"keyframes":result["keyframes"],"delta":i64::MAX})).unwrap();
assert_eq!(s.history.undo.len(), undo + 1, "a boundary no-op creates no edit");
}
#[test]
fn invalid_references_and_hostile_requests_cannot_make_partial_edits() {
let (mut s, keys) = fixture();
let before = s.project.clone();
let undo = s.history.undo.len();
let mut invalid = keys[1].clone();
invalid["param"] = json!("missing");
for request in [
json!({"keyframes":[keys[0],invalid],"delta":100}),
json!({"keyframes":[keys[0],keys[0]],"delta":100}),
json!({"keyframes":[],"delta":100}),
json!({"keyframes":vec![keys[0].clone();MAX_KEYS+1],"delta":100}),
json!({"keyframes":[{"clip":keys[0]["clip"],"effect":u64::MAX,"param":"scale","time":0}],"delta":i64::MIN}),
json!({"keyframes":keys,"delta":"fast"}),
] {
assert!(s.execute(COMMAND, request).is_err());
assert!(Arc::ptr_eq(&before, &s.project));
assert_eq!(s.history.undo.len(), undo);
}
}
#[test]
fn media_times_follow_clip_speed_and_reverse() {
for reverse in [false, true] {
let (mut s, keys) = fixture();
let clip = ClipId(keys[0]["clip"].as_u64().unwrap());
s.edit_sequence("test speed", |q, _, _| {
let it = q.find_item_mut(clip).unwrap().1;
it.speed = 2.0;
it.reverse = reverse;
Ok(())
})
.unwrap();
let delta = Tick::from_seconds_f64(0.1);
let result = s.execute(COMMAND, json!({"keyframes":keys,"delta":delta.0})).unwrap();
for (old, new) in keys.iter().zip(result["keyframes"].as_array().unwrap()) {
let diff = new["time"].as_i64().unwrap() - old["time"].as_i64().unwrap();
assert_eq!(diff, delta.0 * 2 * if reverse { -1 } else { 1 });
}
}
}
}

138
crates/engine/src/lib.rs
View File

@ -23,6 +23,7 @@ pub mod graphic_templates;
pub mod graphics;
pub mod interchange;
pub mod keyboard;
mod keyframe_move;
mod marker_export;
pub mod masks;
pub mod media_browser;
@ -192,6 +193,10 @@ impl Services for FsServices {
}
}
/// A point in the undo history (its top step), from [`Session::undo_mark`].
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct UndoMark(Option<usize>);
/// Undo history of whole-project snapshots.
#[derive(Clone, Default)]
pub struct History {
@ -230,6 +235,29 @@ pub struct GapSelection {
pub range: TimeRange,
}
/// An effect of a clip (`effects.select`): its id and which of the clip's effects with that id it
/// is (0 = the first). Not its index, which applying an effect shifts (standard effects go first).
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EffectSel {
pub clip: ClipId,
pub effect: String,
pub instance: usize,
}
impl EffectSel {
/// The selection of `effects[index]` of `clip`.
pub fn of(clip: ClipId, effects: &[filmcraft_project::EffectInstance], index: usize) -> Option<Self> {
let id = &effects.get(index)?.effect;
let instance = effects.iter().take(index).filter(|e| &e.effect == id).count();
Some(Self { clip, effect: id.clone(), instance })
}
/// Where the selected effect is in the clip's `effects`.
pub fn index(&self, effects: &[filmcraft_project::EffectInstance]) -> Option<usize> {
effects.iter().enumerate().filter(|(_, e)| e.effect == self.effect).nth(self.instance).map(|(i, _)| i)
}
}
/// Editing state that commands depend on (not project data, but headless-relevant).
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct EditorState {
@ -291,6 +319,11 @@ pub struct EditorState {
/// The mask being edited on the Program monitor (Effect Controls selection).
#[serde(default)]
pub selected_mask: Option<masks::MaskSel>,
/// The effect selected in Effect Controls (its header or one of its properties): Motion and
/// Transform show their handles on the Program monitor. Selecting a mask clears it, and
/// selecting an effect clears the mask selection.
#[serde(default)]
pub selected_effect: Option<EffectSel>,
/// Multi-Camera Audio Follows Video: switching a multi-camera clip's angle switches its linked
/// audio clips too.
#[serde(default)]
@ -357,6 +390,9 @@ pub struct Session {
pub journal: Vec<(String, Value)>,
/// Background jobs (exports, render previews…).
pub jobs: Vec<Job>,
/// Export job → the offline items it drew the slate for, filled when the export finishes
/// (#1054; `offlineMedia` in `file.exportMedia` and queue results).
pub export_offline: std::collections::HashMap<u64, Arc<std::sync::Mutex<Vec<media_pool::OfflineMedia>>>>,
/// User preferences (`prefs.*` commands) and where they persist (None = not persisted).
pub prefs: autosave::Preferences,
pub prefs_path: Option<std::path::PathBuf>,
@ -476,6 +512,7 @@ impl Session {
events: Vec::new(),
journal: Vec::new(),
jobs: Vec::new(),
export_offline: Default::default(),
prefs: Default::default(),
prefs_path: None,
persistence: None,
@ -833,6 +870,77 @@ impl Session {
Ok(r)
}
/// Run several undoable edits as one undo step labelled `label`. If `f` fails, every edit it made
/// is rolled back and the history is as before.
pub fn edit_group<R>(&mut self, label: &str, f: impl FnOnce(&mut Session) -> Result<R>) -> Result<R> {
let (project, state, undo, redo, limit) =
(self.project.clone(), self.state.clone(), self.history.undo.len(), self.history.redo.clone(), self.history.limit);
// keep every step the group makes, however many, so the first one can be found afterwards
self.history.limit = usize::MAX;
// a panic inside the group must not leave the history unlimited: restore, then let it go on
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(self)));
self.history.limit = limit;
let first = self.history.undo.get(undo).map(|e| e.1.clone());
let rolled_back = |s: &mut Session| {
s.history.undo.truncate(undo);
s.history.redo = redo.clone();
s.history.merge_key = None;
s.project = project.clone();
s.state = state.clone();
s.bump();
};
match (r, first) {
(Ok(Ok(v)), Some(first)) => {
self.history.undo.truncate(undo);
self.history.undo.push((label.to_string(), first));
if self.history.undo.len() > limit {
self.history.undo.remove(0);
}
self.history.merge_key = None;
Ok(v)
}
(Ok(Ok(v)), None) => Ok(v),
(Ok(Err(e)), _) => {
rolled_back(self);
Err(e)
}
(Err(panic), _) => {
rolled_back(self);
std::panic::resume_unwind(panic)
}
}
}
/// Where the undo history stands now, for [`Session::revert_to`].
pub fn undo_mark(&self) -> UndoMark {
UndoMark(self.history.undo.last().map(|u| Arc::as_ptr(&u.1) as usize))
}
/// Take back every edit made since `mark` (a gesture cancelled with Escape, #580), merged or
/// not: the project returns to how it was at `mark`, and neither undo nor redo keeps them.
/// Nothing happens when `mark` has left the history. Returns how many steps were taken back.
pub fn revert_to(&mut self, mark: UndoMark) -> usize {
if let UndoMark(Some(p)) = mark
&& !self.history.undo.iter().any(|u| Arc::as_ptr(&u.1) as usize == p)
{
return 0;
}
let mut earliest = None;
let mut n = 0;
while self.undo_mark() != mark {
let Some((_, prev)) = self.history.undo.pop() else { break };
earliest = Some(prev);
n += 1;
}
if let Some(prev) = earliest {
self.project = prev;
self.history.merge_key = None;
self.fix_state();
self.bump();
}
n
}
/// Edit the active sequence with the edit-algebra context.
pub fn edit_sequence<R>(&mut self, label: &str, f: impl FnOnce(&mut Sequence, &mut EditCtx, &mut EditorState) -> Result<R>) -> Result<R> {
self.edit_sequence_as(label, None, f)
@ -902,6 +1010,11 @@ impl Session {
self.state.selected_mask = None;
}
}
if let Some(e) = &self.state.selected_effect
&& self.active_sequence().and_then(|q| q.find_item(e.clip)).and_then(|(_, it)| e.index(&it.effects)).is_none()
{
self.state.selected_effect = None;
}
if let Some(s) = self.state.active_sequence
&& p.sequence(s).is_none()
{
@ -1024,11 +1137,22 @@ impl Session {
/// [`Session::render_program_at`], with the reason when nothing can be rendered.
pub fn try_render_program_at(&self, scale: f32, t: Tick) -> Result<filmcraft_render::Image> {
self.render_program_with(scale, t, &self.media.provider(self.project.clone(), self.services.clone()))
}
/// [`Session::try_render_program_at`], also naming the offline items the frame drew the slate
/// for (#1054).
pub fn try_render_program_with_offline_at(&self, scale: f32, t: Tick) -> Result<(filmcraft_render::Image, Vec<media_pool::OfflineMedia>)> {
let provider = self.media.provider(self.project.clone(), self.services.clone()).recording();
let img = self.render_program_with(scale, t, &provider)?;
Ok((img, provider.offline_used()))
}
fn render_program_with(&self, scale: f32, t: Tick, provider: &dyn filmcraft_render::SourceProvider) -> Result<filmcraft_render::Image> {
let seq = self.renderable_sequence(scale)?;
let t = self.sequence_rate().snap(t.max(Tick::ZERO));
let provider = self.media.provider(self.project.clone(), self.services.clone());
let opts = filmcraft_render::RenderOptions { scale, captions: true, ..Default::default() };
Ok(filmcraft_render::render_sequence(&self.project, seq, t, opts, &provider))
Ok(filmcraft_render::render_sequence(&self.project, seq, t, opts, provider))
}
/// The active sequence, if a frame of it at `scale` is within the image size limits.
@ -1086,12 +1210,16 @@ const MAX_SUBCLIP_CHAIN: usize = 16;
#[cfg(test)]
mod aaf_omf_tests;
#[cfg(test)]
mod animated_image_tests;
#[cfg(test)]
mod audio_effects_tests;
#[cfg(test)]
mod audio_placement_tests;
#[cfg(test)]
mod autosave_tests;
#[cfg(test)]
mod avi_import_tests;
#[cfg(test)]
mod clip_ops_tests;
#[cfg(test)]
mod color_tests;
@ -1114,6 +1242,8 @@ mod keyboard_tests;
#[cfg(test)]
mod linked_speed_tests;
#[cfg(test)]
mod locked_clip_edits_tests;
#[cfg(test)]
mod masks_tests;
#[cfg(test)]
mod media_browser_tests;
@ -1140,6 +1270,8 @@ mod par_tests;
#[cfg(test)]
mod paste_media_tests;
#[cfg(test)]
mod place_items_tests;
#[cfg(test)]
mod presets_tests;
#[cfg(test)]
mod previews_tests;
@ -1156,6 +1288,8 @@ mod remix_tests;
#[cfg(test)]
mod ripple_delete_tests;
#[cfg(test)]
mod roll_linked_tests;
#[cfg(test)]
mod scopes_tests;
#[cfg(test)]
mod sequence_inspect_tests;

84
crates/engine/src/locked_clip_edits_tests.rs Normal file
View File

@ -0,0 +1,84 @@
//! #911: a locked track refuses timing and source-range edits the way a locked destination
//! refuses `timeline.move`.
use super::*;
use filmcraft_project::ClipId;
use serde_json::json;
fn fixture() -> (Session, u64) {
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Lock check"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 0, "width": 16, "height": 16})).unwrap();
s.execute("file.newOfflineFile", json!({"name": "Media", "seconds": 4, "fps": 25, "video": true, "audio": false, "width": 16, "height": 16})).unwrap();
let item = s.project.items.values().find(|i| i.as_media().is_some()).map(|i| i.id.0).unwrap();
s.execute("timeline.place", json!({"item": item, "track": "V1", "time": 0, "sourceIn": 0, "duration": 1_016_064_000_000i64})).unwrap();
let clip = s.active_sequence().unwrap().video_tracks[0].items[0].id.0;
(s, clip)
}
fn v1_clip(s: &Session, id: u64) -> filmcraft_project::TrackItem {
s.active_sequence().unwrap().find_item(ClipId(id)).unwrap().1.clone()
}
/// Place a 100-frame clip, lock V1, trim Out by −20 frames: the command errors and duration stays 100.
#[test]
fn locked_track_refuses_trim_out_and_keeps_duration() {
let (mut s, clip) = fixture();
let before = v1_clip(&s, clip);
assert_eq!(s.sequence_rate().frame_at(before.duration), 100);
s.execute("timeline.setTrack", json!({"track": "V1", "locked": true})).unwrap();
let err = s.execute("timeline.trim", json!({"clip": clip, "edge": "out", "mode": "regular", "deltaFrames": -20})).unwrap_err();
assert!(err.to_string().contains("locked"), "{err}");
let after = v1_clip(&s, clip);
assert_eq!(after.duration, before.duration);
assert_eq!(after.start, before.start);
assert_eq!(after.source_in, before.source_in);
assert_eq!(after.speed, before.speed);
}
#[test]
fn unlocked_track_still_trims_out() {
let (mut s, clip) = fixture();
s.execute("timeline.trim", json!({"clip": clip, "edge": "out", "mode": "regular", "deltaFrames": -20})).unwrap();
assert_eq!(s.sequence_rate().frame_at(v1_clip(&s, clip).duration), 80);
}
#[test]
fn locked_track_refuses_move_on_the_same_track() {
let (mut s, clip) = fixture();
let before = v1_clip(&s, clip);
s.execute("timeline.setTrack", json!({"track": "V1", "locked": true})).unwrap();
let err = s.execute("timeline.move", json!({"moves": [{"clip": clip, "track": "V1", "time": 1_016_064_000_000i64}], "linked": false})).unwrap_err();
assert!(err.to_string().contains("locked"), "{err}");
assert_eq!(v1_clip(&s, clip).duration, before.duration);
}
#[test]
fn locked_track_refuses_roll_slip_slide_speed() {
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Lock check"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 0, "width": 16, "height": 16})).unwrap();
s.execute("file.newOfflineFile", json!({"name": "Media", "seconds": 4, "fps": 25, "video": true, "audio": false, "width": 16, "height": 16})).unwrap();
let item = s.project.items.values().find(|i| i.as_media().is_some()).map(|i| i.id.0).unwrap();
let frame = 10_160_640_000i64;
s.execute("timeline.place", json!({"item": item, "track": "V1", "time": 0, "sourceIn": frame * 10, "duration": frame * 60})).unwrap();
s.execute("timeline.place", json!({"item": item, "track": "V1", "time": frame * 60, "sourceIn": 0, "duration": frame * 100})).unwrap();
let (left, right) = {
let q = s.active_sequence().unwrap();
(q.video_tracks[0].items[0].id.0, q.video_tracks[0].items[1].id.0)
};
s.execute("timeline.setTrack", json!({"track": "V1", "locked": true})).unwrap();
let snap = s.project.clone();
for cmd in [
json!({"id": "timeline.roll", "p": {"left": left, "right": right, "deltaFrames": 10}}),
json!({"id": "timeline.slip", "p": {"clip": left, "deltaFrames": -10}}),
json!({"id": "timeline.slide", "p": {"clip": left, "deltaFrames": 10}}),
json!({"id": "timeline.rateStretch", "p": {"clip": left, "edge": "out", "deltaFrames": 20}}),
json!({"id": "clip.speedDuration", "p": {"clips": [left], "speed": 200}}),
json!({"id": "timeline.trim", "p": {"clip": left, "edge": "in", "mode": "regular", "deltaFrames": 20}}),
] {
let err = s.execute(cmd["id"].as_str().unwrap(), cmd["p"].clone()).unwrap_err();
assert!(err.to_string().contains("locked"), "{}: {err}", cmd["id"]);
assert_eq!(*s.project, *snap, "{} must leave the sequence unchanged", cmd["id"]);
}
}

11
crates/engine/src/masks.rs
View File

@ -91,9 +91,12 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
/// A mask path from JSON: the serialized [`MaskPath`] object, an array of vertices
/// (`{"p":[x,y],"in":[x,y]?,"out":[x,y]?}`), or an array of `[x, y]` / `{"x","y"}` points.
pub fn path_from_json(v: &Value) -> Option<MaskPath> {
if let Ok(p) = serde_json::from_value::<MaskPath>(v.clone()) {
return Some(p);
}
// The vertex forms first: serde also accepts `{"p","in","out"}` vertices (the `masks.list` /
// `masks.add` form) but silently drops the unknown `in` / `out` tangents.
vertex_path(v).or_else(|| serde_json::from_value::<MaskPath>(v.clone()).ok())
}
fn vertex_path(v: &Value) -> Option<MaskPath> {
let arr = v.as_array().or_else(|| v.get("vertices").and_then(Value::as_array))?;
let closed = v.get("closed").and_then(Value::as_bool).unwrap_or(true);
let mut vertices = Vec::new();
@ -244,6 +247,7 @@ fn add(s: &mut Session, p: &Value) -> Result<Value> {
e.masks.push(Mask::new(name.clone(), path));
let mi = e.masks.len() - 1;
st.selected_mask = Some(MaskSel { clip, effect: ei, mask: mi });
st.selected_effect = None;
Ok((ei, mi, name))
})?;
Ok(json!({"clip": clip.0, "effect": ei, "mask": mi, "name": name}))
@ -508,6 +512,7 @@ fn select(s: &mut Session, p: &Value) -> Result<Value> {
return Err(bad("masks.select", format!("no mask {mi}")));
}
s.state.selected_mask = Some(MaskSel { clip, effect: ei, mask: mi });
s.state.selected_effect = None;
Ok(Value::Null)
}

19
crates/engine/src/masks_tests.rs
View File

@ -380,3 +380,22 @@ fn rounding_an_animated_mask_rounds_every_keyframe_and_only_the_chosen_corners()
assert_eq!(v.len(), 6);
assert!((v[0]["p"][0].as_f64().unwrap() - 450.0).abs() < 1e-6, "{}", v[0]);
}
/// `path_to_json` → `path_from_json` keeps the Bézier tangents (they used to come back as zero, so
/// a pen mask's dragged-out curves were committed as corners; #853).
#[test]
fn path_json_round_trip_keeps_tangents() {
use filmcraft_geom::Vec2;
use filmcraft_project::mask::{MaskPath, MaskVertex};
let path = MaskPath {
vertices: vec![
MaskVertex { p: Vec2::new(10.0, 20.0), t_in: Vec2::new(-5.0, 0.0), t_out: Vec2::new(5.0, 0.0) },
MaskVertex::corner(Vec2::new(80.0, 20.0)),
MaskVertex { p: Vec2::new(40.0, 90.0), t_in: Vec2::new(3.0, -4.0), t_out: Vec2::new(-3.0, 4.0) },
],
closed: true,
};
assert_eq!(crate::masks::path_from_json(&crate::masks::path_to_json(&path)), Some(path.clone()));
// the serialized project form still parses
assert_eq!(crate::masks::path_from_json(&serde_json::to_value(&path).unwrap()), Some(path));
}

Some files were not shown because too many files have changed in this diff Show More