shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge origin/main into #260 (keep both new HostHooks fields: run_dialog, open_url) 

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MrCloudy2 2026-10-10 22:02:52 +02:00
parent 7d6c53d03d a6b323056f
commit 82242c3817
212 changed files with 18939 additions and 1107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AGENTS.md
View File

@ -142,6 +142,7 @@ as PhotoCraft does with `cargo xtask corpus`.
- [docs/agents.md](docs/agents.md): driving FilmCraft over MCP / the control channel, and the agent work loop
- [docs/control-protocol.md](docs/control-protocol.md): control-channel method reference
- [ATTRIBUTION.md](ATTRIBUTION.md): asset index
- [ROADMAP.md](ROADMAP.md), [docs/target-app-parity.md](docs/target-app-parity.md), [docs/gaps.md](docs/gaps.md): stage, parity with Premiere Pro and the ranked work list (craftrules `standards/progress-docs.md`)
## Contributor credits (About window)

2
CLAUDE.md
View File

@ -4,7 +4,7 @@ FilmCraft is a clean-room, open-source, pure-Rust non-linear video editor target
## Start every session here
`plan/` is maintainer-local (gitignored, not in the repo). Each step names the public equivalent to use if you don't have it.
1. Read `plan/STATUS.md` (current milestone, next unchecked task, blockers). Public: [`ROADMAP.md`](ROADMAP.md).
1. Read `plan/STATUS.md` (current milestone, next unchecked task, blockers). Public: [`ROADMAP.md`](ROADMAP.md), then [`docs/gaps.md`](docs/gaps.md) (ranked shortfalls) and [`docs/roadmap.md`](docs/roadmap.md) (current focus).
2. Read the task in `plan/execution-plan.md` §3, the relevant section of `plan/architecture.md`, and the README/docs of the crate you touch. Public: [`docs/architecture.md`](docs/architecture.md), [`docs/contributing.md`](docs/contributing.md), [`docs/testing.md`](docs/testing.md). Visual/behaviour reference: `plan/premiere/` (local only, never committed).
3. Follow the autonomous operation protocol (`plan/execution-plan.md` §7; public summary: [`docs/agents.md`](docs/agents.md) §5–7). Don't stop to ask unless it lists the decision as the user's.

67
Cargo.lock generated
View File

@ -1667,12 +1667,18 @@ dependencies = [
name = "filmcraft"
version = "0.5.0"
dependencies = [
"arboard",
"chrono",
"cpal",
"eframe",
"egui",
"filmcraft-engine",
"filmcraft-export",
"filmcraft-gpu",
"filmcraft-platform",
"filmcraft-project",
"filmcraft-render",
"filmcraft-time",
"filmcraft-ui-egui",
"glob",
"log",
@ -1680,6 +1686,7 @@ dependencies = [
"objc2 0.6.4",
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2",
"open",
"rfd",
"serde_json",
"sys-locale",
@ -1781,11 +1788,15 @@ dependencies = [
"filmcraft-automation",
"filmcraft-codecs",
"filmcraft-engine",
"filmcraft-export",
"filmcraft-gpu",
"filmcraft-media",
"filmcraft-mpegts",
"filmcraft-mxf",
"filmcraft-ogg",
"filmcraft-platform",
"filmcraft-project",
"filmcraft-render",
"filmcraft-time",
"serde_json",
"tokio",
@ -1879,6 +1890,7 @@ dependencies = [
"filmcraft-testkit",
"filmcraft-text",
"filmcraft-time",
"filmcraft-tts",
"image",
"log",
"serde",
@ -1970,12 +1982,14 @@ name = "filmcraft-gpu"
version = "0.5.0"
dependencies = [
"filmcraft-color",
"filmcraft-export",
"filmcraft-frame",
"filmcraft-geom",
"filmcraft-media",
"filmcraft-project",
"filmcraft-render",
"filmcraft-time",
"image",
"log",
"pollster 0.4.0",
"rayon",
@ -2223,6 +2237,28 @@ dependencies = [
"serde",
]
[[package]]
name = "filmcraft-tts"
version = "0.5.0"
dependencies = [
"candle-core",
"candle-nn",
"filmcraft-audio-dsp",
"filmcraft-tts-text",
"serde",
"serde_json",
"thiserror 2.0.21",
"zip",
]
[[package]]
name = "filmcraft-tts-text"
version = "0.5.0"
dependencies = [
"serde",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-ui-egui"
version = "0.5.0"
@ -2236,6 +2272,7 @@ dependencies = [
"filmcraft-color",
"filmcraft-edit",
"filmcraft-engine",
"filmcraft-export",
"filmcraft-frame",
"filmcraft-geom",
"filmcraft-gpu",
@ -2247,6 +2284,7 @@ dependencies = [
"filmcraft-scopes",
"filmcraft-text",
"filmcraft-time",
"filmcraft-tts",
"image",
"log",
"pollster 0.4.0",
@ -3195,6 +3233,25 @@ dependencies = [
"generic-array",
]
[[package]]
name = "is-docker"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3"
dependencies = [
"once_cell",
]
[[package]]
name = "is-wsl"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5"
dependencies = [
"is-docker",
"once_cell",
]
[[package]]
name = "itertools"
version = "0.15.0"
@ -4120,6 +4177,16 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "open"
version = "5.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa576c76302b7b808eecc68061e67336c47833ef9d22caa74dda10fa9675eebc"
dependencies = [
"is-wsl",
"libc",
]
[[package]]
name = "openssl-probe"
version = "0.2.1"

2
Cargo.toml
View File

@ -44,6 +44,8 @@ filmcraft-interchange = { path = "crates/interchange" }
filmcraft-text = { path = "crates/text" }
filmcraft-captions = { path = "crates/captions" }
filmcraft-speech = { path = "crates/speech" }
filmcraft-tts = { path = "crates/tts" }
filmcraft-tts-text = { path = "crates/tts-text" }
filmcraft-render = { path = "crates/render" }
filmcraft-gpu = { path = "crates/gpu" }
filmcraft-scopes = { path = "crates/scopes" }

11
README.md
View File

@ -274,20 +274,23 @@ The control protocol is documented in [docs/control-protocol.md](docs/control-pr
| [docs/control-protocol.md](docs/control-protocol.md) | Control-channel and MCP reference |
| [docs/project-files.md](docs/project-files.md) | `.fcproj` format, schema migrations, auto-save and crash recovery |
| [docs/graphics.md](docs/graphics.md) · [docs/captions.md](docs/captions.md) | Text engine, graphic clips and tools; caption tracks and formats |
| [ROADMAP.md](ROADMAP.md) | Honest assessment, what's missing, milestones and estimates |
| [ROADMAP.md](ROADMAP.md) | Stage, parity numbers, languages, what's next and the progress log |
| [docs/target-app-parity.md](docs/target-app-parity.md) · [docs/gaps.md](docs/gaps.md) · [docs/roadmap.md](docs/roadmap.md) | Parity assessment against Premiere Pro, the ranked list of shortfalls, milestones and current focus |
| [docs/codec-parity.md](docs/codec-parity.md) · [docs/file-format-parity.md](docs/file-format-parity.md) · [docs/hardware-parity.md](docs/hardware-parity.md) · [docs/ui-parity.md](docs/ui-parity.md) · [docs/localization-parity.md](docs/localization-parity.md) | Parity checklists by dimension |
## Status
FilmCraft is young and moving fast. Editing, trimming, multicam, colour, keyframes, effects, titles, captions, mixing, codecs and export work today.
We track two numbers ([ROADMAP.md](ROADMAP.md#honest-assessment-2026-10-05)):
FilmCraft is in **alpha**. We track two numbers ([docs/target-app-parity.md](docs/target-app-parity.md), re-measured 2026-10-10 against Premiere Pro 26.5.2):
- **Feature checklist: ~87%.** Premiere Pro's menu items, effects, transitions, panels and formats that exist in FilmCraft.
- **Ready for real work: ~50–60%.** Our honest estimate of how close FilmCraft is to replacing Premiere on real projects.
- **Feature breadth: ~86%.** Premiere Pro's menu items, effects, transitions, panels and formats that exist in FilmCraft (menus 92% and effects 100% measured).
- **Ready for real work: ~56%.** Our honest estimate of how close FilmCraft is to replacing Premiere on real projects.
The biggest gaps today:
- **Speed on big footage.** Hardware decoding works on macOS and Windows, and for H.264 and HEVC on Linux (VA-API, or NVDEC with NVIDIA's driver; VP9 and AV1 decode in software there). Blend modes and the most common effects run on the GPU, but Lumetri, keys and export rendering still run on the CPU; H.264 encoding can use the hardware encoder on macOS and on Windows and Linux with an NVIDIA GPU (opt-in, Export ▸ Hardware encoding), and H.265 export is hardware-only on both (HDR sequences export Main 10 PQ / HLG with NVENC; VideoToolbox writes 8-bit SDR) ([#30](https://github.com/storytold/filmcraft/issues/30)).
- **Premiere projects.** `.prproj` files can't be opened yet; bring sequences over as FCP7 XML, AAF or OTIO.
- **No plugins.** No VST3 / Audio Units or OpenFX hosting.
- **Delivery codecs.** H.264 is our only software delivery-codec export; H.265 exports only through a hardware encoder (NVIDIA on Windows: Main 8-bit SDR and Main 10 HDR PQ / HLG; VideoToolbox on macOS: 8-bit Main, SDR); no AV1 export yet.
- **Real-world media and platforms.** Our decoders are bit-exact on conformance streams, but camera and phone files in the wild are less tested. Windows and Linux get far less testing than macOS.

246
ROADMAP.md
View File

@ -1,164 +1,111 @@
# FilmCraft Roadmap
Progress toward parity with Adobe Premiere Pro, with estimates. Updated as milestones land.
**Stage: alpha** · next: beta, ~19 points of ready-for-real-work and ~400–700 h away, plus opening Premiere's `.prproj` ([why](docs/target-app-parity.md#stage); passes the [core-workflow gate](docs/roadmap.md#alpha-gate))
**Last updated:** 2026-10-05 · **Feature checklist:** ~87% · **Ready for real work:** ~50–60% · **Code:** 35 crates, 1605+ tests
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (full re-measure against Premiere Pro 2026 26.5.2; scorecard, gaps, milestones and localization moved into docs/ per the craftrules progress-docs standard) · **Target:** Adobe Premiere Pro 2026 (26.5.2)
Two numbers, because they answer different questions. The **feature checklist** counts whether
Premiere's menu items, effects, panels and formats exist in FilmCraft. **Ready for real work** is
our honest estimate of how close FilmCraft is to replacing Premiere for an editor on real
projects: depth, correctness, robustness, speed and the pro ecosystem. The gap between the two
is the work that matters most now. Agents: read [Where we are lacking](#where-we-are-lacking)
before choosing work.
Progress toward parity with Adobe Premiere Pro. This page is the summary; the detail is in
[docs/target-app-parity.md](docs/target-app-parity.md) (assessment, methodology, calibration),
[docs/gaps.md](docs/gaps.md) (the ranked work list: **agents, read it before choosing work**) and
[docs/roadmap.md](docs/roadmap.md) (milestones and current focus).
## Honest assessment (2026-10-05)
## Headline numbers
### How the checklist number is made, and what it doesn't tell you
- **Partly measured.** The menu figure comes from diffing a dump of Premiere 26.5's menus against
`filmcraft-cli commands`. The effects, transitions and audio-effects counts are checked by unit
tests against Premiere's Effects panel tree (`premiere_26_video_effects_catalogue`,
`premiere_audio_set_is_complete_and_foldered`, `premiere_26_transition_tree`).
- **Partly agent-reported.** The other areas' percentages are written by the agents doing the work.
No automated parity check runs in CI (PhotoCraft has `cargo xtask parity`; we don't yet), so
nothing stops a number drifting upward.
- **Presence, not quality.** An item counts once it exists. An effect described as an
"approximation" counts the same as an exact one. Nothing compares our rendering with
Premiere's.
- **Weights understate what users hit first.** Performance carries 5%. Plugins and AI features
aren't in the scorecard at all.
### By dimension
| Dimension | Checklist says | Honest estimate | Evidence |
|---|---|---|---|
| **Feature breadth** (menus, commands, panels) | ~86–95% | **~85%** | Menu diff against Premiere 26.5; effect, transition and audio-effect catalogues complete by test. The long tail is mostly there. |
| **Correctness in depth** | not tracked | **~60–70%** | A first-time contributor found four real bugs in core paths within hours (#5–#8: AIFF import failed, Slide left linked audio behind, transitions didn't follow ripple trims, export ignored start/end times). User reports: Color Matte always grey with no picker (#29); ALSA audio error on Linux (#23). Settings are "most wired", not all. |
| **Codecs and media** | ~90% | **~75%** | Our decoders are bit-exact on conformance streams. Real camera and phone media (variable frame rate, damaged files, unusual containers) is far less tested. No camera RAW (RED, BRAW, ARRIRAW), no E-AC-3. |
| **Export** | ~92% | **~70%** | Delivery: H.264, plus H.265 through a hardware encoder only (NVENC on Windows: Main 8-bit SDR and Main 10 HDR PQ / HLG; VideoToolbox on macOS: Main 8-bit SDR); ProRes, DNxHR and image sequences cover mastering. No AV1 export, no 10-bit / HDR HEVC on macOS. AAF and OMF have never been validated in Avid Media Composer or Pro Tools. Export renders and encodes on the CPU only. |
| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS (VideoToolbox, #33; active in the desktop app only since #41, so releases up to 0.2.1 decoded in software) and Windows (Media Foundation / Direct3D 11 DXVA, H.264, HEVC Main / Main 10, VP9 and AV1 4:2:0 8/10-bit, #30), and for H.264 and HEVC on Linux (VA-API: 4K H.264 at 11 ms of CPU per frame instead of 417 ms, 4K HEVC at 10 ms instead of 203 ms). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects and export rendering are CPU; H.264 encoding can use VideoToolbox on macOS (opt-in per export, no B-frames) and NVENC on Windows and Linux with an NVIDIA GPU (opt-in per export), and H.265 (HEVC) export exists on macOS and Windows only, through the same hardware encoders. Linux decodes H.264 and HEVC in hardware (VP9 and AV1 there are software); its only hardware encoder is NVENC H.264. 8K isn't real-time (AV1 is on Windows with a hardware decoder). The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. |
| **Stability** | not tracked | **improving, unproven** | The never-crash pass (no panics in product code, last-resort guards) landed on 2026-10-04. There's no field record from real users' projects yet. |
| **Plugins and pro ecosystem** | not tracked | **~0–10%** | No audio plugin hosting (VST3 / Audio Units) and no third-party video effects (OpenFX). For many professional editors this alone rules FilmCraft out. Team Projects and Productions are out of scope by design. |
| **AI features** | partial | **~25–35%** | Speech to text exists but is off in default builds (the `whisper` feature). Enhance Speech is a DSP chain, not a model. Auto Reframe is approximate. No Generative Extend, and no media-intelligence search or auto colour. |
| **Platforms** | not tracked | **macOS solid; Windows and Linux thin** | Development and testing happen on macOS. Windows and Linux are built for releases but barely exercised at runtime. The web build is a compile target; the browser app shell isn't finished. |
| **UI fidelity** | ~85% | **~80%** | Agents compare with Premiere reference screenshots; this hasn't been checked independently. |
### Where we are lacking
In priority order. Agents should prefer this work over adding more checklist items.
1. **Hardware acceleration** (#30), the most visible gap to users:
- hardware decode on Linux (VA-API): H.264 and HEVC are in; VP9 and AV1 remain;
- zero-copy decoded frames into wgpu;
- the remaining effects on the GPU: Lumetri, keys, Vignette, Video Limiter, masks (31 common effects are done);
- GPU export, then hardware encode: H.264 and H.265 on macOS are in (H.264 opt-in; H.265 has no other encoder), H.264 and H.265 (Main 8-bit, and Main 10 HDR PQ / HLG) on NVENC (Windows); Main 10 / HDR HEVC on VideoToolbox, Linux and other Windows vendors remain.
2. **A measured parity number.** Add `cargo xtask parity`, run in CI. It should cover menus,
commands, effects, transitions, panels, preferences and formats against the Premiere snapshot in
`plan/premiere/`. It should report presence and fidelity separately (exact, approximate, stub).
3. **Real-world media corpus.** Phone, camera and screen recordings, variable frame rate, damaged
files, long GOPs, odd containers. Each gets an import → edit → export round-trip test with an
ffprobe / ffmpeg oracle.
4. **Workflow acceptance tests.** Scripted MCP sessions that do real jobs end to end: assemble and
trim a scene, colour it, mix and loudness-normalise it, caption it, deliver it, round-trip
through XML / AAF.
5. **Windows and Linux at runtime.** CI that runs the test suite and a headless playback/export
smoke test on both, not just release builds. Fix the Linux audio path (#23).
6. **Delivery codecs.** HEVC export is in on macOS (VideoToolbox, 8-bit, no software encoder) and on Windows (NVENC, 8-bit and Main 10 HDR); still to do: HEVC elsewhere, 10-bit / HDR HEVC on macOS, and AV1 (`rav1e`, BSD-2).
7. **Plugin hosting** (VST3 / Audio Units, then OpenFX). Needs FFI, so it falls under the isolated
`unsafe` crate rule (AGENTS.md §0.3). **Owner decision needed** before starting.
8. **AI features** with openly licensed local models: transcription on by default, text-based
editing, speech enhancement, reframing.
9. **The web app shell**: file access, WebCodecs, audio.
### Where we're going
**Next:** close items 1–5. Hardware acceleration fixes the gap users notice first (CPU-bound 4K playback and export). The
parity tool, media corpus and workflow tests make the progress number trustworthy. Windows and
Linux CI makes "runs everywhere" true.
**After that:** delivery codecs, plugin hosting (pending the owner's decision) and AI features,
measured with the same tools.
**Goal:** "ready for real work" at 85%+, measured rather than estimated, on all three desktop
platforms.
## Feature checklist scorecard
Measured against Premiere Pro 26.5 on this machine. Menu items: the native menu bar dump, minus
Adobe-cloud-only items (Team Projects, Productions, Firefly, Stock, Dynamic Link, account/help pages),
matched against `filmcraft-cli commands` and the UI command table. Effects: the Effects panel tree.
| Area | Weight | Measured | Coverage |
|---|---|---|---|
| Editing, timeline, trimming, multicam | 20% | core edit algebra, trim modes, sync, multicam, menu long tail, Scene Edit Detection, Automate to Sequence, Premiere default keyboard (143 new commands; 39 cloud/absent-panel keys skipped) | ~86% |
| Menus / commands | (cross-check) | ~325 of 344 in-scope menu items | ~95% |
| Effects and transitions | 12% | video effects 93/93 (+Legacy, Obsolete), audio effects 53/53, video transitions 84/84 (+21 Legacy), audio transitions 3/3; some approximations (Warp Stabilizer 2-D, Morph Cut, Auto Reframe) | ~92% |
| Media I/O and codecs | 12% | H.264, HEVC, VP9, AV1, ProRes, DNx, MJPEG, MPEG-2/MPEG-1 (4:2:2 incl. IMX/XDCAM), Opus, AAC, AC-3, MP2; MP4/MOV/MKV/WebM, MXF (incl. MPEG-2), MPEG TS/PS (AVCHD, broadcast, DVD), Ogg, image sequences, BWF; no E-AC-3, camera raw, HW decode | ~90% |
| Panels and UI fidelity | 12% | all main panels incl. Metadata, Timecode, Events, Progress, Reference Monitor, full Lumetri Scopes, audio effect editors; Project panel List/Icon/Freeform with view presets and hover scrub; Media Browser with Favorites/navigation/ingest | ~85% |
| Audio | 10% | mixer, automation (track + clip), Essential Sound, meters, all 53 effects, 5.1 tracks/buses/panner/export, voice-over record, Remix | ~88% |
| Colour | 8% | Lumetri complete, LUTs, colour management, HDR, all Lumetri Scopes (+ `scopes.read` for agents) | ~88% |
| Graphics and captions | 8% | text engine, shapes, per-character styles, our own graphics templates (.fcgt, 8 built-ins, export/install/edit), rolls/crawls, responsive pins + time, captions SRT/VTT/SCC/MCC/STL/TTML/DFXP, transcripts | ~85% |
| Export | 8% | own H.264/AAC, ProRes, DNxHR, MXF OP1a/OP-Atom, image sequences, GIF, WAV, 5.1; preset library + user presets, full Export-mode settings, queue, Quick Export; AAF (Edit Protocol) import/export, OMF 2.0 export; no AAF/OMF validation against Avid/Pro Tools yet | ~92% |
| Preferences and project management | 5% | Settings dialog with 16 categories (most settings wired), project settings, scratch disks, search bins, templates | ~80% |
| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; blend modes on the GPU (#32); 8K and AV1 not yet real-time on a loaded machine; H.264 and HEVC decode in hardware on Linux (VA-API), and on Windows only decode and opt-in NVENC H.264 export (see the honest assessment: ~35–40% against Premiere) | ~62% |
| **Weighted total** | | | **~87%** |
## Estimate to parity
This estimate is for the **feature checklist**. Closing the honest-assessment gaps (above) is the
"Robust on real-world material" row, plus plugins and AI features, which it doesn't include.
Measured throughput in the last work block (2026-10-01 night → 10-02): five Opus 5.5 agents in
parallel for ~4.5 wall-clock hours (~18 agent-hours including integration) moved parity by ~4 points
(≈4–5 agent-hours per point), on a machine that was heavily overloaded (load 150–300 on 14 cores) and
once ran out of disk. The remaining points are a long tail (each effect, dialog and preference page is
small, but there are many), so the estimate applies a 1.3–1.5× tail factor.
| | Opus 5.5 agent-hours | Wall-clock (4–5 parallel agents + integrator) |
| | Value | How |
|---|---|---|
| Feature parity by checklist (~19 points left) | ~90–125 | **~20–32 h** |
| Robust on real-world material (codec edge cases, 4K/8K performance, pro workflows) | +120–200 | **+30–50 h** |
| **Total to "100% and better"** | **~210–325** | **~50–82 h (≈2–3.5 days, 24/7)** |
| **Feature breadth** | **~86%** | Does each Premiere feature exist? Menus 92% (measured: label diff against Premiere 26.5.2's menu dump), effects / transitions / audio effects 100% (measured by tests), other areas estimated |
| **Ready for real work** | **~56%** (51–61%) | How close FilmCraft is to replacing Premiere for an editor on real projects: depth, correctness, file compatibility, speed, stability, ecosystem. Estimated depth per area, combined by a [written formula](docs/target-app-parity.md#three-readiness-numbers) |
| **Mainstream practitioner** | **~55%** | A typical professional editor's weekly work only, discounted ×0.90 interaction, ×0.92 stability, ×0.88 file exchange with Premiere users ([method](docs/target-app-parity.md#mainstream-practitioner)) |
| **Essentials user** | **~60%** | Import, cut, title, music, export at default settings, discounted ×0.88 launch / stability, ×0.90 discoverability, ×0.95 opening files people send ([method](docs/target-app-parity.md#essentials-user)) |
| **To beta** | **~400–700 Opus 5.5 agent-hours** | Estimated, calibrated from this repo's PR history |
| **To full parity** | **~1,550–2,750 agent-hours** | ~70% parallelizes; plugins, AI models, native-speaker review and some hardware need a human |
| Code (measured) | 45 crates, ~343k lines of Rust, 2,520 tests, 268 merged PRs, 69 authors | `git`, `gh`, `grep` |
The 2026-10-02 block moved parity ~62% → ~72% with six agents in ~6 wall-clock hours, despite the
disk filling twice; the remaining work is mostly panels (scopes, Metadata, Media Browser), export
presets/queue, codecs (MXF, image sequences, hardware decode), performance and graphics templates.
Two numbers, because they answer different questions. The gap between them is the work that
matters most now: breadth is nearly done, depth is not.
Limits on speed: CPU and disk on one machine (more than ~5 agents slows everyone down: each worktree
build is 6–7 GB and a full `cargo xtask ci` takes 20–60 min under load), and a single integrator
merging, resolving conflicts (e.g. two agents bumping the project schema) and re-running CI. With one
agent and no parallelism, multiply wall-clock by ~3–4.
### Readiness by audience
Not reachable clean-room and locally: **Generative Extend** (needs a large video-generation model).
**Enhance Speech** and **Auto Reframe** are feasible only with openly licensed models we can ship.
| Audience | Ready % | Opus 5.5 agent wall-clock hours to ~95% | Work that dominates |
|---|---|---|---|
| Full target (ready for real work) | ~56% | ~1,450–2,600 (~70% parallelizes) | Plugin hosting, AI models, localization incl. RTL / Devanagari, camera RAW and delivery codecs, `.prproj`, hardware acceleration, UI fidelity |
| Mainstream practitioner | ~55% | ~500–850 (~65% parallelizes) | `.prproj` import (60–120), UI fidelity (120–200), stability and PR CI (40–80), 10-bit / 4:2:2 camera decode (30–50), playback / hardware (60–100), import and export gaps (60–100) |
| Essentials user | ~60% | ~180–350 (~60% parallelizes) | Launch crashes on Windows / Intel / older macOS (40–80), discoverability and timeline friction (60–110), playback lag (30–60), common import gaps (20–40), audio devices (15–30) |
## Milestones
Method and calibration: [docs/target-app-parity.md](docs/target-app-parity.md#three-readiness-numbers).
Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining agent-hours.
## By dimension
| # | Milestone | Status | Done | Remaining | Est. |
|---|---|---|---|---|---|
| M0 | Skeleton + visual shell | ✅ | Workspace, 20 crates, dock/workspaces, Premiere 26 look, native menus, control channel, MCP, xtask gates (layers, wasm) | — | — |
| M1 | Media I/O | ✅ | MP4/MOV demux+mux, WAV, stills, MJPEG, symphonia audio (MP3/FLAC/ALAC/Vorbis), GOP seek + frame cache, import | Media Browser polish | 2 |
| M2 | H.264 decoder | ✅ | Own decoder, bit-exact on 37+ streams, 500–600 fps 1080p | — | — |
| M3 | Editing core | 🟡 | Edit algebra (insert/overwrite/razor/lift/extract/ripple/roll/slip/slide/rate-stretch/nest/paste), tools, markers, trim mode + Trim Monitor + dynamic J/K/L trimming, Keyboard Shortcuts editor with FilmCraft/Premiere/FCP/Avid presets; Edit/Clip/File menu commands (Label colours and Select Label Group, Paste/Remove Attributes, Select All Matching, Remove Unused, Consolidate Duplicates, Sequence From Clip, Bin From Selection, Offline File, Close Project, Make/Edit Subclip, Modify Audio Channels/Timecode, Frame Hold Options/Add Frame Hold/Insert Frame Hold Segment, Time Interpolation with frame blending, Fit/Fill frame, Breakout to Mono, Extract Audio, Replace With Clip; M3.11: Scene Edit Detection (pure-Rust cut detection, background job), Normalize Mix Track, Simplify Sequence, Transcribe Sequence, Find/Find Next and search bins, Automate to Sequence, Edit Original, Edit Offline, Source Settings, Update Metadata (XMP), Generate Audio Waveform, Project Settings General/Scratch Disks, Get Media File Properties, Save as Template, Selection as FilmCraft Project, Avid Log Exchange export, Flash Cue markers, Dynamic Audio Waveforms, Reveal Log Files, System Compatibility Report); M3.12 keyboard parity: Premiere's default keyboard (~115 keyboard-only commands: edit-point navigation on targeted/any track, select clip at playhead/next/previous, extend edit to playhead, nudge/slip/slide selection, target and source-patch toggles, clip volume ±1 dB/many, frame maximize/full screen/panel cycling, monitor zoom, track heights, Project and Text panel keyboard navigation, text size/leading/alignment, Export Frame, poster frames; see docs/keyboard.md); multicam (Create Multi-Camera Source Sequence, Multi-Camera view with live switching on 1–9, angle switching, Enable/Flatten, Edit Cameras) and sync (Synchronize, Merge Clips; In/Out/timecode/marker/audio — GCC-PHAT, sample-accurate) | Multicam paging >16 angles and grid thumbnails, optical flow (renders as frame blending), 39 Premiere default shortcuts skipped with reasons in docs/keyboard.md (Productions, AI/cloud tools, work area bar, Production/Search panels) | 6–10 |
| M4 | Playback | 🟡 | Audio-clock master, prefetch with cancellation, J/K/L, correct dropped-frame stats, playback resolution, render bar + content-hashed render previews, App Nap opt-out; 1080p H.264 and 3 stacked 1080p streams play with 0 dropped frames | 4K under load, 8K, frame-threaded AV1 decode, reduced-resolution decode for multicam grids | 6–10 |
| M5 | Effects, keyframes, GPU | 🟡 | All 93 Premiere 26 video effects and 84 transitions (CPU), blend modes and 31 common effects on the GPU (#32, GPU2), keyframes + value/velocity graphs, wgpu compositor, effect + opacity masks (ellipse/polygon/pen, feather, expansion, modes; CPU/WGSL parity), mask tracking (Lucas–Kanade + RANSAC), adjustment layers, effect presets (built-in + user, JSON import/export) | Effects on the GPU (in progress, #30), GPU masks in the live path, GPU export, exact Warp Stabilizer (3-D) and Morph Cut | 20–30 |
| M6 | Export | ✅ | Own H.264 encoder (High/Main/Baseline, B-frames, VBR/CBR/2-pass) → MP4 or QuickTime + own AAC; ProRes, DNxHR, MJPEG, PNG/TIFF/BMP sequences (Premiere-style numbering), GIF, WAV, AIFF; background jobs; M6.5: Export mode parity (frame size / rate / scaling / pixel aspect, profile / level, CBR / VBR 1- and 2-pass, keyframe distance, audio codec / rate / channels / bitrate / sample size, multiplexer, burn-in or sidecar captions, image / name / timecode overlays, video limiter, loudness normalization with true-peak limiter, metadata, ranges, estimated size, summary), 24 built-in presets + user presets with favourites and import/export (Preset Manager), export queue (reorder, cancel, retry, several sequences / ranges), Quick Export; `export.*` commands and `filmcraft-cli export --preset` | Interlaced encoding, 5.1 audio, smart render, publishing destinations | 6–8 |
| M7 | Audio | 🟡 | Mixer graph (tracks → submixes → Mix, pre/post-fader inserts and sends, latency-compensated, sample-accurate, ~6× realtime for 24 tracks × 3 effects on one core), Audio Track Mixer + Audio Clip Mixer panels, track automation (Off/Read/Latch/Touch/Write, recorded live while playing, thinned to keyframes, timeline lanes with pen editing), solo/solo-safe, channel mapping basics, peak + BS.1770 loudness meters (match ffmpeg), DSP crate with 16 clip/track effects, Audio Gain (set/adjust/normalize), Constant Power / Constant Gain / Exponential Fade, Essential Sound (types, Loudness auto-match, Repair incl. DeEss/DeReverb, Clarity, Creative, Ducking keyframes, presets; full Dialogue chain 22× realtime) | Music duration remix, ML speech enhancement, 5.1 panner and multichannel buses, voice-over record, effect editor windows (EQ curve), remaining effects (multiband, convolution reverb), clip-mixer automation recording | 6–9 |
| M8 | Colour | 🟡 | Lumetri: basic, creative + looks, RGB & hue curves, wheels, HSL secondary, vignette, section bypass; Input LUT / Look LUT (.cube 1D/3D/shaper, .3dl; tetrahedral CPU + WGSL; project LUT library; built-in camera conversions); Colour Match (Oklab tonal-range statistics, skin protection, solved in Lumetri wheels); colour management: Rec. 709 / Rec. 2100 PQ / HLG working spaces + wide gamut, Interpret Footage colour space (S-Log3, V-Log, Canon Log 2/3, LogC3/4, Apple Log, D-Log from published specs), metadata auto-detect, BT.2390 tone mapping, gamut mapping, HDR export signalling (VUI/colr/mdcv/clli/SEI, ffprobe-verified); Lumetri Scopes: Vectorscope YUV (75/100 % targets, skin-tone line) and HLS, Histogram, Parade (RGB/YUV/RGB-White), Waveform (RGB/Luma/YC/YC no Chroma), presets, Rec. 601/709/2100, 8-bit/float/HDR (cd/m²) scales, multi-scope grid, `scopes.read` | HDR-aware Lumetri maths, macOS EDR monitors, mastering metadata → tone-map peak, D-Log M (no published formula), HSL Secondary refine | 3–5 |
| M9 | More codecs | 🟡 | ProRes decode+encode, AAC decode+encode, HEVC Main/Main 10 decoder (bit-exact on 41 fixtures, ~225 fps 1080p), VP9 decoder (profiles 0–3, 8/10/12-bit, bit-exact on 50+ fixtures; WebM/MKV `V_VP9` and MP4 `vp09` import with key-frame-checked seeking), Matroska/WebM import (H.264/HEVC/VP9/ProRes/MJPEG + AAC/Opus/FLAC/MP3/Vorbis/PCM), Opus decoder (SILK/CELT/hybrid, 5.1/7.1 multistream; all RFC 8251 vectors range-exact; WebM/MKV/MP4), DNxHD/DNxHR decoder (all SMPTE ST 2019-1 CIDs, 8/10/12-bit, 4:2:2/4:4:4, interlaced, alpha; within IDCT precision of ffmpeg on 22 fixtures) + DNxHR LB/SQ/HQ/HQX/444 encoder and MOV `AVdh` export, AV1 decoder (Main profile, 8/10-bit, all intra/inter tools, loop filters, superres, film grain, intra BC, spatial layers; bit-exact vs libdav1d on 19 SVT fixtures + 22 libaom vectors; MP4 `av01` / WebM `V_AV1` import), MXF import (OP1a/OP-Atom; AVC with POC-ordered B pictures bit-exact, DNxHD/DNxHR, ProRes, PCM/AES3; MPEG-2 reported unsupported; index-table seeking, material package timecode), Ogg Opus/Vorbis (granule seeking, pre-skip, end trimming), image sequences, Broadcast WAV timecode | VP9 frame threading, AV1 threading/SIMD (~4 fps 1080p today), AV1 High/Professional profiles, MPEG-2 decoder, hardware decode | 12–20 |
| M10 | Graphics & captions | 🟡 | Caption tracks (Subtitle/CEA-608/708/Teletext formats, track style), SRT/WebVTT/SCC import+export (frame-exact, property-tested), caption editing (add/split/merge/trim/move, sync-locked insert/extract), Text panel Captions tab, burn-in in Program monitor and export; text engine (`crates/text`: bundled + system fonts, harfrust shaping, bidi, line breaking, paragraph layout, glyph cache; 3-line 1080p title ≈ 0.15 ms warm); graphic clips with text + shape layers (fill, 2 strokes, background, shadow, keyframable transform), Type tool with on-monitor editing, shape/pen tools, Properties/Essential Graphics editor, align/distribute; graphics templates (own `.fcgt` format, 8 original built-ins, Browse tab with engine-rendered thumbnails, export / install / apply / editable properties; never reads .mogrt), rolls/crawls with ease and pre/postroll, responsive pins and intro/outro, per-character styles, Upgrade Caption to Graphic, source graphics, Replace Fonts; MCC (608+708), EBU STL, TTML/IMSC1, DFXP (frame-exact, property-tested) | 608/708 embedding in video streams, speech-to-text | 12–18 |
| M11 | Interchange & project management | ✅ | `.fcproj` schema versions + migrations, atomic saves, Save a Copy/Revert, auto-save ring + crash-recovery journal (Preferences ▸ Auto Save, recovery prompt), FCP7 XML, FCPXML, EDL, OTIO; offline media (own slate) + Link Media (fingerprint-checked relink, folder remap, search, Align Timecode, Make Offline); proxies (ProRes Proxy/LT, H.264 ¼/½ background jobs, attach/detach/reconnect, monitor toggle, export full-res) + ingest (copy/transcode/proxies); Project Manager (collect, consolidate + transcode with handles, size estimate) | Rename media to clip names, image-sequence conversion, Media Browser-driven relink, smart (cross-drive) path tracking | 1–2 |
| M12–M16 | Multicam, web (WASM), platform, long tail | 🟡 | Multicam: audio sync (GCC-PHAT, sub-sample), Merge Clips, multi-camera source sequences, Multi-Camera view with live switching (1–9); L0–L4 crates compile to wasm32 | Web app shell (file access, WebCodecs, audio), scene detection, auto reframe, ~800 remaining commands and dialogs, performance hardening | 35–55 |
| Dimension | Ready | Hours | Doc |
|---|---|---|---|
| Features (editing, effects, colour, audio, graphics, export settings) | ~66% | 350–620 | [target-app-parity.md](docs/target-app-parity.md#by-feature-area) |
| UI / UX fidelity | ~60% | 120–200 | [ui-parity.md](docs/ui-parity.md) |
| File formats (containers, `.prproj`, interchange, captions, stills) | ~60% | 140–240 | [file-format-parity.md](docs/file-format-parity.md) |
| Codecs | ~60% | 150–250 | [codec-parity.md](docs/codec-parity.md) |
| Hardware acceleration | ~45% | 120–200 | [hardware-parity.md](docs/hardware-parity.md) |
| Localization | ~35% | 150–230 | [localization-parity.md](docs/localization-parity.md) |
| Performance | ~55% | 60–120 | [performance.md](docs/performance.md) |
| Stability | ~45% | 60–120 | [gaps.md G4](docs/gaps.md#g4-crashes-at-start-up-and-no-test-ci-on-pull-requests) |
| Platforms (macOS, Windows, Linux, FreeBSD, web) | ~60% | 50–90 | [gaps.md G9](docs/gaps.md#g9-windows-and-linux-at-run-time) |
| Plugins and ecosystem | ~5% | 200–400 | [gaps.md G11](docs/gaps.md#g11-no-plugin-hosting) |
| AI features | ~25% | 150–300 | [gaps.md G12](docs/gaps.md#g12-ai-features) |
## Running now
## Features
- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Windows: Media Foundation / Direct3D 11 H.264 + HEVC decode (HW2) and VP9 + AV1 (HW3), NVENC H.264 export (HW4, opt-in) and H.265 export. Linux: VA-API H.264 + HEVC decode (HW5). Next: VA-API VP9 / AV1, zero-copy upload, Lumetri on the GPU.
- **Next:** [Where we are lacking](#where-we-are-lacking) items 1–5. GPU export waits for the export-crate work (#19 and the never-crash follow-up) to land.
| Area | Breadth | Ready | Hours |
|---|---|---|---|
| Editing, timeline, trimming, multicam | ~90% | ~70% | 80–140 |
| Effects and transitions | 100% (measured) | ~60% | 80–140 |
| Media import and codecs | ~75% | ~60% | 150–250 |
| Audio | ~88% | ~65% | 50–90 |
| Panels and UI fidelity | ~88% | ~60% | 120–200 |
| Colour | ~90% | ~70% | 40–70 |
| Graphics, titles, captions | ~85% | ~65% | 50–90 |
| Export and delivery | ~80% | ~65% | 60–110 |
| Project files and interchange | ~70% | ~45% | 90–160 |
| Preferences and project management | ~80% | ~65% | 20–40 |
## Log
Evidence for every row: [docs/target-app-parity.md](docs/target-app-parity.md#by-feature-area).
## Languages
Premiere ships 10 interface languages. Measured from our catalogs (3,498 strings):
| Language | Status | Strings |
|---|---|---|
| English | full | 100% |
| Simplified Chinese | partial (whole interface; engine errors English) | 99.6% |
| Spanish | partial (whole interface) | 99.9% |
| Hindi | none (no Devanagari shaping in the interface) | 0% |
| Arabic | none (no RTL interface) | 0% |
| French | none | 0% |
| Portuguese (Brazil) | menus only | 9.9% |
| Indonesian | none | 0% |
| Japanese | partial (whole interface) | 99.7% |
| German | none | 0% |
| Korean | none | 0% |
| Vietnamese | none | 0% |
Also shipped: Ukrainian (menus only, 10%). Detail: [docs/localization-parity.md](docs/localization-parity.md).
## Upcoming
Ranked; detail and "done when" in [docs/roadmap.md](docs/roadmap.md) and [docs/gaps.md](docs/gaps.md).
1. **No launch crashes, and CI on pull requests** (macOS 12, Intel UHD, Windows 11 reports; no workflow runs the tests today). 40–80 h.
2. **Open Premiere's `.prproj`**, the beta blocker. 60–120 h.
3. **10-bit / 4:2:2 camera media without hardware** (H.264 High 10 / 4:2:2, HEVC 4:2:2; #626). 30–50 h.
4. **Measure fidelity against Premiere** and add `cargo xtask parity`. 50–90 h.
5. **Real-world media corpus and workflow acceptance tests.** 40–70 h.
6. **Hardware acceleration, continued** (#30): zero-copy upload, more effects on the GPU, Quick Sync / AMD encoders, VideoToolbox Main 10. 120–200 h.
7. **UI fidelity:** Program monitor transform handles, Effect Controls, docking. 120–200 h.
8. **Plugin hosting** (VST3 / AU, then OpenFX). Owner decision needed. 200–400 h.
## Progress log
- **2026-10-10 (re-measure):** full parity re-measure against Premiere Pro 26.5.2: menu label diff 338 / 366 (92%), installed bundle listings (10 UI languages, exporter and codec frameworks), 188 open issues. Breadth ~86% (file formats now counted), ready ~55%, stage **alpha**; progress docs reorganized per the craftrules standard ([target-app-parity](docs/target-app-parity.md), [gaps](docs/gaps.md), [roadmap](docs/roadmap.md), [codec](docs/codec-parity.md), [file-format](docs/file-format-parity.md), [hardware](docs/hardware-parity.md), [UI](docs/ui-parity.md), [localization](docs/localization-parity.md)).
- **2026-10-10:** ~75 community PRs merged: GPU export rendering (Auto) with Vignette, Video Limiter and Lumetri basic / creative / vignette on the GPU (#421); Linux NVDEC H.264 / HEVC decode (#549); text-to-speech narrations with Kokoro-82M (#278); Japanese for the whole interface (#449) and Ukrainian menus (#407); ALSA default-device fallback (#254, fixes #23); drop import into the timeline (#517); system appearance (#505); timecode scrub editing (#576); NVENC RGBA input (#486); marker CSV export (#454); many fixes (#526–#575).
- **2026-10-09:** release 0.5.0. NVENC H.265 export on Windows, Main and Main 10 HDR (#328, #360); NVENC H.264 export (#386); VA-API H.264 / HEVC decode (#358); Source monitor playback (#423); multi-stream container audio, schema v13 (#361); New Sequence and Sequence Settings dialogs (#269, #355); Spanish for the panels (#238); background decoders and memory stress tests (#385, #381).
- **2026-10-08 (Source range dragging):** implicit full-clip In/Out, video-only/audio-only/linked both controls, picture/waveform drag gestures and one-command undoable timeline placement. Shared Source/Program draggable frame-bounded handles and range translation, cancellation, Source marker/navigation routing and marked-range playback/looping.
- **2026-10-08 (Source playback):** normal forward playback for video/audio media and subclips in the Source monitor, an independent Source clock, Play/Pause and focused Space, frame stepping, device-clock audio with wall-clock fallback, seeking, and end-of-clip stopping. Windows desktop review verified a real video file and standalone WAV, including In/Out marking. Reverse/shuttle Source playback and source sequences remain unsupported; no parity percentage changed.
- **2026-10-08 (night):** HDR H.265 (HEVC Main 10, PQ and HLG, BT.2020, limited range) export on Windows through NVENC (#30): an HDR sequence now exports as real 10-bit HDR (`rgbf_to_yuv420_10` into P010 input buffers; VUI BT.2020 + PQ / HLG; the HDR10 mastering-display and content-light messages as SEI on every IDR for PQ, through the generic SEI payload array because NVENC 12.1 has no mastering-display field; `colr` / `mdcv` / `clli` in the sample entry), where the GPU reports 10-bit HEVC (new `register_hdr_probe` / `hdr_available` in the export crate; elsewhere, and on macOS, HDR sequences still tone-map to 8-bit SDR HEVC, and `settings.sdr` forces that). ffprobe reads `hevc` / `Main 10` / `hvc1` / `yuv420p10le` / `bt2020` / `smpte2084` or `arib-std-b67` / `bt2020nc` / `tv` with the mastering-display and content-light side data for PQ; ffmpeg's decode equals ours code for code; the in-process round trip is at 71.1 dB luma PSNR on the 10-bit scale with 803 distinct levels on a ramp and the peak code 940 intact (RTX 5060). 8-bit SDR H.265 is unchanged.
@ -209,6 +156,13 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining
- **2026-09-30 (late morning):** keyframe value/velocity graphs; xtask gates; H.264 MP4 export; Lumetri curves, wheels, looks, HSL secondary.
- **2026-09-30 (early morning):** H.264 decoder, ProRes, AAC; GPU compositor; MCP; Premiere 26 visual fidelity pass.
### Interface localization
## Revision history
Edit > Language switches English/Japanese/Spanish/Portuguese (Brazilian)/Ukrainian/Simplified Chinese and persists across restarts through `general.interfaceLanguage` (also available in Settings > General). Until a language is chosen, System Language (the default) follows the operating system's or browser's preferred languages and falls back to English. Commands `app.language.english`, `app.language.japanese`, `app.language.spanish`, `app.language.portuguese`, `app.language.ukrainian` and `app.language.chinese` are reachable through the control channel. Translations live in one catalog per language (`crates/ui-egui/src/i18n/<code>.tsv`); UI code wraps its strings in `tl!("…")` / `tlf!("…{name}", name)`, and names that come from registries (effects and their parameters, settings, commands, panels, workspaces, presets) are translated where they are drawn. **Spanish, Japanese and Simplified Chinese cover the whole interface**: menus, panels, dialogs, the Settings pages, the Effects/Effect Controls registry, the Keyboard Shortcuts dialog and status messages; tests fail when a `tl!` literal, a menu label or a registry name has no Spanish entry (`ja.tsv` has the same entries, but the tests do not require them yet; `zh-cn.tsv` must cover every `tl!` literal and menu label too). Searches match the displayed labels as well as English sources. Brazilian Portuguese and Ukrainian cover the core menus; untranslated labels use English. Ukrainian uses the bundled Cyrillic interface fonts. Engine error messages, the control channel, the CLI and MCP stay in English. Japanese text uses the Japanese fonts from [craft-fonts](https://github.com/storytold/craft-fonts) when FilmCraft is built with them (`CRAFT_FONTS_DIR`; all official releases, including the web build), otherwise a Japanese font already installed on the system; with neither, switching to Japanese is refused with a message. Simplified Chinese (`zh-cn`; System Language picks it for `zh`, `zh-CN`, `zh-SG` and `zh-Hans`, not for Traditional locales) uses the Chinese fallback font the interface always installs (craft-fonts, otherwise an installed system face); with none, switching to it is refused the same way. Existing vertical text support is preserved.
| Date | Change | Summary |
|---|---|---|
| 2026-10-10 | minor | Readiness-by-audience table with hours to ~95% per audience |
| 2026-10-10 | minor | Headline adds mainstream practitioner (~55%) and essentials user (~60%); ready for real work ~55% -> ~56% (formula written out, same inputs); beta distance ~19 points |
| 2026-10-10 | minor | Stage banner links the core-workflow gate (all six workflows pass on macOS; stays alpha) |
| 2026-10-10 | major | Restructured to the craftrules progress-docs shape: stage banner, headline numbers, dimension / feature / language tables, upcoming; re-measured against Premiere 26.5.2 (breadth ~87 → ~86%, ready ~50–60% → ~55%); honest assessment and scorecard moved to docs/target-app-parity.md, "Where we are lacking" to docs/gaps.md, milestones to docs/roadmap.md, interface localization to docs/localization-parity.md |
| 2026-10-08 | minor | Log entries for HEVC export, Source playback and range dragging |
| 2026-10-05 | major | Honest assessment added: checklist ~87% vs ready for real work ~50–60% |

6
apps/filmcraft-cli/Cargo.toml
View File

@ -19,13 +19,17 @@ whisper = ["filmcraft-engine/whisper", "filmcraft-engine/speech-download"]
[dependencies]
filmcraft-engine = { workspace = true }
filmcraft-export = { workspace = true }
filmcraft-project = { workspace = true }
filmcraft-render = { workspace = true }
filmcraft-time = { workspace = true }
filmcraft-gpu = { workspace = true }
filmcraft-media = { workspace = true }
filmcraft-codecs = { workspace = true }
filmcraft-platform = { workspace = true }
filmcraft-mxf = { workspace = true }
filmcraft-mpegts = { workspace = true }
filmcraft-ogg = { workspace = true }
filmcraft-time = { workspace = true }
filmcraft-automation = { workspace = true }
serde_json = { workspace = true }
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }

40
apps/filmcraft-cli/src/args.rs
View File

@ -19,6 +19,7 @@ const VALUED: &[&str] = &[
"--end",
"--data-dir",
"--settings",
"--gpu-rendering",
];
#[derive(Debug, Default)]
@ -60,6 +61,17 @@ impl Args {
self.opts.iter().any(|(n, _)| n == k)
}
/// A version query: `--version` anywhere, or `-V` in the subcommand position, so global
/// options may come first (`--demo --version`, `--compact -V`).
pub fn wants_version(&self) -> bool {
self.flag("--version") || self.pos(0) == Some("-V")
}
/// The first option that is not in `allowed`, if any.
pub fn unknown_opt(&self, allowed: &[&str]) -> Option<&str> {
self.opts.iter().map(|(n, _)| n.as_str()).find(|n| !allowed.contains(n))
}
/// Command params from positionals `from..`: one JSON object, or `key=value` pairs
/// (dotted keys nest: `color.r=1`).
pub fn params_from(&self, from: usize) -> Result<Value, String> {
@ -135,9 +147,37 @@ mod tests {
assert_eq!(a.params_from(2).unwrap(), json!({"a": {"b": [1]}}));
}
#[test]
fn version_after_global_options() {
for s in ["--version", "-V", "--demo --version", "--compact -V", "--data-dir d --version", "export --version"] {
assert!(args(s).wants_version(), "{s}");
}
for s in ["version", "--data-dir -V", "exec x -V", "-- --version"] {
assert!(!args(s).wants_version(), "{s}");
}
}
#[test]
fn unknown_opt_reports_first_unlisted_option() {
let allowed = ["--bridge", "--project"];
assert_eq!(args("mcp --automation-read-root X").unknown_opt(&allowed), Some("--automation-read-root"));
assert_eq!(args("mcp --bridge=a --typo=1").unknown_opt(&allowed), Some("--typo"));
assert_eq!(args("mcp --project p --bridge a").unknown_opt(&allowed), None);
}
#[test]
fn double_dash_ends_options() {
let a = args("import -- --weird-name.mov a.mov");
assert_eq!(a.positionals, ["import", "--weird-name.mov", "a.mov"]);
}
#[test]
fn gpu_rendering_takes_a_space_separated_value() {
let a = args("export --gpu-rendering off out.wav --start 0 --end 0.1");
assert_eq!(a.positionals, ["export", "out.wav"]);
assert_eq!(a.opt("--gpu-rendering"), Some("off"));
let b = args("export out.wav --gpu-rendering auto");
assert_eq!(b.positionals, ["export", "out.wav"]);
assert_eq!(b.opt("--gpu-rendering"), Some("auto"));
}
}

43
apps/filmcraft-cli/src/main.rs
View File

@ -75,6 +75,7 @@ SUBCOMMANDS
inspect [project|sequence] project tree or active sequence as JSON (default: both)
import <file>... import media into the project
export <out> [--preset name] [--format f] [--range r] [--start s --end s] [--settings json]
[--gpu-rendering auto|off]
[--scale f] [--quality 0-100] [--no-audio] [--queue]
export the active sequence and wait for it to finish: with an
export preset (`export --list-presets`; built-in or the user's), or
@ -214,6 +215,30 @@ fn format_for(path: &str) -> Option<&'static str> {
/// OS hardware video decoders (VideoToolbox on macOS, Media Foundation on Windows) in front of our
/// own, as in the desktop app. Also what `mcp` and the headless commands decode with.
/// The GPU export frame renderer (filmcraft-gpu's off-screen compositor behind filmcraft-export's
/// frame-renderer hook): exports with GPU rendering Auto composite on the GPU and fall back to the
/// CPU reference renderer wherever it cannot.
struct GpuFrameRenderer(filmcraft_gpu::ExportRenderer);
impl filmcraft_export::FrameRenderer for GpuFrameRenderer {
fn render(
&mut self,
project: &filmcraft_project::Project,
seq: filmcraft_project::ItemId,
t: filmcraft_time::Tick,
opts: filmcraft_render::RenderOptions,
sources: &dyn filmcraft_render::SourceProvider,
) -> Option<filmcraft_render::Image> {
self.0.render(project, seq, t, opts, sources)
}
}
fn register_gpu_frame_renderer() {
filmcraft_export::register_frame_renderer(|| {
filmcraft_gpu::ExportRenderer::new().map(|r| Box::new(GpuFrameRenderer(r)) as Box<dyn filmcraft_export::FrameRenderer>)
});
}
fn register_hardware_decoders() -> filmcraft_platform::Availability {
filmcraft_platform::register()
}
@ -227,11 +252,6 @@ async fn main() {
}
async fn cli() {
if matches!(std::env::args().nth(1).as_deref(), Some("--version" | "-V")) {
outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION"));
return;
}
register_hardware_decoders();
let a = Args::parse(std::env::args().skip(1));
// `--help` anywhere (`filmcraft-cli --help`, `filmcraft-cli export --help`) prints the reference:
// the parser takes it as a flag option, so it never reaches the subcommand match.
@ -239,6 +259,13 @@ async fn cli() {
out!("{HELP}");
return;
}
// `--version` anywhere and `-V` as the subcommand, also after global options (`--demo --version`, #710).
if a.wants_version() {
outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION"));
return;
}
register_hardware_decoders();
register_gpu_frame_renderer();
let Some(cmd) = a.pos(0) else { usage("missing subcommand") };
match cmd {
"help" | "-h" => out!("{HELP}"),
@ -396,6 +423,9 @@ async fn cli() {
if a.flag("--no-audio") {
p["audio"] = json!(false);
}
if let Some(v) = a.opt("--gpu-rendering") {
p["gpuRendering"] = json!(v); // off | auto (validated by the export command)
}
let mut b = Backend::open(&a);
let t0 = std::time::Instant::now();
let r = if a.flag("--queue") {
@ -433,6 +463,9 @@ async fn cli() {
diag(format_args!("rendered {}x{} in {:.1} ms → {out}", img.w, img.h, dt.as_secs_f64() * 1000.0));
}
"mcp" => {
if let Some(opt) = a.unknown_opt(&["--bridge", "--project", "--demo", "--data-dir"]) {
usage(format!("unknown mcp option `{opt}`"));
}
let server = match a.opt("--bridge") {
Some(addr) => filmcraft_automation::FilmcraftMcp::bridge(addr).unwrap_or_else(|e| fail(e)),
None => match Backend::open(&a) {

10
apps/filmcraft-cli/tests/cli.rs
View File

@ -24,6 +24,16 @@ fn help_and_usage_errors() {
assert_eq!(cli(&["exec", "file.newBin", "notkv"]).status.code(), Some(2));
}
#[test]
fn version_after_global_options() {
let want = format!("filmcraft-cli {}\n", env!("CARGO_PKG_VERSION"));
for args in [&["--version"][..], &["-V"], &["version"], &["--demo", "--version"], &["--compact", "-V"], &["--demo", "version"]] {
let o = cli(args);
assert!(o.status.success(), "{args:?}: {}", String::from_utf8_lossy(&o.stderr));
assert_eq!(String::from_utf8_lossy(&o.stdout), want, "{args:?}");
}
}
#[test]
fn long_help_matches_help_command() {
let reference = cli(&["help"]);

24
apps/filmcraft-web/src/download_pacing.rs Normal file
View File

@ -0,0 +1,24 @@
//! Spacing of browser downloads (pure, so it is tested natively in `tests/download_pacing.rs`; the
//! anchor clicks live in [`crate::fs::download`]).
//!
//! Chrome drops anchor-click downloads that start in one tight burst, so an image-sequence export
//! (one file per frame) delivered only about half of its frames (#378). Each download is given a
//! slot [`GAP_MS`] after the previous one.
/// Milliseconds between two download clicks.
pub const GAP_MS: f64 = 125.0;
/// Hands out click times.
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct Pacer {
next_free: f64,
}
impl Pacer {
/// Reserve the next slot at time `now` (ms); returns how long to wait before clicking.
pub fn reserve(&mut self, now: f64) -> f64 {
let start = now.max(self.next_free);
self.next_free = start + GAP_MS;
start - now
}
}

20
apps/filmcraft-web/src/fs.rs
View File

@ -54,6 +54,7 @@ struct Fs {
thread_local! {
static FS: RefCell<Fs> = RefCell::new(Fs::default());
static DOWNLOADS: RefCell<crate::download_pacing::Pacer> = RefCell::new(crate::download_pacing::Pacer::default());
}
/// Wake-up hook for arriving data.
@ -405,7 +406,8 @@ impl Services for WebServices {
}
}
/// Offer bytes as a browser download named `name`.
/// Offer bytes as a browser download named `name`. Downloads are clicked at least
/// [`crate::download_pacing::GAP_MS`] apart: Chrome drops a burst of them (#378).
pub fn download(name: &str, data: &[u8]) -> Result<(), wasm_bindgen::JsValue> {
let doc = web_sys::window().and_then(|w| w.document()).ok_or("no document")?;
let blob = bytes_to_blob(data, "application/octet-stream")?;
@ -414,10 +416,20 @@ pub fn download(name: &str, data: &[u8]) -> Result<(), wasm_bindgen::JsValue> {
a.set_href(&url);
a.set_download(file_name(name));
a.style().set_property("display", "none")?;
doc.body().ok_or("no body")?.append_child(&a)?;
a.click();
a.remove();
let delay = DOWNLOADS.with(|d| d.borrow_mut().reserve(js_sys::Date::now()));
let name = name.to_owned();
wasm_bindgen_futures::spawn_local(async move {
if delay > 0.0 {
sleep(delay as i32).await;
}
let clicked = match doc.body() {
Some(body) => body.append_child(&a).map(|_| a.click()),
None => Err("no body".into()),
};
a.remove();
if let Err(e) = clicked {
log::warn!("download {name}: {e:?}");
}
sleep(60_000).await;
let _ = web_sys::Url::revoke_object_url(&url);
});

1
apps/filmcraft-web/src/lib.rs
View File

@ -20,6 +20,7 @@
pub mod api;
pub mod audio;
pub mod download_pacing;
pub mod fs;
pub mod import;
pub mod opfs;

26
apps/filmcraft-web/tests/download_pacing.rs Normal file
View File

@ -0,0 +1,26 @@
//! Download spacing. It is pure, so it runs natively even though the rest of the crate is
//! wasm32-only.
#[path = "../src/download_pacing.rs"]
mod download_pacing;
use download_pacing::{GAP_MS, Pacer};
/// #378: 20 downloads requested in the same instant are spread out, not clicked in one burst.
#[test]
fn burst_of_downloads_is_spaced_out() {
let mut p = Pacer::default();
let delays: Vec<f64> = (0..20).map(|_| p.reserve(1000.0)).collect();
assert_eq!(delays[0], 0.0);
for (i, d) in delays.iter().enumerate() {
assert_eq!(*d, i as f64 * GAP_MS);
}
}
/// A download after a quiet period is not delayed.
#[test]
fn download_after_a_pause_is_immediate() {
let mut p = Pacer::default();
assert_eq!(p.reserve(0.0), 0.0);
assert_eq!(p.reserve(10_000.0), 0.0);
}

11
apps/filmcraft/Cargo.toml
View File

@ -14,9 +14,16 @@ workspace = true
default = []
# Speech-to-text (Text panel ▸ Transcribe) with Whisper on the CPU, and model downloads.
whisper = ["filmcraft-engine/whisper", "filmcraft-engine/speech-download"]
# Natural Text to Speech voices (Kokoro-82M on the CPU, downloaded on first use).
neural-voices = ["filmcraft-engine/neural-voices"]
[dependencies]
filmcraft-engine = { workspace = true }
filmcraft-export = { workspace = true }
filmcraft-project = { workspace = true }
filmcraft-render = { workspace = true }
filmcraft-time = { workspace = true }
filmcraft-gpu = { workspace = true }
filmcraft-ui-egui = { workspace = true }
filmcraft-platform = { workspace = true }
egui = { workspace = true }
@ -25,8 +32,12 @@ serde_json = { workspace = true }
log = { workspace = true }
rfd = { workspace = true }
cpal = "0.16"
# Files and images on the system clipboard for Paste (#611); egui already uses it for text.
arboard = "3"
chrono = { version = "0.4", default-features = false, features = ["clock"] }
sys-locale = "0.3"
# Help menu / About / Discord links in the system browser (MIT; as PhotoCraft).
open = "5"
[dev-dependencies]
glob = "0.3"

329
apps/filmcraft/src/dev_icon.rs Normal file
View File

@ -0,0 +1,329 @@
//! Dev-time Linux desktop entry: the shell icon fallback.
//!
//! The Linux shell (Wayland *and* X11) resolves the window to a `.desktop` file by app id (set in
//! `main.rs` via `eframe`'s `with_app_id`) and takes the taskbar / window-list icon from its
//! `Icon=` entry in the hicolor theme. Packaging installs both
//! (`packaging/linux/ai.storyteller.filmcraft.desktop` and the hicolor PNGs) — but a source-tree
//! run has neither, so KDE/GNOME fall back to the generic Wayland icon. The fix is to ensure a
//! user-level entry for those runs: when no entry for the app id exists in any XDG data dir,
//! install one (plus the hicolor icon PNGs, read from this checkout's `assets/app-icon`, so
//! nothing is embedded in the binary) into `$XDG_DATA_HOME`/`~/.local/share`. An existing
//! packaged entry is never overridden; the user-owned dev copy is rewritten when it points at
//! another binary (a dev build moves between checkouts). The shell picks the icon up at the *next* app start;
//! `update-desktop-database`/`kbuildsycoca` is not needed for icon lookup.
//!
//! Opt-in only: nothing is written unless `FILMCRAFT_DEV_DESKTOP_ENTRY=1` is set, so ordinary
//! `cargo run`s (agents, worktrees, CI) never touch the user's profile. Packaged runs are
//! excluded even then — AppImage (`APPIMAGE`) mounts at a fresh `/tmp/.mount_*` each launch,
//! so its `Exec` path changes every time and it ships its own entry, and a Flatpak
//! (`FLATPAK_ID`) installs its entry through the manifest. `packaging/linux/install.sh` writes the packaged
//! `Exec=` the same two-layer way as here; keep the two in sync.
//!
//! Every failure is logged and skipped — a missing dev icon must never stop the app
//! ([`AGENTS.md`](/AGENTS.md) §0 "never crash").
/// Install the user-level desktop entry and icons for a source-tree run (Linux).
pub fn ensure_dev_desktop_entry() {
// Packaged and sandboxed runs never write into the user's profile: an AppImage mounts at a
// fresh /tmp/.mount_* every launch (the Exec path changes each time) and a Flatpak installs
// its own entry.
if std::env::var_os("APPIMAGE").is_some() || std::env::var_os("FLATPAK_ID").is_some() {
return;
}
// Opt-in only: a plain `cargo run` (agents, worktrees, other checkouts) must not write into
// the user's profile, and a packaged release never does (packages and install.sh install
// the real entry).
if std::env::var_os("FILMCRAFT_DEV_DESKTOP_ENTRY").is_none_or(|v| v != "1") {
return;
}
if let Err(e) = ensure() {
log::warn!("desktop entry (taskbar icon) not installed: {e}");
}
}
/// Install missing files; `Ok` whether anything was written or an entry already existed.
fn ensure() -> Result<(), DevIconError> {
let file_name = format!("{}.desktop", crate::APP_ID);
let user_dir = user_data_dir()?;
let exe = std::env::current_exe().map_err(DevIconError::Io)?;
// Not `to_string_lossy`: a substituted U+FFFD would install an `Exec` that can never start
// the app, and the same lossy string would match on every later run, so the broken entry
// would never be rewritten or warned about.
let exe = exe.to_str().ok_or(DevIconError::BadExecPath)?;
if exe.contains('=') || exe.contains('%') || exe.chars().any(|c| c.is_control()) {
// A desktop entry cannot carry these (install.sh rejects them for the same reason):
// '=' separates key and value, '%' introduces a field code, control characters are
// not valid in the string layer.
return Err(DevIconError::BadExecPath);
}
for dir in search_dirs(&user_dir) {
let candidate = dir.join("applications").join(&file_name);
if !candidate.is_file() {
continue;
}
// A packaged entry (any system dir) already gives the shell its icon; only our own
// user-level copy is kept fresh.
if dir != user_dir {
return Ok(());
}
let body = std::fs::read_to_string(&candidate).map_err(DevIconError::Io)?;
if desktop_file_exec_is_ours(&body, exe) {
return Ok(());
}
break; // a dev entry from another checkout: fall through and rewrite it
}
for size in ICON_SIZES {
let rel = std::path::Path::new("hicolor").join(format!("{size}x{size}")).join("apps").join(format!("{}.png", crate::APP_ID));
let dest = user_dir.join("icons").join(&rel);
if !dest.is_file()
&& let Some(parent) = dest.parent()
{
// The checkout this binary was built from; a missing icon only costs the icon.
let src = std::path::Path::new(ICON_DIR).join(&rel);
if !src.is_file() {
log::warn!("dev desktop entry: icon {} not found; skipping it", src.display());
continue;
}
std::fs::create_dir_all(parent).map_err(DevIconError::Io)?;
std::fs::copy(&src, &dest).map_err(DevIconError::Io)?;
}
}
let applications = user_dir.join("applications");
std::fs::create_dir_all(&applications).map_err(DevIconError::Io)?;
let body = desktop_file_content(crate::APP_ID, "FilmCraft", exe);
std::fs::write(applications.join(&file_name), body).map_err(DevIconError::Io)?;
Ok(())
}
/// The hicolor icon sizes installed for the taskbar (a subset large enough for panel and topbar;
/// the shell picks the nearest size, so 256 and 512 cover both).
const ICON_SIZES: [u16; 2] = [256, 512];
/// The app icons in the source tree this binary was built from (a path, not embedded bytes:
/// the entry is a dev-checkout convenience and release binaries must not carry the PNGs).
const ICON_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../assets/app-icon");
/// The user data dir we install into: `$XDG_DATA_HOME`, else `$HOME/.local/share` (XDG base
/// directory spec). Untrusted environment: an empty or relative `XDG_DATA_HOME` means the
/// default, and a missing or relative `HOME` is an error — never a guess that would write
/// into the working directory.
fn user_data_dir() -> Result<std::path::PathBuf, DevIconError> {
if let Some(v) = std::env::var_os("XDG_DATA_HOME")
&& !v.is_empty()
&& std::path::Path::new(&v).is_absolute()
{
return Ok(std::path::PathBuf::from(v));
}
match std::env::var_os("HOME").filter(|h| !h.is_empty() && std::path::Path::new(h).is_absolute()) {
Some(home) => Ok(std::path::PathBuf::from(home).join(".local").join("share")),
None => Err(DevIconError::NoHome),
}
}
/// The XDG data dirs to search for an existing entry, user dir first, then `$XDG_DATA_DIRS` (or
/// the spec default `/usr/local/share:/usr/share`). Only absolute paths are taken; the spec
/// requires them and a relative one would read the working directory.
fn search_dirs(user: &std::path::Path) -> Vec<std::path::PathBuf> {
let mut dirs = vec![user.to_path_buf()];
let system = std::env::var_os("XDG_DATA_DIRS")
.filter(|v| !v.is_empty())
.map(|v| v.to_string_lossy().into_owned())
.unwrap_or_else(|| "/usr/local/share:/usr/share".into());
for dir in system.split(':') {
if std::path::Path::new(dir).is_absolute() {
dirs.push(std::path::PathBuf::from(dir));
}
}
dirs
}
/// Whether the `.desktop` body's `Exec` points at our binary exactly (a stale dev entry from
/// another checkout — `/a/filmcraft-old` is not `/a/filmcraft` — does not, and is rewritten; a
/// packaged `Exec=filmcraft` never matches a source-tree path).
fn desktop_file_exec_is_ours(body: &str, exe: &str) -> bool {
exec_line(body).is_some_and(|arg| arg == exe)
}
/// The first `Exec=` argument (the executable), unescaped through both Desktop Entry layers:
/// first the string layer of the file value (`\\` `\s` `\n` `\t` `\r`, per
/// <https://specifications.freedesktop.org/desktop-entry/latest/>), then the shell layer of the
/// argument (double quotes, and `\` before `` ` `"` `\` `$` inside them). `None` when there is no
/// `Exec=` line or its first argument cannot be parsed.
fn exec_line(body: &str) -> Option<String> {
let value = body.lines().find(|l| l.starts_with("Exec=")).and_then(|l| l.strip_prefix("Exec="))?;
let value = unescape_string_value(value);
if let Some(rest) = value.strip_prefix('"') {
let mut out = String::new();
let mut chars = rest.chars();
while let Some(c) = chars.next() {
match c {
'"' => return Some(out),
'\\' => match chars.next() {
Some(e @ ('\\' | '"' | '`' | '$')) => out.push(e),
Some(e) => {
out.push('\\');
out.push(e);
}
None => return None, // a dangling escape ends the argument
},
c => out.push(c),
}
}
None // no closing quote
} else {
Some(value.split(' ').next().unwrap_or_default().to_string())
}
}
/// Undo the desktop entry's string layer: `\\` `\s` `\n` `\t` `\r` are the defined sequences; a
/// backslash before anything else is kept as the character itself (lenient, like desktop file
/// parsers in the wild).
fn unescape_string_value(value: &str) -> String {
let mut out = String::with_capacity(value.len());
let mut chars = value.chars();
while let Some(c) = chars.next() {
if c != '\\' {
out.push(c);
continue;
}
match chars.next() {
Some('\\') => out.push('\\'),
Some('s') => out.push(' '),
Some('n') => out.push('\n'),
Some('t') => out.push('\t'),
Some('r') => out.push('\r'),
Some(e) => out.push(e),
None => out.push('\\'),
}
}
out
}
/// The `.desktop` body: same fields as `packaging/linux/ai.storyteller.filmcraft.desktop`, with
/// `Exec` pointing at this exact binary (a dev build isn't on `$PATH`, so `TryExec` is left out)
/// and the icon by theme name.
fn desktop_file_content(app_id: &str, name: &str, exe: &str) -> String {
let mut out = String::new();
out.push_str("[Desktop Entry]\n");
out.push_str("Type=Application\n");
out.push_str(&format!("Name={name}\n"));
out.push_str("GenericName=Video Editor\n");
out.push_str("Comment=Edit video, color and sound\n");
out.push_str(&format!("Exec=\"{}\" %F\n", escape_exec_argument(exe)));
out.push_str(&format!("Icon={app_id}\n"));
out.push_str("Terminal=false\n");
out.push_str("StartupNotify=true\n");
out.push_str(&format!("StartupWMClass={app_id}\n"));
out.push_str("Categories=AudioVideo;Video;AudioVideoEditing;\n");
out.push_str("Keywords=video;editor;film;timeline;color;grading;nle;\n");
out
}
/// Escape `s` for the `Exec=` value's two layers, like `packaging/linux/install.sh`: first the
/// shell layer (`` ` `` `"` `$` `\` get a backslash, and the argument is double-quoted), then
/// the string layer of the file value (every backslash doubled).
fn escape_exec_argument(s: &str) -> String {
let mut shell = String::with_capacity(s.len() + 2);
for c in s.chars() {
if matches!(c, '`' | '"' | '$' | '\\') {
shell.push('\\');
}
shell.push(c);
}
shell.replace('\\', "\\\\")
}
/// Everything that can keep the dev icon out of the taskbar; logged, never fatal.
#[derive(Debug)]
enum DevIconError {
Io(std::io::Error),
NoHome,
BadExecPath,
}
impl std::fmt::Display for DevIconError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DevIconError::Io(e) => write!(f, "{e}"),
DevIconError::NoHome => write!(f, "no home directory for the user data dir"),
DevIconError::BadExecPath => {
write!(f, "the binary path has a character a desktop entry cannot carry (=, % or a control character), or is not valid UTF-8")
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The entry mirrors the packaged one, derived from the real file so the two cannot drift:
/// minus `TryExec` (a dev build isn't on `$PATH`), plus the quoted dev `Exec` path.
/// The icons the entry installs exist in the checkout.
#[test]
fn icon_sources_exist_in_the_checkout() {
for size in ICON_SIZES {
let p = std::path::Path::new(ICON_DIR).join(format!("hicolor/{size}x{size}/apps/ai.storyteller.filmcraft.png"));
assert!(p.is_file(), "{}", p.display());
}
}
#[test]
fn desktop_file_matches_the_packaged_fields() {
let packaged = include_str!("../../../packaging/linux/ai.storyteller.filmcraft.desktop");
let expected = packaged
.lines()
.filter(|l| !l.starts_with("TryExec="))
.map(|l| if l.starts_with("Exec=") { "Exec=\"/target/debug/filmcraft\" %F" } else { l })
.collect::<Vec<_>>()
.join("\n")
+ "\n";
let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", "/target/debug/filmcraft");
assert_eq!(body, expected);
}
/// Both layers are escaped, like install.sh: shell first (`` ` `` `"` `$` `\` get a
/// backslash), then *every* backslash doubled for the file value — including the escape
/// backslashes just added.
#[test]
fn exec_escapes_both_desktop_entry_layers() {
assert_eq!(escape_exec_argument("/opt/film craft/filmcraft"), "/opt/film craft/filmcraft");
// a\b → shell a\\b → string a\\\\b
assert_eq!(escape_exec_argument("a\\b"), "a\\\\\\\\b");
// a$b → shell a\$b → string a\\$b
assert_eq!(escape_exec_argument("a$b"), "a\\\\$b");
}
/// A written entry reads back as the exact binary it was written for, through both layers.
#[test]
fn exec_round_trips_through_both_layers() {
for exe in ["/home/dev/Projects/filmcraft/target/debug/filmcraft", "/opt/film craft/filmcraft", "/a$b\"c`d\\e"] {
let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", exe);
assert_eq!(exec_line(&body).as_deref(), Some(exe));
assert!(desktop_file_exec_is_ours(&body, exe));
}
}
/// Only the exact executable counts: another checkout's path, a path with our path as a
/// prefix, and a packaged `Exec=filmcraft` are all someone else's entry.
#[test]
fn desktop_file_exec_is_ours_matches_only_our_binary() {
let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", "/a/filmcraft");
assert!(!desktop_file_exec_is_ours(&body, "/a/filmcraft-old"));
assert!(!desktop_file_exec_is_ours(&body, "/b/filmcraft"));
assert!(!desktop_file_exec_is_ours("Exec=filmcraft %F\n", "/a/filmcraft"));
assert!(!desktop_file_exec_is_ours("Name=filmcraft\n", "/a/filmcraft"));
assert!(!desktop_file_exec_is_ours("Exec=\"dangling\n", "/a/filmcraft"));
}
/// The string layer is undone before the argument is split: `\s` becomes a space, which
/// ends a bare argument (the shell layer's quoting, not `\s`, is how a space travels), and
/// `\\` is one backslash. install.sh always quotes, so a space in our own `Exec` never
/// splits.
#[test]
fn exec_reads_the_string_layer() {
assert_eq!(exec_line("Exec=/a\\sb %F\n").as_deref(), Some("/a"));
assert_eq!(exec_line("Exec=\"/a\\sb\" %F\n").as_deref(), Some("/a b"));
assert_eq!(exec_line("Exec=/a\\\\b %F\n").as_deref(), Some("/a\\b"));
}
}

46
apps/filmcraft/src/main.rs
View File

@ -28,6 +28,8 @@ mod args;
mod audio;
mod audio_in;
mod control_server;
#[cfg(target_os = "linux")]
mod dev_icon;
mod file_filters;
#[cfg(any(target_os = "windows", test))]
mod graphics;
@ -108,12 +110,15 @@ fn main() -> eframe::Result {
}
}
app_nap::disable();
#[cfg(target_os = "linux")]
dev_icon::ensure_dev_desktop_entry();
// Panics anywhere go to <data dir>/Logs/crash-<day>.log with a backtrace; the UI pass and
// frame workers catch them and keep running (see filmcraft_ui_egui::crash).
filmcraft_ui_egui::crash::install(log_dir);
// OS hardware video decoders (VideoToolbox on macOS) in front of our own; Settings ▸ Playback ▸
// Hardware decoding switches them off. Unsupported streams and failures use our decoders.
register_hardware_decoders();
register_gpu_frame_renderer();
let options = eframe::NativeOptions {
viewport: egui::ViewportBuilder::default()
.with_title("FilmCraft")
@ -239,6 +244,11 @@ fn main() -> eframe::Result {
app.hooks.run_dialog = Some(Box::new(run_dialog_off_thread));
}
app.hooks.open_path = Some(Box::new(open_path));
// Help menu, About and Discord links (#642): eframe is built without its `links`
// feature, so `ctx.open_url` does nothing on the desktop. Detached, so a launcher
// that waits for the browser never freezes the UI.
app.hooks.open_url = Some(Box::new(|url: &str| open::that_detached(url).map_err(|e| e.to_string())));
app.hooks.clipboard_media = Some(Box::new(clipboard_media));
// Settings ▸ General ▸ Interface Language ▸ System Language (#218).
app.hooks.cursor_screen_position = Some(Box::new(filmcraft_platform::cursor::cursor_screen_position));
app.hooks.system_languages = Some(Box::new(|| sys_locale::get_locales().collect()));
@ -285,6 +295,18 @@ fn main() -> eframe::Result {
started
}
/// Files copied in the file manager, else an image, on the system clipboard (Paste, #611).
fn clipboard_media() -> Option<filmcraft_ui_egui::ClipboardMedia> {
let mut clipboard = arboard::Clipboard::new().ok()?;
if let Ok(files) = clipboard.get().file_list()
&& !files.is_empty()
{
return Some(filmcraft_ui_egui::ClipboardMedia::Files(files.iter().map(|p| p.to_string_lossy().into_owned()).collect()));
}
let image = clipboard.get_image().ok()?;
Some(filmcraft_ui_egui::ClipboardMedia::Image { width: image.width, height: image.height, rgba: image.bytes.into_owned() })
}
/// Open a file in its default application, or reveal it in the file manager (Edit Original,
/// Reveal Log Files).
fn open_path(path: &str, reveal: bool) -> Result<(), String> {
@ -312,6 +334,30 @@ fn open_path(path: &str, reveal: bool) -> Result<(), String> {
cmd.spawn().map(|_| ()).map_err(|e| format!("can't open {path}: {e}"))
}
/// The GPU export frame renderer (filmcraft-gpu's off-screen compositor behind filmcraft-export's
/// frame-renderer hook): exports with GPU rendering Auto composite on the GPU and fall back to the
/// CPU reference renderer wherever it cannot.
struct GpuFrameRenderer(filmcraft_gpu::ExportRenderer);
impl filmcraft_export::FrameRenderer for GpuFrameRenderer {
fn render(
&mut self,
project: &filmcraft_project::Project,
seq: filmcraft_project::ItemId,
t: filmcraft_time::Tick,
opts: filmcraft_render::RenderOptions,
sources: &dyn filmcraft_render::SourceProvider,
) -> Option<filmcraft_render::Image> {
self.0.render(project, seq, t, opts, sources)
}
}
fn register_gpu_frame_renderer() {
filmcraft_export::register_frame_renderer(|| {
filmcraft_gpu::ExportRenderer::new().map(|r| Box::new(GpuFrameRenderer(r)) as Box<dyn filmcraft_export::FrameRenderer>)
});
}
/// Put the OS hardware video decoders in front of our own. Registered in a statement of its own:
/// a log macro does not evaluate its arguments while no logger takes its level, which left the
/// hardware decoders out of every run.

15
contributors/people.toml
View File

@ -22,6 +22,11 @@ real_name = "Brandon Thomas"
display_name = "Brandon Thomas"
sync_github_name = false
[people.ahmedhosnypro]
real_name = "Ahmed Hosny"
display_name = "Ahmed Hosny"
sync_github_name = false
[people.sanyamsingh001]
display_name = "sanyamsingh001"
sync_github_name = false
@ -51,3 +56,13 @@ sync_github_name = false
[people.liangforstudy]
sync_github_name = true
[people.maxm11]
real_name = "Max Murphy"
display_name = "Max Murphy"
sync_github_name = false
[people.vangel76]
real_name = "Valentin Klein"
display_name = "Valentin Klein"
sync_github_name = false

2
crates/audio-dsp/src/channels.rs
View File

@ -2,7 +2,7 @@
//!
//! **Channel order.** 5.1 audio is always in the SMPTE / WAVE order **L, R, C, LFE, Ls, Rs**
//! ([`L`], [`R`], [`C`], [`LFE`], [`LS`], [`RS`]); stereo is L, R; mono is one channel. Codecs with a
//! different native order (AAC: C, L, R, Ls, Rs, LFE) reorder at the encoder boundary.
//! different native order (AAC: C, L, R, Ls, Rs, LFE) reorder at the codec boundary.
//!
//! **Downmix** follows ITU-R BS.775 (Table 2, "3/2 source" rows), with `k = 1/√2` (−3 dB):
//!

45
crates/captions/src/burn.rs
View File

@ -49,13 +49,10 @@ impl Overlay {
}
}
/// The caption face: Inter SemiBold at `px`.
fn caption_style(px: f32) -> TextStyle {
TextStyle { style: "SemiBold".into(), size: px, ..Default::default() }
}
fn measure(text: &str, px: f32) -> f32 {
filmcraft_text::measure(text, &caption_style(px))
/// The caption face: the track's font family (Inter when empty or not installed) in SemiBold at `px`.
fn caption_style(font: &str, px: f32) -> TextStyle {
let family = if font.trim().is_empty() { filmcraft_text::fonts::DEFAULT_FAMILY } else { font.trim() };
TextStyle { family: family.into(), style: "SemiBold".into(), size: px, ..Default::default() }
}
fn lin(c: [u8; 4]) -> [f32; 4] {
@ -64,12 +61,12 @@ fn lin(c: [u8; 4]) -> [f32; 4] {
}
/// Word-wrap one line to `max_w` pixels.
fn wrap(line: &str, px: f32, max_w: f32) -> Vec<String> {
fn wrap(line: &str, ts: &TextStyle, max_w: f32) -> Vec<String> {
let mut out = Vec::new();
let mut cur = String::new();
for word in line.split(' ') {
let cand = if cur.is_empty() { word.to_string() } else { format!("{cur} {word}") };
if !cur.is_empty() && measure(&cand, px) > max_w {
if !cur.is_empty() && filmcraft_text::measure(&cand, ts) > max_w {
out.push(std::mem::take(&mut cur));
cur = word.to_string();
} else {
@ -115,16 +112,17 @@ pub fn render_caption(c: &Caption, style: &CaptionStyle, w: usize, h: usize) ->
let scale = h as f32 / 1080.0;
let px = (style.size * scale).max(4.0);
let max_w = w as f32 * 0.9;
let ts = caption_style(&style.font, px);
let mut lines: Vec<String> = Vec::new();
// (speaker names are metadata; like Premiere, they are not burned in)
for l in c.plain_lines() {
lines.extend(wrap(l.trim(), px, max_w));
lines.extend(wrap(l.trim(), &ts, max_w));
}
lines.retain(|l| !l.is_empty());
if lines.is_empty() {
return None;
}
let vm = filmcraft_text::fonts::face(filmcraft_text::resolve("Inter", "SemiBold").face).metrics(px);
let vm = filmcraft_text::fonts::face(filmcraft_text::resolve(&ts.family, &ts.style).face).metrics(px);
let (asc, desc) = (vm.ascent, vm.descent);
let lh = px * style.line_spacing.max(0.8);
let pad_x = (px * 0.3).round();
@ -141,7 +139,7 @@ pub fn render_caption(c: &Caption, style: &CaptionStyle, w: usize, h: usize) ->
.max(0.0)
.round();
let side = w as f32 * 0.05;
let widths: Vec<f32> = lines.iter().map(|l| measure(l, px)).collect();
let widths: Vec<f32> = lines.iter().map(|l| filmcraft_text::measure(l, &ts)).collect();
let xs: Vec<f32> = widths
.iter()
.map(|&lw| match align {
@ -176,7 +174,7 @@ pub fn render_caption(c: &Caption, style: &CaptionStyle, w: usize, h: usize) ->
}
}
let baseline = lt + (lh - (asc + desc)) / 2.0 + asc;
let l = layout(line, &caption_style(px), &ParagraphStyle::default());
let l = layout(line, &ts, &ParagraphStyle::default());
let mut m = filmcraft_text::Mask { w: ow, h: oh, a: std::mem::take(&mut cover) };
render::draw(&l, &render::at(xs[i], baseline.round()), &mut m, (ox, oy));
cover = m.a;
@ -305,6 +303,27 @@ mod tests {
assert!(o.px.chunks(4).any(|p| p[0] > 0.9 && p[1] < 0.05 && p[3] > 0.9), "red outline pixels");
}
#[test]
fn track_font_is_used() {
// the style's font used to be ignored (always Inter)
let inter = render_caption(&cap("Hello world"), &CaptionStyle::default(), 1920, 1080).unwrap();
let serif = CaptionStyle { font: "Noto Serif".into(), ..Default::default() };
let serif = render_caption(&cap("Hello world"), &serif, 1920, 1080).unwrap();
assert_ne!((inter.w, &inter.px), (serif.w, &serif.px), "Noto Serif renders like Inter");
let mono = CaptionStyle { font: "jetbrains mono".into(), ..Default::default() };
let mono = render_caption(&cap("Hello world"), &mono, 1920, 1080).unwrap();
assert_ne!((inter.w, &inter.px), (mono.w, &mono.px), "JetBrains Mono renders like Inter");
}
#[test]
fn unknown_or_empty_font_falls_back_to_inter() {
let inter = render_caption(&cap("Hello"), &CaptionStyle::default(), 1280, 720).unwrap();
for font in ["", " ", "No Such Font 123"] {
let st = CaptionStyle { font: font.into(), ..Default::default() };
assert_eq!(render_caption(&cap("Hello"), &st, 1280, 720), Some(inter.clone()), "font {font:?}");
}
}
#[test]
fn hidden_track_draws_nothing() {
let mut t = CaptionTrack::new(TrackId(1), "S".into(), CaptionFormat::Subtitle);

5
crates/captions/src/lib.rs
View File

@ -307,7 +307,8 @@ pub fn parse_clock(s: &str) -> Option<Tick> {
/// Format ticks as `HH:MM:SS<sep>mmm` (rounded to the nearest millisecond).
pub fn format_clock(t: Tick, sep: char) -> String {
let ms = (t.0.max(0) + TICKS_PER_MS / 2) / TICKS_PER_MS;
// The rounding addition must not overflow for an input-derived Tick near i64::MAX.
let ms = (i128::from(t.0.max(0)) + i128::from(TICKS_PER_MS) / 2) / i128::from(TICKS_PER_MS);
let (h, m, s, f) = (ms / 3_600_000, (ms / 60_000) % 60, (ms / 1000) % 60, ms % 1000);
format!("{h:02}:{m:02}:{s:02}{sep}{f:03}")
}
@ -368,6 +369,8 @@ mod tests {
assert_eq!(parse_clock("00:61:00,000"), None);
assert_eq!(parse_clock("aa:00:00,000"), None);
assert_eq!(format_clock(Tick(TICKS_PER_SECOND * 3723 + TICKS_PER_MS * 45), ','), "01:02:03,045");
assert!(format_clock(Tick(i64::MAX), ',').ends_with(",077"));
assert_eq!(format_clock(Tick(i64::MIN), '.'), "00:00:00.000");
}
#[test]

39
crates/captions/src/stl.rs
View File

@ -215,15 +215,22 @@ fn encode_text(text: &str) -> Vec<u8> {
if ch == '<'
&& let Some(end) = rest.find('>')
{
match rest[1..end].trim().to_ascii_lowercase().as_str() {
"i" => out.push(0x80),
"/i" => out.push(0x81),
"u" => out.push(0x82),
"/u" => out.push(0x83),
_ => {}
let tag = &rest[1..end];
// Only a syntactically plausible tag is markup. Comparisons such as
// "2 < 3 > 1" and "x < y > z" must not lose their middle text.
let name = tag.strip_prefix('/').unwrap_or(tag);
let is_tag = name.as_bytes().first().is_some_and(u8::is_ascii_alphabetic);
if is_tag {
match tag.trim().to_ascii_lowercase().as_str() {
"i" => out.push(0x80),
"/i" => out.push(0x81),
"u" => out.push(0x82),
"/u" => out.push(0x83),
_ => {}
}
rest = &rest[end + 1..];
continue;
}
rest = &rest[end + 1..];
continue;
}
if ch == '\n' {
out.push(0x8a);
@ -451,6 +458,22 @@ mod tests {
assert!(parse(&b[..500]).is_err());
}
#[test]
fn literal_angle_brackets_are_not_discarded_as_markup() {
let literal = "2 < 3 > 1, x < y > z, and 5 <7> 6";
let doc = Document {
cues: vec![Cue { start: FrameRate::FPS_25.tick_of(25), end: FrameRate::FPS_25.tick_of(75), text: literal.into(), ..Default::default() }],
..Default::default()
};
let encoded = write(&doc, Some(FrameRate::FPS_25));
let decoded = parse(&encoded).expect("valid EBU STL");
assert_eq!(decoded.cues.len(), 1);
assert_eq!(decoded.cues[0].text, literal);
// Recognized styling still becomes STL control bytes.
assert_eq!(encode_text("<i>Hi</i>"), vec![0x80, b'H', b'i', 0x81]);
assert_eq!(encode_text("<i >Hi</i >"), vec![0x80, b'H', b'i', 0x81]);
}
#[test]
fn programme_start_is_subtracted() {
let doc = Document {

48
crates/captions/src/ttml.rs
View File

@ -252,6 +252,10 @@ fn parse_time(s: &str, tm: Timing) -> Option<Tick> {
let (h, m) = (num(parts.first()?)?, num(parts.get(1)?)?);
let sec_part = parts.get(2)?;
let (si, sf) = sec_part.split_once('.').unwrap_or((sec_part, ""));
let seconds = num(si)?;
if m >= 60 || seconds >= 60 {
return None;
}
let mut t = (h * 3600 + m * 60) * tps + decimal_ticks(si, sf, tps)?;
if let Some(fp) = parts.get(3) {
if parts.len() > 4 {
@ -259,13 +263,15 @@ fn parse_time(s: &str, tm: Timing) -> Option<Tick> {
}
let (fi, fs) = fp.split_once('.').unwrap_or((fp, ""));
let frames = num(fi)?;
let _ = tm.base;
let sub: i128 = if fs.is_empty() { 0 } else { num(fs)? };
if frames >= i128::from(tm.base) || sub >= i128::from(tm.sub_frame_rate) {
return None;
}
// frames at the effective rate; sub-frames divide a frame
let ft = tm.rate.tick_of(1).0 as i128;
t += frames * ft + sub * ft / tm.sub_frame_rate.max(1) as i128;
}
return Some(Tick(t as i64));
return i64::try_from(t).ok().map(Tick);
}
let unit_start = s.find(|c: char| c.is_ascii_alphabetic())?;
let (v, unit) = s.split_at(unit_start);
@ -281,8 +287,12 @@ fn parse_time(s: &str, tm: Timing) -> Option<Tick> {
"f" => {
let n: i128 = vi.parse().ok()?;
let fd = tm.rate.tick_of(1).0 as i128;
let whole = tm.rate.tick_of(n as i64).0 as i128;
whole + decimal_ticks("0", vf, fd)?
let whole = tm.rate.tick_of(i64::try_from(n).ok()?);
// tick_of saturates; a saturated count overflowed rather than named a real time
if whole == Tick(i64::MAX) {
return None;
}
whole.0 as i128 + decimal_ticks("0", vf, fd)?
}
"t" => {
let n: i128 = vi.parse().ok()?;
@ -290,7 +300,7 @@ fn parse_time(s: &str, tm: Timing) -> Option<Tick> {
}
_ => return None,
};
Some(Tick(t as i64))
i64::try_from(t).ok().map(Tick)
}
#[derive(Clone, Copy, Default, PartialEq)]
@ -552,6 +562,34 @@ mod tests {
assert_eq!(parse_time("1:2", t), None);
}
#[test]
fn invalid_and_overflowed_time_expressions_are_ignored() {
let normal = tm(FrameRate::FPS_25);
for expr in [
"999999999999999h",
"999999999999999m",
"999999999999999s",
"999999999999999ms",
"999999999999999f",
"999999999999999t",
"00:60:00",
"00:00:60",
"00:00:01:25",
] {
assert_eq!(parse_time(expr, normal), None, "{expr}");
}
let subframes = Timing { sub_frame_rate: 2, ..normal };
assert!(parse_time("00:00:01:12.1", subframes).is_some());
assert_eq!(parse_time("00:00:01:12.2", subframes), None);
// Bad cues must not wrap to a plausible but unrelated timeline position.
let xml =
r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div><p begin="999999999999999s" end="2s">Bad</p><p begin="1s" end="2s">Good</p></div></body></tt>"#;
let doc = parse(xml).expect("well-formed TTML");
assert_eq!(doc.cues.len(), 1);
assert_eq!(doc.cues[0].text, "Good");
assert!(!doc.warnings.is_empty());
}
#[test]
fn reads_styles_spans_and_nesting() {
let x = r#"<?xml version="1.0"?>

15
crates/cfb/src/read.rs
View File

@ -84,7 +84,6 @@ impl<'a> CompoundFile<'a> {
let mini_cutoff = u32_at(data, 56) as u64;
let first_minifat = u32_at(data, 60);
let first_difat = u32_at(data, 68);
let n_difat = u32_at(data, 72) as usize;
let mut header_clsid = [0u8; 16];
header_clsid.copy_from_slice(&data[8..24]);
let max_sectors = data.len() / sector_size + 1;
@ -108,8 +107,10 @@ impl<'a> CompoundFile<'a> {
let mut seen = 0;
while d <= MAXREGSECT && fat_sectors.len() < n_fat {
seen += 1;
if seen > n_difat.max(1) + max_sectors {
return Err(invalid("DIFAT chain loops"));
// a DIFAT chain cannot have more sectors than the file holds; the header's DIFAT sector
// count (up to 2^32 - 1) is not trusted, or a self-pointing sector loops that often
if seen > max_sectors {
return Err(invalid(format!("DIFAT chain loops: more than {max_sectors} sectors in a {}-byte file", data.len())));
}
let s = cf.sector(d)?;
let per = sector_size / 4 - 1;
@ -169,13 +170,17 @@ impl<'a> CompoundFile<'a> {
let mut out = Vec::new();
let mut s = start;
let mut steps = 0usize;
// A chain cannot visit more sectors than the file holds. Not `self.fat.len()`: the FAT can
// be far longer (one FAT sector named by many header / DIFAT slots), and a looping chain
// would then append a whole sector per FAT entry (hundreds of MB, or an abort).
let max_steps = self.data.len() / self.sector_size + 1;
while s != ENDOFCHAIN {
if s > MAXREGSECT {
return Err(invalid(format!("bad sector {s:#x} in a chain")));
}
steps += 1;
if steps > self.fat.len().max(1) {
return Err(invalid("sector chain loops"));
if steps > max_steps {
return Err(invalid(format!("sector chain loops: more than {max_steps} sectors in a {}-byte file", self.data.len())));
}
let b = self.sector(s)?;
out.extend_from_slice(b);

92
crates/cfb/src/tests.rs
View File

@ -199,3 +199,95 @@ fn mutation_never_panics_or_hangs() {
}
}
}
fn put_u32(b: &mut [u8], at: usize, v: u32) {
b[at..at + 4].copy_from_slice(&v.to_le_bytes());
}
/// A 512-byte header (§2.2) with sector shift `shift`: no directory, mini FAT or DIFAT chain, and
/// all 109 FAT slots free. Tests fill in the fields they attack.
fn bare_header(shift: u16) -> Vec<u8> {
let mut h = vec![0u8; 512];
h[..8].copy_from_slice(&SIGNATURE);
h[24..26].copy_from_slice(&0x3Eu16.to_le_bytes());
h[26..28].copy_from_slice(&(if shift == 9 { 3u16 } else { 4 }).to_le_bytes());
h[28..30].copy_from_slice(&0xFFFEu16.to_le_bytes());
h[30..32].copy_from_slice(&shift.to_le_bytes());
h[32..34].copy_from_slice(&6u16.to_le_bytes());
for at in [48, 60, 68] {
put_u32(&mut h, at, ENDOFCHAIN);
}
put_u32(&mut h, 56, MINI_STREAM_CUTOFF);
h[76..].fill(0xFF);
h
}
/// A DIFAT sector that names no FAT sector and points to itself. The loop guard used to be the
/// header's DIFAT sector count (up to 2^32 - 1), so this 1 KB file kept `open` (and so AAF
/// sniffing) busy for 14-16 s here (minutes with `u32::MAX`), reading a sector each pass. A DIFAT
/// chain cannot be longer than the file, so the walk now stops after as many passes as the file
/// has sectors (1024 / 512 + 1 = 3 here) and fails on the 4th.
#[test]
fn difat_self_loop_fails_fast() {
let mut b = bare_header(9);
put_u32(&mut b, 44, 1); // one FAT sector, to be found through the DIFAT
put_u32(&mut b, 68, 0); // first DIFAT sector
put_u32(&mut b, 72, 100_000_000); // claimed DIFAT sector count (inflates the old loop bound)
let mut difat = vec![0xFF; 512]; // no FAT sector numbers...
put_u32(&mut difat, 508, 0); // ...and the next DIFAT sector is this one
b.extend_from_slice(&difat);
// The message names the bound the walk stopped at: the file's 3 sectors, not the header's
// 100,000,000 (the old guard allowed 100,000,003 passes).
let expect = format!("DIFAT chain loops: more than 3 sectors in a {}-byte file", b.len());
assert_eq!(CompoundFile::open(&b).unwrap_err(), Error::Invalid(expect));
}
/// All 109 header slots name the same FAT sector, so the FAT is 109 copies of it, and the
/// directory chain loops (sector 0 → 0). The loop guard used to be the FAT length, so this 450 KB
/// file appended a sector per FAT entry before failing: a 457 MB buffer per open (117 GB with
/// 64 KB sectors in a 7 MB file). A chain cannot visit more sectors than the file has.
#[test]
fn duplicated_fat_sector_chain_loop_fails_fast() {
let ss = 4096;
let mut b = bare_header(12);
put_u32(&mut b, 44, 109); // FAT sectors: the 109 header slots...
for i in 0..109 {
put_u32(&mut b, 76 + i * 4, 0); // ...all naming sector 0
}
put_u32(&mut b, 48, 0); // the directory starts at sector 0
b.resize(ss, 0);
let mut fat = vec![0xFF; ss];
put_u32(&mut fat, 0, 0); // FAT[0] = 0: sector 0 is followed by itself
b.extend_from_slice(&fat);
b.resize(110 * ss, 0); // big enough for 109 FAT sectors to pass the size check
// The message names the bound the walk stopped at: 111, this file's 110 sectors + 1, not the
// 109 * 1024 = 111,616 entries of the inflated FAT that the old guard allowed (a 457 MB buffer).
let expect = format!("sector chain loops: more than 111 sectors in a {}-byte file", b.len());
assert_eq!(CompoundFile::open(&b).unwrap_err(), Error::Invalid(expect));
}
/// Positive control for the chain bound: the longest chains a file can hold still read back, in
/// both versions: a directory (read without a size limit) and a stream, each filling every sector
/// but the header and the FAT (and the directory, for the stream).
#[test]
fn chains_filling_the_file_still_read() {
for v in [Version::V3, Version::V4] {
let ss = v_sector(v);
let mut w = Writer::new(v);
for i in 0..1000 {
w.stream(Writer::ROOT, &format!("e{i}"), Vec::new()).unwrap();
}
let b = w.finish();
assert!(b.len() / ss <= (1001 * 128usize).div_ceil(ss) + 3, "header, directory, FAT");
let cf = CompoundFile::open(&b).unwrap();
assert_eq!(cf.root().children.len(), 1000);
assert_eq!(cf.read_path("e999").unwrap(), Vec::<u8>::new());
let mut w = Writer::new(v);
let data = pattern(64 * ss, 3);
w.stream(Writer::ROOT, "big", data.clone()).unwrap();
let b = w.finish();
assert_eq!(b.len(), (1 + 64 + 2) * ss, "header, stream, directory, FAT");
assert_eq!(CompoundFile::open(&b).unwrap().read_path("big").unwrap(), data);
}
}

88
crates/codecs/src/audio.rs
View File

@ -16,6 +16,7 @@ use crate::{CodecError, Result};
enum Inner {
/// Our own AAC-LC decoder. `up` interpolates HE-AAC's core to the output rate (see [`Upsample2x`]).
/// Surround output is reordered to WAV/SMPTE order (see [`aac_to_wav_order`]).
Aac { dec: Box<filmcraft_aac::Decoder>, asc: Vec<u8>, up: Option<Box<Upsample2x>> },
/// AAC whose configuration arrives with the first frame (ADTS / LATM).
AacPending,
@ -52,6 +53,29 @@ fn vorbis_to_wav_order(channels: usize) -> Option<&'static [usize]> {
})
}
/// Channel order for AAC surround (ISO/IEC 14496-3 Table 1.19, our decoder's element order: C, L R,
/// surrounds, LFE): output channel → decoded channel, giving the WAV/SMPTE order (L R C LFE Ls Rs)
/// used elsewhere, as the AC-3 decoder does and the AAC encoder's caller undoes on export. Only the
/// unambiguous channel configurations 3–6; 1, 2, 7 and PCE layouts keep the decoded order.
fn aac_to_wav_order(channel_config: u8) -> Option<&'static [usize]> {
Some(match channel_config {
3 => &[1, 2, 0],
4 => &[1, 2, 0, 3],
5 => &[1, 2, 0, 3, 4],
6 => &[1, 2, 0, 5, 3, 4],
_ => return None,
})
}
/// `channels` reordered by `order` (output channel → decoded channel). A decode whose channel count
/// does not match the order is returned unchanged.
fn reorder(mut channels: Vec<Vec<f32>>, order: Option<&[usize]>) -> Vec<Vec<f32>> {
match order {
Some(order) if order.len() == channels.len() => order.iter().map(|&c| channels.get_mut(c).map(std::mem::take).unwrap_or_default()).collect(),
_ => channels,
}
}
/// Duration of an Opus packet in 48 kHz samples from its TOC and frame count (RFC 6716 §3.1),
/// without parsing frame lengths. For a multistream packet this is the first stream's TOC, which
/// every stream shares. `None` for an empty or invalid (> 120 ms) packet.
@ -152,18 +176,17 @@ impl PacketDecoder {
if let Some(up) = up {
out = up.process(out);
}
let out = reorder(out, aac_to_wav_order(dec.config().channel_config));
self.channels = out.len();
return Ok(out);
}
Inner::Opus { dec, order } => {
// A corrupt packet is concealed like a lost one (keeps timing and decoder state).
let mut out = match dec.decode(Some(data)) {
let out = match dec.decode(Some(data)) {
Ok(o) => o,
Err(_) => dec.decode(None).map_err(|e| CodecError::Decode(e.to_string()))?,
};
if let Some(order) = order {
out = order.iter().map(|&c| std::mem::take(&mut out[c])).collect();
}
let out = reorder(out, *order);
self.channels = out.len();
return Ok(out);
}
@ -811,6 +834,63 @@ mod he_aac_tests {
}
}
#[cfg(test)]
mod surround_order_tests {
use super::*;
/// Power of `x` at `hz` (Goertzel), normalised by length.
fn power(x: &[f32], hz: f32, rate: f32) -> f32 {
let k = 2.0 * (std::f32::consts::TAU * hz / rate).cos();
let (mut s1, mut s2) = (0.0f32, 0.0f32);
for &v in x {
let s0 = v + k * s1 - s2;
s2 = s1;
s1 = s0;
}
(s1 * s1 + s2 * s2 - k * s1 * s2) / x.len() as f32
}
/// #721: 5.1 AAC (configuration 6, coded C L R Ls Rs LFE) decodes to WAV/SMPTE order
/// L R C LFE Ls Rs, as the rest of FilmCraft and the AAC export path expect.
#[test]
fn aac_51_decodes_in_wav_order() {
let rate = 48_000u32;
// tones per speaker, in WAV order: L, R, C, LFE, Ls, Rs
let hz = [300.0f32, 400.0, 500.0, 60.0, 700.0, 800.0];
let tone = |f: f32| (0..rate as usize).map(|i| 0.25 * (std::f32::consts::TAU * f * i as f32 / rate as f32).sin()).collect::<Vec<f32>>();
let wav: Vec<Vec<f32>> = hz.iter().map(|&f| tone(f)).collect();
// the encoder takes AAC element order (as the exporter feeds it): C L R Ls Rs LFE
let coded: Vec<&[f32]> = [2usize, 0, 1, 4, 5, 3].iter().map(|&c| wav[c].as_slice()).collect();
let mut enc = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(rate, 6, 384_000)).unwrap();
let mut units = enc.encode(&coded);
units.extend(enc.flush());
let mut dec = PacketDecoder::aac(&enc.audio_specific_config(), rate).unwrap();
let mut out = vec![Vec::new(); 6];
for u in &units {
for (o, c) in out.iter_mut().zip(dec.decode(u, 0).unwrap()) {
o.extend(c);
}
}
assert_eq!(dec.channels, 6);
for (ch, samples) in out.iter().enumerate() {
let window = &samples[4800..38_400];
let own = power(window, hz[ch], rate as f32);
for (other, &f) in hz.iter().enumerate().filter(|&(o, _)| o != ch) {
let p = power(window, f, rate as f32);
assert!(own > 100.0 * p, "channel {ch}: {} Hz {own} vs {f} Hz (channel {other}) {p}", hz[ch]);
}
}
}
#[test]
fn reorder_leaves_mismatched_channel_counts_alone() {
let chans = vec![vec![1.0], vec![2.0]];
assert_eq!(reorder(chans.clone(), aac_to_wav_order(6)), chans);
assert_eq!(reorder(vec![vec![0.0], vec![1.0], vec![2.0]], aac_to_wav_order(3)), vec![vec![1.0], vec![2.0], vec![0.0]]);
}
}
#[cfg(test)]
mod packet_time_tests {
use super::*;

57
crates/codecs/src/hw_frame.rs
View File

@ -82,6 +82,13 @@ pub(crate) fn obus(data: &[u8]) -> Vec<(u8, &[u8])> {
out
}
/// What an AV1 sequence header fixes about the pictures: profile, maximum size and the colour
/// configuration (bit depth, chroma format, colour description). `None` when it does not parse.
fn av1_format(seq: &[u8]) -> Option<(u8, u32, u32, filmcraft_av1::ColorConfig)> {
let h = filmcraft_av1::SequenceHeader::parse(seq).ok()?;
Some((h.profile, h.max_frame_width, h.max_frame_height, h.color))
}
/// VP9 colour as our decoder reports it: `color_space` and range from the bitstream, transfer and
/// primaries from the container when it says. (Shared with `Vp9Decoder`.)
pub(crate) fn vp9_color(width: u32, height: u32, color_space: u8, full_range: bool, transfer: Option<Transfer>, primaries: Option<Primaries>) -> ColorInfo {
@ -243,14 +250,58 @@ impl FrameStreamInfo {
}
/// Whether `sample` declares parameters the hardware session was not set up for: an AV1
/// sequence header that differs from `av1C`'s, a VP9 key frame of another size, bit depth or
/// chroma format. The decoder then hands the stream to our software decoder.
/// sequence header whose picture format (profile, size, colour configuration) differs from
/// `av1C`'s, a VP9 key frame of another size, bit depth or chroma format. The decoder then
/// hands the stream to our software decoder.
pub fn parameters_changed(&self, sample: &[u8]) -> bool {
match self.codec {
FrameCodec::Av1 => obus(sample).into_iter().any(|(t, p)| t == 1 && self.sequence_header.as_deref() != Some(p)),
// Only what the picture format depends on counts: encoders such as SVT-AV1 put a
// provisional sequence header in `av1C` whose coding tool flags (CDEF, restoration,
// warped motion…) differ from the in-band one, and decoders follow the in-band one.
FrameCodec::Av1 => obus(sample).into_iter().any(|(t, p)| t == 1 && self.sequence_header.as_deref().is_none_or(|s| av1_format(s) != av1_format(p))),
FrameCodec::Vp9 => {
self.picture_params(sample).is_some_and(|p| p.size != self.size || p.bit_depth != self.bit_depth || p.subsampling != self.subsampling)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// An `av1C` sequence header from SVT-AV1 (1280x720 10-bit) and the in-band one of the same
/// stream: they differ only in coding tool flags.
const AV1C: [u8; 14] = [0x0a, 0x0c, 0x02, 0x00, 0x00, 0x2d, 0x6a, 0x67, 0xfd, 0x9e, 0x01, 0x7c, 0x20, 0x20];
const IN_BAND: [u8; 14] = [0x0a, 0x0c, 0x02, 0x00, 0x00, 0x2d, 0x6a, 0x67, 0xfd, 0x9e, 0x35, 0x7c, 0xe0, 0x20];
fn info() -> FrameStreamInfo {
let c = filmcraft_isobmff::Av1Config {
seq_profile: 0,
seq_level_idx_0: 0,
seq_tier_0: false,
high_bitdepth: true,
twelve_bit: false,
monochrome: false,
chroma_subsampling_x: true,
chroma_subsampling_y: true,
chroma_sample_position: 0,
initial_presentation_delay_minus_one: None,
config_obus: AV1C.to_vec(),
};
FrameStreamInfo::from_av1(&c, (0, 0)).unwrap()
}
#[test]
fn av1_sequence_headers_compare_by_picture_format() {
let i = info();
assert_eq!((i.size, i.bit_depth), ((1280, 720), 10));
assert!(!i.parameters_changed(&AV1C), "the same header");
// used to send every SVT-AV1 MP4 to the software decoder from its first frame
assert!(!i.parameters_changed(&IN_BAND), "only coding tool flags differ");
// a header that does not parse is a change
assert!(i.parameters_changed(&[0x0a, 0x02, 0xe0, 0x00]));
// no sequence header in the sample: nothing changed
assert!(!i.parameters_changed(&[0x12, 0x00]));
}
}

1
crates/codecs/src/lib.rs
View File

@ -30,6 +30,7 @@ pub mod mxf;
pub mod ogg;
mod stream_color;
pub mod video;
mod webm_alpha;
use std::sync::{Arc, RwLock};

150
crates/codecs/src/mkv.rs
View File

@ -58,6 +58,8 @@ pub struct MkvSource {
/// Video codec as an ISO-BMFF sample entry (for the decoder factories).
ventry: Option<SampleEntry>,
video: GopCache,
/// The alpha layer of a VP9 track with `AlphaMode` 1 (WebM transparency, #402).
alpha: Option<Arc<crate::webm_alpha::AlphaLayer>>,
/// The playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
audios: Vec<MkvAudio>,
}
@ -102,10 +104,9 @@ fn audio_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, rate: i64) -> Vec
fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_opus::OpusHead, rate: i64) -> Vec<i64> {
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
let (n, d) = tb(t);
let scale_ns = (n as i128 * 1_000_000_000 / d as i128).max(1);
let delay_ns = t.codec_delay_ns as i128;
// The demuxer's rounding of CodecDelay to ticks (half away from zero), undone here.
let delay_ticks = ((delay_ns + scale_ns / 2) / scale_ns) as i64;
// The demuxer's rounding of CodecDelay to ticks, undone here.
let delay_ticks = codec_delay_ticks(t);
let skip = if t.codec_delay_ns > 0 { (delay_ns * rate as i128 / 1_000_000_000) as i64 } else { head.pre_skip as i64 };
let at = |pts: i64| ((pts.saturating_add(delay_ticks) as i128 * n as i128 * rate as i128 / d as i128) as i64).saturating_sub(skip);
let packets = t.samples.iter().enumerate().map(|(i, s)| crate::audio::PacketTime {
@ -116,6 +117,46 @@ fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_o
crate::audio::contiguous_starts(packets, stamp_tolerance(t, rate))
}
/// `CodecDelay` in timestamp ticks, rounded as the demuxer rounds it (half away from zero) before
/// subtracting it from every timestamp of the track.
fn codec_delay_ticks(t: &filmcraft_matroska::Track) -> i64 {
let (n, d) = tb(t);
let scale_ns = (n as i128 * 1_000_000_000 / d as i128).max(1);
i64::try_from((t.codec_delay_ns as i128 + scale_ns / 2) / scale_ns).unwrap_or(i64::MAX)
}
/// The track timestamp that media time zero stands for (#715): the earliest start among the
/// playable tracks. Block timestamps count from the start of the Segment, and a remux that keeps
/// its source's timestamps (`ffmpeg -copyts`, a cut from a recording) starts its first picture
/// seconds in; taken as media time, that repeated the opening picture until then. An audio track
/// starts where its first packet does before `CodecDelay` was taken off (the delay only hides
/// priming), and never before zero.
fn media_origin(file: &MkvFile, vtrack: Option<usize>, atracks: &[usize]) -> i64 {
let start = |i: usize| {
let t = file.tracks.get(i)?;
let first = t.samples.iter().map(|s| s.pts).min()?;
Some(if t.kind == TrackKind::Audio { first.saturating_add(codec_delay_ticks(t)).max(0) } else { first })
};
vtrack.into_iter().chain(atracks.iter().copied()).filter_map(start).min().unwrap_or(0).max(0)
}
/// Move every track back by `origin` timestamp ticks, so the streams keep their relative timing
/// (a delay between picture and sound survives).
fn rebase(file: &mut MkvFile, origin: i64) {
if origin <= 0 {
return;
}
for s in file.tracks.iter_mut().flat_map(|t| t.samples.iter_mut()) {
s.pts = s.pts.saturating_sub(origin);
}
}
/// End of the last sample of track `i`, in its timebase.
fn track_end(file: &MkvFile, i: usize) -> Option<(i64, &filmcraft_matroska::Track)> {
let t = file.tracks.get(i)?;
Some((t.samples.iter().map(|s| s.pts.saturating_add(i64::try_from(s.duration).unwrap_or(i64::MAX))).max().unwrap_or(0), t))
}
/// Two timestamp ticks in sample frames at `rate`, plus one: how far a packet's Matroska timestamp
/// can be from its exact start.
fn stamp_tolerance(t: &filmcraft_matroska::Track, rate: i64) -> i64 {
@ -144,7 +185,7 @@ fn tb(t: &filmcraft_matroska::Track) -> (i64, i64) {
fn to_tick(t: &filmcraft_matroska::Track, pts: i64) -> Tick {
let (n, d) = tb(t);
Tick::from_rational(pts * n, 1, d)
Tick::from_rational(pts, n, d)
}
/// A time as the track timestamp to look a frame up by: rounded to the nearest tick, as muxers
@ -273,7 +314,7 @@ impl MkvSource {
/// Open from a random-access reader: only the index is read now, samples on demand.
pub fn open_reader(name: &str, reader: filmcraft_media::SharedReader) -> crate::Result<Self> {
let bytes = crate::Src(reader);
let file = filmcraft_matroska::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?;
let mut file = filmcraft_matroska::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?;
let vtrack = file.tracks.iter().position(|t| t.kind == TrackKind::Video && !t.samples.is_empty());
// every audio track is a stream (the track count comes from the file: capped)
let atracks: Vec<usize> = file
@ -287,6 +328,24 @@ impl MkvSource {
if vtrack.is_none() && atracks.is_empty() {
return Err(CodecError::Unsupported("no playable tracks".into()));
}
let main = vtrack.or(atracks.first().copied());
// The Segment's end (`Info/Duration` or the last sample), in Segment time.
let segment_end = match file.duration_ns() {
Some(ns) => Tick::from_rational(i64::try_from(ns).unwrap_or(i64::MAX), 1, 1_000_000_000),
None => main.and_then(|i| track_end(&file, i)).map(|(end, t)| to_tick(t, end)).unwrap_or_default(),
};
// Media time zero is the first picture (or sound): the streams start at their common origin.
let origin = media_origin(&file, vtrack, &atracks);
rebase(&mut file, origin);
// `Info/Duration` is the Segment's end: the media lasts from the origin to there (never less
// than its main track's samples).
let duration = match main.and_then(|i| track_end(&file, i)) {
Some((end, t)) if origin > 0 => {
let (n, d) = tb(t);
(segment_end - Tick::from_rational(origin, n, d)).max(to_tick(t, end))
}
_ => segment_end,
};
let mut explicit_color = None;
let mut ventry = None;
// display rotation from the track's Projection (portrait phone video is stored landscape)
@ -310,7 +369,7 @@ impl MkvSource {
if explicit {
explicit_color = Some(color);
}
let secs = file.duration_ns().unwrap_or(0) as f64 / 1e9;
let secs = duration.seconds();
let bitrate = (secs > 0.0).then(|| (t.samples.iter().map(|s| s.size as u64).sum::<u64>() as f64 * 8.0 / secs) as u64);
let ((w, h), par) = if rotation % 2 == 1 { ((h, w), (v.pixel_aspect().1, v.pixel_aspect().0)) } else { ((w, h), v.pixel_aspect()) };
VideoStreamInfo {
@ -340,17 +399,6 @@ impl MkvSource {
AudioStreamInfo { sample_rate: rate, channels, codec: codec_label(&t.codec), bits_per_sample: a.bit_depth.map(|b| b as u32) }
})
.collect();
let duration = match file.duration_ns() {
Some(ns) => Tick::from_rational(ns as i64, 1, 1_000_000_000),
None => vtrack
.or(atracks.first().copied())
.map(|i| {
let t = &file.tracks[i];
let end = t.samples.iter().map(|s| s.pts + s.duration as i64).max().unwrap_or(0);
to_tick(t, end)
})
.unwrap_or_default(),
};
let info = MediaInfo {
name: name.to_string(),
kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly },
@ -380,7 +428,12 @@ impl MkvSource {
Some(MkvAudio { track: i, state, starts, preroll })
})
.collect();
Ok(Self { info, bytes, file, vtrack, ventry, video: GopCache::new(explicit_color).with_rotation(rotation), audios })
// WebM transparency: VP9 alpha in each block's BlockAdditional (ID 1)
let alpha = vtrack.and_then(|i| file.tracks.get(i)).filter(|t| {
matches!(t.codec, Codec::Vp9 { .. }) && t.video.as_ref().is_some_and(|v| v.alpha_mode != 0) && t.samples.iter().any(|s| s.addition.is_some())
});
let alpha = alpha.map(|t| Arc::new(crate::webm_alpha::AlphaLayer::new(bytes.clone(), t.samples.iter().map(|s| (s.pts, s.addition)))));
Ok(Self { info, bytes, file, vtrack, ventry, video: GopCache::new(explicit_color).with_rotation(rotation), alpha, audios })
}
fn read(&self, track: usize, i: usize) -> crate::Result<Vec<u8>> {
@ -486,9 +539,15 @@ impl VideoSamples for MkvVideo<'_> {
self.src.read(self.track, i)
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
match &self.src.ventry {
Some(e) => make_video_decoder(e),
None => Err(CodecError::Unsupported(format!("no decoder for {} video", codec_label(&self.src.file.tracks[self.track].codec)))),
let Some(e) = &self.src.ventry else {
return Err(CodecError::Unsupported(format!("no decoder for {} video", codec_label(&self.src.file.tracks[self.track].codec))));
};
let color = make_video_decoder(e)?;
let Some(layer) = &self.src.alpha else { return Ok(color) };
// the alpha layer always goes through our own decoder (it is luma only)
match crate::software_video_decoder(e) {
Ok(alpha) => Ok(Box::new(crate::webm_alpha::AlphaDecoder::new(color, alpha, layer.clone()))),
Err(_) => Ok(color),
}
}
}
@ -601,3 +660,52 @@ mod multi_audio_tests {
assert!(source.audio_stream(2, 0, 50, 48_000).is_err());
}
}
#[cfg(test)]
mod origin_tests {
use filmcraft_frame::PixelData;
use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};
use filmcraft_media::{FrameRequest, MediaSource};
use filmcraft_time::Tick;
/// A flat grey Motion-JPEG picture: its level says which frame it is.
fn picture(k: u8) -> Vec<u8> {
let img = image::GrayImage::from_pixel(16, 16, image::Luma([20 + 20 * k]));
let mut out = Vec::new();
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, 95).encode_image(&img).unwrap();
out
}
/// #715: a remux that keeps its source's timestamps (first picture at 1.48 s, sound 0.2 s after
/// it) starts at its first picture, lasts as long as its pictures, and keeps the sound's delay.
/// It repeated picture 0 until 1.48 s, and the clip ran to the Segment's absolute end.
#[test]
fn timestamp_preserving_remux_starts_at_the_first_picture() {
let mut v = TrackSpec::new(TrackKind::Video, "V_MJPEG");
v.video_size = Some((16, 16));
v.default_duration_ns = Some(40_000_000);
let mut a = TrackSpec::new(TrackKind::Audio, "A_PCM/INT/LIT");
a.audio = Some((48_000.0, 1, Some(16)));
let mut w = MkvWriter::new(std::io::Cursor::new(Vec::new()), vec![v, a], MuxOptions::default()).unwrap();
for k in 0..10u8 {
let pts = 1_480_000_000 + i64::from(k) * 40_000_000;
w.write_frame(0, pts, true, &picture(k), None).unwrap();
if k == 5 {
let pcm: Vec<u8> = (0..960).flat_map(|_| 16_384i16.to_le_bytes()).collect();
w.write_frame(1, 1_680_000_000, true, &pcm, Some(20_000_000)).unwrap();
}
}
let src = super::MkvSource::open("shifted.mkv", w.finish().unwrap().into_inner().into()).unwrap();
assert_eq!(src.info().duration, Tick::from_rational(10, 1, 25), "ten pictures at 25 fps");
let level = |k: i64| {
let f = src.video_frame(FrameRequest::full(Tick::from_rational(k, 1, 25))).unwrap();
let PixelData::Rgba8(px) = &f.data else { panic!("MJPEG decodes to RGBA8") };
px[0]
};
let seen: Vec<u8> = (0..10).map(level).collect();
assert!(seen.windows(2).all(|p| p[0] < p[1]), "one picture per frame, in order: {seen:?}");
let sound = src.audio(0, 48_000 * 2 / 5, 48_000).unwrap();
assert!(sound.channels[0][9_590].abs() < 0.001, "silent before the delay");
assert!((sound.channels[0][9_610] - 0.5).abs() < 0.001, "sound 0.2 s after the first picture");
}
}

147
crates/codecs/src/mp4.rs
View File

@ -9,7 +9,7 @@ use std::sync::{Arc, Mutex};
use filmcraft_color::ColorInfo;
use filmcraft_frame::{AudioBuffer, Region, VideoFrame};
use filmcraft_isobmff::{CleanAperture, CodecConfig, Mp4File, TrackKind};
use filmcraft_isobmff::{ByteSource, CleanAperture, CodecConfig, Mp4File, TrackKind};
use filmcraft_media::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, SharedSource, VideoStreamInfo};
use filmcraft_time::{FrameRate, Tick};
@ -162,6 +162,48 @@ fn color_from(entry: &filmcraft_isobmff::SampleEntry, w: u32, h: u32) -> ColorIn
crate::stream_color::resolve(w, h, &sources)
}
/// First bytes of a Sony `rtmd` sample needed for the start-timecode block (`frames` is last).
const RTMD_HEADER_LEN: usize = 0x12;
/// Sony XAVC `rtmd` (real-time metadata) start timecode fields, read from the first sample: hours,
/// minutes, seconds, a drop-frame flag and frames. Sony stores them as raw decimal bytes in the
/// sample's fixed header. The layout is proprietary and undocumented, so it is reproduced here from
/// real footage; it is the same block ffmpeg reports as the `timecode` tag of an `rtmd` stream.
///
/// Fields outside SMPTE bounds — including the labels drop-frame counting skips — are rejected
/// (no fabricated start time from damaged metadata).
fn rtmd_fields(sample: &[u8], rate: FrameRate) -> Option<(i64, i64, i64, i64, bool)> {
let f = sample.get(0x0d..0x12)?;
let (h, m, s) = (f[0] as i64, f[1] as i64, f[2] as i64);
let frames = f[4] as i64;
let drop = f[3] != 0;
let base = rate.timecode_base();
if h > 23 || m > 59 || s > 59 || frames >= base {
return None;
}
// Drop-frame counting skips the first `base / 15` frame labels of every minute except each
// tenth minute, so those labels never occur in valid footage (SMPTE ST 12-1).
if drop && rate.supports_drop_frame() && s == 0 && m % 10 != 0 && frames < base / 15 {
return None;
}
Some((h, m, s, frames, drop))
}
/// Start timecode from a Sony `rtmd` track as a frame count at `rate` (issue #460), used when the
/// file has no `tmcd` track. `None` if there is no `rtmd` track or it has no usable first sample.
/// Only the fixed-size header is read (a damaged `stsz` size can't drive a huge allocation).
fn rtmd_start_timecode(file: &Mp4File, bytes: &crate::Src, rate: FrameRate) -> Option<i64> {
let t = file.tracks.iter().find(|t| t.entries.first().is_some_and(|e| e.format.0 == *b"rtmd"))?;
let s = t.samples.first()?;
if s.size < RTMD_HEADER_LEN as u32 {
return None;
}
let mut buf = [0u8; RTMD_HEADER_LEN];
bytes.read_at(s.offset, &mut buf).ok()?;
let (h, m, s, f, drop) = rtmd_fields(&buf, rate)?;
Some(filmcraft_time::fields_to_frames(h, m, s, f, rate, drop))
}
impl Mp4Source {
pub fn open(name: &str, bytes: Arc<[u8]>) -> crate::Result<Self> {
Self::open_reader(name, Arc::new(filmcraft_media::reader::MemReader(bytes)))
@ -170,7 +212,7 @@ impl Mp4Source {
/// Open from a random-access reader: only the index is read now, samples on demand.
pub fn open_reader(name: &str, reader: filmcraft_media::SharedReader) -> crate::Result<Self> {
let bytes = crate::Src(reader);
let file = filmcraft_isobmff::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?;
let mut file = filmcraft_isobmff::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?;
// A track needs samples, a sample description and a timescale to be playable (damaged
// files can lack them: indexing `entries[0]` or dividing by the timescale used to panic).
let playable = |t: &&filmcraft_isobmff::Track, kind| t.kind == kind && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0;
@ -271,6 +313,9 @@ impl Mp4Source {
Some(CodecConfig::Timecode(tc)) => tc.start_frame.map(|f| f as i64),
_ => None,
});
// Sony XAVC files with no `tmcd` track carry the start timecode in an `rtmd` metadata
// track; convert its fields to a frame count using the video frame rate.
let start_timecode = start_timecode.or_else(|| rtmd_start_timecode(&file, &bytes, video.as_ref()?.frame_rate));
let info = MediaInfo {
name: name.to_string(),
kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly },
@ -281,7 +326,7 @@ impl Mp4Source {
start_timecode,
file_size: Some(bytes.0.len()),
};
let audios: Vec<Mp4Audio> = atracks
let mut audios: Vec<Mp4Audio> = atracks
.iter()
.zip(&info.audio_streams)
.filter_map(|(&i, ainfo)| {
@ -300,6 +345,7 @@ impl Mp4Source {
Some(Mp4Audio { track: i, state, starts, offset, preroll })
})
.collect();
rebase_unedited_video(&mut file, vtrack, &mut audios);
Ok(Self { info, bytes, file, vtrack, video: GopCache::new(explicit_color).with_rotation(rotation).with_crop(aperture), audios })
}
@ -605,6 +651,35 @@ pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource, Media
Some(Mp4Source::open(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
/// Start the file at its first picture when its video track has no edit list (#714). Without an
/// edit list the media timeline is the presentation, so a B-frame stream's first picture sits at its
/// composition delay (two frames on an x264 MP4 written without `elst`, and on fragmented MP4s):
/// media time zero repeated it and the clip, which lasts the media duration, lost its last pictures.
/// Every track moves back by the earliest start among them, video and audio alike, so the streams
/// stay in sync and a delay between them keeps its meaning; a track with an edit list is never
/// rebased on its own.
fn rebase_unedited_video(file: &mut Mp4File, vtrack: Option<usize>, audios: &mut [Mp4Audio]) {
let Some(vi) = vtrack else { return };
let Some(t) = file.tracks.get(vi).filter(|t| t.edits.is_empty() && t.timescale > 0) else { return };
let Some(first) = t.samples.iter().map(|s| s.pts).min().map(|p| p.saturating_add(t.edit_offset)).filter(|&p| p > 0) else { return };
let vts = i64::from(t.timescale);
let video_start = Tick::from_rational(first, 1, vts);
let audio_ts = |a: &Mp4Audio| file.tracks.get(a.track).map_or(1, |t| i64::from(t.timescale.max(1)));
// an audio track starts at its offset (a negative one only hides priming before zero)
let shift = audios.iter().fold(video_start, |m, a| m.min(Tick::from_rational(a.offset.max(0), 1, audio_ts(a))));
if shift <= Tick::ZERO {
return;
}
for a in audios.iter_mut() {
let d = shift.to_rational_round(1, audio_ts(a));
a.offset = a.offset.saturating_sub(d);
}
let v = if shift == video_start { first } else { shift.to_rational_round(1, vts) };
if let Some(t) = file.tracks.get_mut(vi) {
t.edit_offset = t.edit_offset.saturating_sub(v);
}
}
/// How long a track plays. With an edit list that is the sum of its edits (ISO/IEC 14496-12
/// §8.6.6, movie timescale), as `tkhd` records it. The media duration (`mdhd`) runs on the decode
/// timeline: it is longer than the presentation by the B-frame delay an edit skips (`media_time`, one
@ -700,6 +775,30 @@ mod tests {
assert_eq!(frames(&Mp4Source::open("zero.mov", mov_with_edits(zero)).expect("open")), 4);
}
/// #714: a video track without an edit list whose pictures carry a composition delay (B-frame
/// style, as x264 writes them with no `elst`) starts at its first picture. Media time zero
/// repeated picture 0 for the length of the delay and the last pictures never showed.
#[test]
fn unedited_composition_delay_starts_at_the_first_picture() {
let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).expect("writer");
let t = mux.add_track(TrackConfig::new(SampleEntry::prores(FourCc(*b"apch"), 64, 32), 25)).expect("track");
for k in 0..4u16 {
let mut fr = filmcraft_prores::Frame::new(64, 32, filmcraft_prores::ChromaFormat::Yuv422, 10, false);
fr.y.fill(100 + 200 * k);
let data = filmcraft_prores::Encoder::new(filmcraft_prores::Profile::Hq, 64, 32).encode(&fr).expect("encode");
mux.write_sample(t, WriteSample { data: &data, duration: 1, composition_offset: 2, is_sync: true }).expect("sample");
}
let src = Mp4Source::open("delay.mov", mux.finish().expect("finish").into_inner().into()).expect("open");
assert_eq!(src.info().duration, Tick::from_rational(4, 1, 25));
let luma = |k: i64| {
let f = src.video_frame(FrameRequest::full(Tick::from_rational(k, 1, 25))).expect("frame");
let l = f.luma8();
l.iter().map(|&v| u32::from(v)).sum::<u32>() / l.len().max(1) as u32
};
let seen: Vec<u32> = (0..4).map(luma).collect();
assert!(seen.windows(2).all(|w| w[0] < w[1]), "one picture per frame, in order: {seen:?}");
}
/// Overwrite the big-endian u32 `skip` bytes after the first `fourcc` box type.
fn patch_u32(b: &mut [u8], fourcc: &[u8; 4], skip: usize, v: u32) {
let k = b.windows(4).position(|x| x == fourcc).expect("box") + 4 + skip;
@ -914,6 +1013,48 @@ mod tests {
assert_eq!((v.pixel_format.as_str(), v.has_alpha), (want, alpha), "{}", String::from_utf8_lossy(fourcc));
}
}
#[test]
fn rtmd_fields_parse_and_reject_damaged_metadata() {
let rate = FrameRate::FPS_59_94;
// 03:37:12:34, non-drop (a Sony FX3/a6400 XAVC header).
let mut s = [0u8; 24];
s[0x0d..0x12].copy_from_slice(&[3, 37, 12, 0, 34]);
assert_eq!(rtmd_fields(&s, rate), Some((3, 37, 12, 34, false)));
s[0x10] = 1;
assert_eq!(rtmd_fields(&s, rate), Some((3, 37, 12, 34, true)));
// Too short to hold the block: no fields, no panic.
assert_eq!(rtmd_fields(&[0u8; 6], rate), None);
// Damaged fields are rejected rather than normalized into a plausible time.
for bad in [[3, 37, 12, 0, 60], [3, 60, 0, 0, 0], [24, 0, 0, 0, 0], [0, 0, 60, 0, 0]] {
let mut b = [0u8; 24];
b[0x0d..0x12].copy_from_slice(&bad);
assert_eq!(rtmd_fields(&b, rate), None, "{bad:?}");
}
}
#[test]
fn rtmd_fields_reject_drop_frame_skipped_labels() {
let df = |h, m, s, f| {
let mut b = [0u8; 24];
b[0x0d..0x12].copy_from_slice(&[h, m, s, 1, f]);
rtmd_fields(&b, FrameRate::FPS_59_94)
};
// At 59.94 DF, frames 00-03 of a non-tenth minute are skipped, so they never occur.
assert_eq!(df(0, 1, 0, 0), None);
assert_eq!(df(0, 1, 0, 3), None);
assert_eq!(df(0, 1, 0, 4), Some((0, 1, 0, 4, true)));
assert_eq!(df(0, 10, 0, 0), Some((0, 10, 0, 0, true))); // tenth minute: label exists
assert_eq!(df(0, 1, 1, 0), Some((0, 1, 1, 0, true))); // seconds > 0: label exists
// At 29.97 DF only frames 00-01 are skipped.
let df30 = |m, s, f| {
let mut b = [0u8; 24];
b[0x0d..0x12].copy_from_slice(&[0, m, s, 1, f]);
rtmd_fields(&b, FrameRate::FPS_29_97)
};
assert_eq!(df30(1, 0, 0), None);
assert_eq!(df30(1, 0, 2), Some((0, 1, 0, 2, true)));
}
}
#[cfg(test)]

62
crates/codecs/src/tests.rs
View File

@ -387,6 +387,68 @@ fn mkv_vp9_422_seeks_bit_exact() {
check_vp9_seeks("noise_vp9_422.mkv", "Matroska", "yuv422p", &["-profile:v", "1", "-deadline", "realtime", "-speed", "8"], 30);
}
/// #402: a WebM whose VP9 track carries an alpha layer (`AlphaMode` 1, a second VP9 stream in
/// each block's `BlockAdditional`) decodes with that alpha. Frames requested out of order (seeks
/// across key frames, then forward) match libvpx's decode exactly, colour and alpha.
#[test]
fn vp9_webm_alpha_layer_matches_libvpx() {
let (w, h, frames) = (176usize, 144usize, 30usize);
// the alpha changes every frame, so a picture paired with the wrong alpha frame shows
let Some(path) = fixture_path(
"alpha_vp9.webm",
&[
"-f",
"lavfi",
"-i",
"testsrc2=s=176x144:r=25,format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='mod(X*2+N*7\\,256)'",
"-frames:v",
"30",
"-c:v",
"libvpx-vp9",
"-pix_fmt",
"yuva420p",
"-g",
"10",
"-deadline",
"realtime",
"-speed",
"8",
"-b:v",
"600k",
],
) else {
return;
};
// the reference: ffmpeg's libvpx decoder (its native VP9 decoder ignores the alpha layer)
let src_path = path.to_string_lossy().into_owned();
let Some(reference) =
fixture("alpha_vp9.webm.yuva420p.yuv", &["-c:v", "libvpx-vp9", "-i", &src_path, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "yuva420p"])
else {
return;
};
let (luma, chroma) = (w * h, (w / 2) * (h / 2));
let frame_len = luma * 2 + chroma * 2;
if reference.len() != frame_len * frames {
eprintln!("libvpx reference has {} bytes, not {}; skipping", reference.len(), frame_len * frames);
return;
}
let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into();
let src = crate::open_bytes("alpha_vp9.webm", bytes).unwrap();
let info = src.info().clone();
assert_eq!(info.container, "WebM");
assert!(info.video.as_ref().unwrap().has_alpha);
let rate = info.frame_rate();
let order: Vec<usize> = [29, 3, 17, 12, 0, 25, 21, 9].into_iter().chain(10..frames).collect();
for k in order {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
let r = &reference[k * frame_len..(k + 1) * frame_len];
let filmcraft_frame::PixelData::Yuv8 { planes, alpha, .. } = &f.data else { panic!("frame {k}: not 8-bit YUV: {}", f.format_label()) };
assert_eq!(planes[0].as_slice(), &r[..luma], "frame {k}: luma");
let alpha = alpha.as_ref().unwrap_or_else(|| panic!("frame {k}: no alpha plane"));
assert_eq!(alpha.as_slice(), &r[luma + 2 * chroma..], "frame {k}: alpha");
}
}
/// Every sample flagged as sync (as in an MP4 without `stss`): the GOP cache must still start
/// decoding at a real VP9 key frame.
#[test]

290
crates/codecs/src/webm_alpha.rs Normal file
View File

@ -0,0 +1,290 @@
//! WebM alpha (#402): a VP9 track with `AlphaMode` 1 carries, in each block's `BlockAdditional`
//! with `BlockAddID` 1, a second VP9 stream coded independently of the colour one. Its luma plane,
//! full range, is the frame's alpha (the WebM convention, as libvpx-based decoders read it).
//!
//! [`AlphaDecoder`] wraps the colour decoder: every sample it decodes, it also decodes that
//! sample's alpha frame with a second (software) VP9 decoder and attaches the luma plane to the
//! colour picture with the same timestamp, holding a colour picture back until its alpha is out. The alpha stream has key frames of its own, which need
//! not coincide with the colour ones: after a seek (or any other break in the order samples are
//! fed), the alpha decoder restarts from the nearest alpha key frame at or before the sample.
//!
//! Alpha is best effort: a missing, damaged or undecodable alpha frame leaves that picture opaque,
//! never fails the colour decode.
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use filmcraft_frame::{PixelData, VideoFrame};
use crate::video::{DecodedFrame, VideoDecoder};
/// How far back the alpha decoder looks for a key frame when it resynchronises.
const MAX_RESYNC: usize = 600;
/// Decoded alpha pictures waiting for their colour picture, and colour pictures waiting for their
/// alpha (VP9 pictures come out in order, so a few are plenty; past that a picture goes out opaque
/// rather than stall).
const MAX_PENDING: usize = 8;
/// Where each sample's alpha frame is stored.
pub(crate) struct AlphaLayer {
src: crate::Src,
/// Per sample, in file (decode) order.
additions: Vec<Option<(u64, u32)>>,
pts: Vec<i64>,
/// First sample with each pts.
by_pts: HashMap<i64, usize>,
}
impl AlphaLayer {
/// `samples`: (pts, addition) of every sample of the track, in file order.
pub(crate) fn new(src: crate::Src, samples: impl Iterator<Item = (i64, Option<(u64, u32)>)>) -> Self {
let (mut additions, mut all_pts) = (Vec::new(), Vec::new());
let mut by_pts = HashMap::new();
for (i, (pts, add)) in samples.enumerate() {
additions.push(add);
all_pts.push(pts);
by_pts.entry(pts).or_insert(i);
}
Self { src, additions, pts: all_pts, by_pts }
}
fn read(&self, i: usize) -> Option<Vec<u8>> {
let (offset, size) = (*self.additions.get(i)?)?;
let end = offset.checked_add(size as u64)?;
if end > self.src.0.len() {
return None;
}
let mut data = vec![0; size as usize];
self.src.0.read_at(offset, &mut data).ok()?;
Some(data)
}
}
/// A colour decoder plus the decoder of its alpha layer.
///
/// Both decoders may hand pictures back later than the sample that made them (frame threading),
/// and not necessarily after the same call. A colour picture is therefore held until its alpha
/// picture has come out, or until it is certain that it won't: the alpha decoder has moved past
/// it (pictures come out in order), it failed, or the sample has no alpha frame.
pub(crate) struct AlphaDecoder {
color: Box<dyn VideoDecoder>,
alpha: Box<dyn VideoDecoder>,
layer: Arc<AlphaLayer>,
/// Sample the alpha decoder expects next; `None` after a reset or a failure.
next: Option<usize>,
/// Decoded alpha pictures not yet paired, by pts.
pending: VecDeque<(i64, VideoFrame)>,
/// Latest pts the alpha decoder has produced since the last reset.
alpha_max: Option<i64>,
/// Colour pictures waiting for their alpha, in output order.
held: VecDeque<DecodedFrame>,
}
impl AlphaDecoder {
pub(crate) fn new(color: Box<dyn VideoDecoder>, alpha: Box<dyn VideoDecoder>, layer: Arc<AlphaLayer>) -> Self {
Self { color, alpha, layer, next: None, pending: VecDeque::new(), alpha_max: None, held: VecDeque::new() }
}
fn keep(&mut self, frames: Vec<DecodedFrame>) {
for f in frames {
self.alpha_max = Some(self.alpha_max.map_or(f.pts, |m| m.max(f.pts)));
self.pending.push_back((f.pts, f.frame));
}
while self.pending.len() > MAX_PENDING {
self.pending.pop_front();
}
}
/// Decode the alpha frame of the sample with `pts`, first catching up from an alpha key frame
/// when the sample doesn't follow the previous one.
fn feed_alpha(&mut self, pts: i64) {
let Some(&i) = self.layer.by_pts.get(&pts) else {
self.next = None;
return;
};
if self.next != Some(i) {
self.alpha.reset();
self.pending.clear();
self.alpha_max = None;
let is_key = |k: usize| self.layer.read(k).is_some_and(|d| filmcraft_vp9::is_keyframe(&d));
let start = (i.saturating_sub(MAX_RESYNC)..=i).rev().find(|&k| is_key(k));
if let Some(start) = start {
for k in start..i {
let (Some(d), Some(&p)) = (self.layer.read(k), self.layer.pts.get(k)) else { continue };
match self.alpha.decode(&d, p) {
Ok(frames) => self.keep(frames),
Err(_) => break,
}
}
}
}
self.next = Some(i + 1);
let Some(d) = self.layer.read(i) else { return };
match self.alpha.decode(&d, pts) {
Ok(frames) => self.keep(frames),
Err(e) => {
log::debug!("WebM alpha frame at pts {pts}: {e}");
self.next = None;
}
}
}
/// Whether the alpha picture for `pts` can still come out of the alpha decoder.
fn alpha_may_come(&self, pts: i64) -> bool {
let has_frame = self.layer.by_pts.get(&pts).is_some_and(|&i| self.layer.additions.get(i).is_some_and(|a| a.is_some()));
has_frame && self.next.is_some() && self.alpha_max.is_none_or(|m| m < pts)
}
/// Release held colour pictures, in order, as far as their alpha is settled. `all`: release
/// every one (flush), with or without alpha.
fn release(&mut self, all: bool) -> Vec<DecodedFrame> {
let mut out = Vec::new();
while let Some(f) = self.held.front() {
let pts = f.pts;
let found = self.pending.iter().position(|(p, _)| *p == pts);
if found.is_none() && !all && self.held.len() <= MAX_PENDING && self.alpha_may_come(pts) {
break;
}
let Some(mut f) = self.held.pop_front() else { break };
if let Some(k) = found
&& let Some((_, a)) = self.pending.remove(k)
{
set_alpha(&mut f.frame, &a);
}
out.push(f);
}
out
}
}
impl VideoDecoder for AlphaDecoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> crate::Result<Vec<DecodedFrame>> {
let out = self.color.decode(sample, pts)?;
self.held.extend(out);
self.feed_alpha(pts);
Ok(self.release(false))
}
fn flush(&mut self) -> Vec<DecodedFrame> {
let out = self.color.flush();
self.held.extend(out);
let rest = self.alpha.flush();
self.keep(rest);
self.release(true)
}
fn reset(&mut self) {
self.color.reset();
self.alpha.reset();
self.next = None;
self.pending.clear();
self.alpha_max = None;
self.held.clear();
}
fn name(&self) -> &str {
self.color.name()
}
fn intra_only(&self) -> bool {
self.color.intra_only()
}
fn is_random_access(&self, sample: &[u8]) -> Option<bool> {
self.color.is_random_access(sample)
}
fn is_disposable(&self, sample: &[u8]) -> bool {
self.color.is_disposable(sample)
}
fn set_draft(&mut self, on: bool) {
self.color.set_draft(on);
self.alpha.set_draft(on);
}
fn thread_limited(&self) -> bool {
self.color.thread_limited()
}
}
/// The luma plane of `a` (an alpha picture) as `frame`'s alpha, converted to `frame`'s bit depth.
/// Nothing happens when the sizes differ or either picture has no usable planes.
pub(crate) fn set_alpha(frame: &mut VideoFrame, a: &VideoFrame) {
if (frame.width, frame.height) != (a.width, a.height) {
return;
}
let n = frame.width as usize * frame.height as usize;
// alpha codes as 16-bit with their depth, so every pairing converts through one path
let (codes, abits): (Vec<u16>, u32) = match &a.data {
PixelData::Yuv8 { planes, .. } => (planes[0].iter().map(|&v| v as u16).collect(), 8),
PixelData::Yuv16 { planes, bits, .. } => (planes[0].to_vec(), (*bits).clamp(1, 16)),
_ => return,
};
if codes.len() != n {
return;
}
let rescale = |v: u16, to: u32| -> u32 {
let (from_max, to_max) = ((1u32 << abits) - 1, (1u32 << to.clamp(1, 16)) - 1);
(v as u32 * to_max + from_max / 2) / from_max
};
match &mut frame.data {
PixelData::Yuv8 { alpha, .. } => *alpha = Some(Arc::new(codes.iter().map(|&v| rescale(v, 8) as u8).collect())),
PixelData::Yuv16 { alpha, bits, .. } => {
let to = *bits;
*alpha = Some(Arc::new(codes.iter().map(|&v| rescale(v, to) as u16).collect()));
}
PixelData::Rgba8(rgba) => {
let px = Arc::make_mut(rgba);
for (p, &v) in px.as_chunks_mut::<4>().0.iter_mut().zip(&codes) {
p[3] = rescale(v, 8) as u8;
}
}
PixelData::RgbaF32(_) => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
use filmcraft_frame::Chroma;
fn yuv8(w: u32, h: u32, luma: Vec<u8>) -> VideoFrame {
let n = (w * h) as usize;
let c = n / 4;
let data = PixelData::Yuv8 { planes: [Arc::new(luma), Arc::new(vec![128; c]), Arc::new(vec![128; c])], chroma: Chroma::C420, alpha: None };
VideoFrame { width: w, height: h, data, color: Default::default(), par: (1, 1), pts: filmcraft_time::Tick::ZERO }
}
fn yuv16(w: u32, h: u32, bits: u32) -> VideoFrame {
let n = (w * h) as usize;
let data =
PixelData::Yuv16 { planes: [Arc::new(vec![0; n]), Arc::new(vec![0; n / 4]), Arc::new(vec![0; n / 4])], chroma: Chroma::C420, bits, alpha: None };
VideoFrame { width: w, height: h, data, color: Default::default(), par: (1, 1), pts: filmcraft_time::Tick::ZERO }
}
#[test]
fn alpha_luma_becomes_the_alpha_plane_at_the_frame_depth() {
let a = yuv8(2, 2, vec![0, 128, 255, 64]);
// 8-bit picture: the luma as is
let mut f = yuv8(2, 2, vec![9; 4]);
set_alpha(&mut f, &a);
let PixelData::Yuv8 { alpha, .. } = &f.data else { panic!() };
assert_eq!(alpha.as_deref(), Some(&vec![0, 128, 255, 64]));
// 10-bit picture: full range kept (255 → 1023)
let mut f = yuv16(2, 2, 10);
set_alpha(&mut f, &a);
let PixelData::Yuv16 { alpha, .. } = &f.data else { panic!() };
assert_eq!(alpha.as_deref(), Some(&vec![0, 514, 1023, 257]));
// RGBA picture (VP9 RGB profiles): the A channel
let mut f = VideoFrame::rgba8(2, 2, vec![7; 16]);
set_alpha(&mut f, &a);
let PixelData::Rgba8(px) = &f.data else { panic!() };
assert_eq!(px.chunks(4).map(|p| p[3]).collect::<Vec<_>>(), [0, 128, 255, 64]);
}
#[test]
fn an_alpha_picture_of_another_size_is_ignored() {
let mut f = yuv8(4, 2, vec![9; 8]);
set_alpha(&mut f, &yuv8(2, 2, vec![0; 4]));
let PixelData::Yuv8 { alpha, .. } = &f.data else { panic!() };
assert!(alpha.is_none());
// a luma plane shorter than the picture (damaged) is ignored too
let mut f = yuv8(2, 2, vec![9; 4]);
set_alpha(&mut f, &yuv8(2, 2, vec![0; 3]));
let PixelData::Yuv8 { alpha, .. } = &f.data else { panic!() };
assert!(alpha.is_none());
}
}

261
crates/codecs/tests/mp4_rtmd.rs Normal file
View File

@ -0,0 +1,261 @@
//! Sony XAVC MP4 with no `tmcd` track: the start timecode lives in an `rtmd` metadata track
//! (issue #460). A hand-built file exercises the whole probe path; the byte layout of the `rtmd`
//! sample is the one real XAVC footage uses (checked against ffprobe's `timecode` tag).
use std::sync::Arc;
use filmcraft_time::FrameRate;
/// Minimal ISO-BMFF box writer (the demuxer only needs headers and sample tables here).
#[derive(Default)]
struct B(Vec<u8>);
impl B {
fn u16(&mut self, v: u16) {
self.0.extend_from_slice(&v.to_be_bytes());
}
fn u32(&mut self, v: u32) {
self.0.extend_from_slice(&v.to_be_bytes());
}
fn bytes(&mut self, b: &[u8]) {
self.0.extend_from_slice(b);
}
fn zeros(&mut self, n: usize) {
self.0.resize(self.0.len() + n, 0);
}
/// A container box (`size` + `kind` + children).
fn container(&mut self, kind: &[u8; 4], f: impl FnOnce(&mut B)) {
let at = self.0.len();
self.u32(0);
self.bytes(kind);
f(self);
let len = (self.0.len() - at) as u32;
self.0[at..at + 4].copy_from_slice(&len.to_be_bytes());
}
/// A full box (`size` + `kind` + version/flags + body).
fn full(&mut self, kind: &[u8; 4], f: impl FnOnce(&mut B)) {
self.container(kind, |b| {
b.zeros(4);
f(b);
});
}
fn leaf(&mut self, kind: &[u8; 4], payload: &[u8]) {
self.container(kind, |b| b.bytes(payload));
}
}
/// A `stsd` visual sample entry (JPEG keeps the video track decodable enough to open).
fn video_entry(b: &mut B) {
b.bytes(&[0; 6]);
b.u16(1);
b.u16(0);
b.u16(0);
b.bytes(&[0; 4]);
b.u32(0);
b.u32(0);
b.u16(1920);
b.u16(1080);
b.u32(0x0048_0000);
b.u32(0x0048_0000);
b.u32(0);
b.u16(1);
b.bytes(&[0; 32]);
b.u16(24);
b.u16(0xFFFF);
}
/// A video track at 59.94 fps, plus a Sony `rtmd` track whose first sample is `rtmd_sample`.
fn file_with_rtmd(rtmd_sample: &[u8]) -> Vec<u8> {
file_with_rtmd_claimed(rtmd_sample, rtmd_sample.len() as u32)
}
/// As [`file_with_rtmd`] but the `rtmd` sample's `stsz` size is `claimed` (which may exceed the
/// bytes actually present, as a corrupt file can).
fn file_with_rtmd_claimed(rtmd_sample: &[u8], claimed: u32) -> Vec<u8> {
let mut b = B::default();
b.leaf(b"ftyp", b"isom\x00\x00\x00\x00isom");
let mdat_start = b.0.len();
let mdat: Vec<u8> = [&[1, 2, 3, 4][..], rtmd_sample].concat();
b.leaf(b"mdat", &mdat);
let video_off = (mdat_start + 8) as u32;
let rtmd_off = video_off + 4;
b.container(b"moov", |b| {
b.full(b"mvhd", |b| {
b.zeros(8);
b.u32(1000);
b.u32(1001);
b.zeros(80);
});
// Video track (id 1): one sample of 1001/60000 s → 59.94 fps.
b.container(b"trak", |b| {
b.full(b"tkhd", |b| {
b.zeros(8);
b.u32(1);
b.zeros(4);
b.u32(1001);
b.zeros(52);
b.u32(1920 << 16);
b.u32(1080 << 16);
});
b.container(b"mdia", |b| {
b.full(b"mdhd", |b| {
b.zeros(8);
b.u32(60000);
b.u32(1001);
b.u16(0x55C4);
b.u16(0);
});
b.full(b"hdlr", |b| {
b.zeros(4);
b.bytes(b"vide");
b.zeros(12);
b.bytes(b"Video\0");
});
b.container(b"minf", |b| {
b.container(b"stbl", |b| {
b.full(b"stsd", |b| {
b.u32(1);
b.container(b"jpeg", video_entry);
});
b.full(b"stts", |b| {
b.u32(1);
b.u32(1);
b.u32(1001);
});
b.full(b"stsc", |b| {
b.u32(1);
b.u32(1);
b.u32(1);
b.u32(1);
});
b.full(b"stsz", |b| {
b.u32(0);
b.u32(1);
b.u32(4);
});
b.full(b"stco", |b| {
b.u32(1);
b.u32(video_off);
});
});
});
});
});
// Sony rtmd metadata track (id 2).
b.container(b"trak", |b| {
b.full(b"tkhd", |b| {
b.zeros(8);
b.u32(2);
b.zeros(4);
b.u32(1);
b.zeros(52);
b.zeros(8);
});
b.container(b"mdia", |b| {
b.full(b"mdhd", |b| {
b.zeros(8);
b.u32(1);
b.u32(1);
b.u16(0x55C4);
b.u16(0);
});
b.full(b"hdlr", |b| {
b.zeros(4);
b.bytes(b"meta");
b.zeros(12);
b.bytes(b"Timed Metadata\0");
});
b.container(b"minf", |b| {
b.container(b"stbl", |b| {
b.full(b"stsd", |b| {
b.u32(1);
b.leaf(b"rtmd", &[0, 0, 0, 0, 0, 0, 0, 1]);
});
b.full(b"stts", |b| {
b.u32(1);
b.u32(1);
b.u32(1);
});
b.full(b"stsc", |b| {
b.u32(1);
b.u32(1);
b.u32(1);
b.u32(1);
});
b.full(b"stsz", |b| {
b.u32(0);
b.u32(1);
b.u32(claimed);
});
b.full(b"stco", |b| {
b.u32(1);
b.u32(rtmd_off);
});
});
});
});
});
});
b.0
}
/// A Sony `rtmd` sample: hours, minutes, seconds at 0x0d–0x0f, a drop-frame flag at 0x10 and the
/// frame number at 0x11, then the start of the SMPTE KLV key.
fn rtmd_sample(h: u8, m: u8, s: u8, f: u8, drop: u8) -> Vec<u8> {
let mut v = vec![0u8; 24];
v[0x0d] = h;
v[0x0e] = m;
v[0x0f] = s;
v[0x10] = drop;
v[0x11] = f;
v[0x12..0x16].copy_from_slice(&[0x06, 0x0e, 0x2b, 0x34]);
v
}
#[test]
fn rtmd_start_timecode_is_read() {
let file = file_with_rtmd(&rtmd_sample(3, 37, 12, 34, 0));
let src = filmcraft_codecs::open_bytes("fx3.mp4", Arc::from(file.into_boxed_slice())).unwrap();
let rate = FrameRate::new(60000, 1001);
let tc = src.info().start_timecode.expect("rtmd start timecode");
assert_eq!(tc, 781_954);
assert_eq!(filmcraft_time::format_timecode_frames(tc, rate, false), "03:37:12:34");
}
#[test]
fn rtmd_without_usable_sample_is_ignored() {
// Too short to carry the timecode block: no start timecode, and no panic.
let file = file_with_rtmd(&[0u8; 6]);
let src = filmcraft_codecs::open_bytes("short.mp4", Arc::from(file.into_boxed_slice())).unwrap();
assert_eq!(src.info().start_timecode, None);
}
#[test]
fn huge_claimed_rtmd_sample_does_not_exhaust_memory() {
// The stsz size claims 16 MiB but only the header bytes exist: only the fixed-size header is
// read, so a corrupt size cannot drive a huge allocation during open (issue #460 review).
let file = file_with_rtmd_claimed(&rtmd_sample(1, 2, 3, 4, 0), 16 * 1024 * 1024);
let src = filmcraft_codecs::open_bytes("big.mp4", Arc::from(file.into_boxed_slice())).unwrap();
assert_eq!(src.info().start_timecode, Some(223_384)); // 01:02:03:04 at 59.94
}
#[test]
fn damaged_timecode_fields_are_not_published() {
// Frame 60 at 59.94 fps (valid range is 0–59): unusable metadata, not a fabricated time.
let file = file_with_rtmd(&rtmd_sample(3, 37, 12, 60, 0));
let src = filmcraft_codecs::open_bytes("bad.mp4", Arc::from(file.into_boxed_slice())).unwrap();
assert_eq!(src.info().start_timecode, None);
}
#[test]
fn drop_frame_skipped_labels_are_not_published() {
// 00:01:00;00 at 59.94 fps: frames 00-03 of a non-tenth minute don't exist in drop-frame
// counting, so this metadata is unusable rather than a start time.
let file = file_with_rtmd(&rtmd_sample(0, 1, 0, 0, 1));
let src = filmcraft_codecs::open_bytes("df_bad.mp4", Arc::from(file.into_boxed_slice())).unwrap();
assert_eq!(src.info().start_timecode, None);
// The first real label of that minute (frame 04) is accepted.
let file = file_with_rtmd(&rtmd_sample(0, 1, 0, 4, 1));
let src = filmcraft_codecs::open_bytes("df_ok.mp4", Arc::from(file.into_boxed_slice())).unwrap();
assert_eq!(src.info().start_timecode, Some(3600));
}

51
crates/edit/src/lib.rs
View File

@ -487,17 +487,45 @@ pub fn move_items(seq: &mut Sequence, moves: &[(ClipId, TrackId, Tick)], insert_
it.start = (*start).max(Tick::ZERO);
placed.push((*dest, it));
}
// carry transitions with moved clips? Premiere drops transitions whose partner is not moved.
let carried = carried_transitions(&work, &placed);
// a transition whose partner stays behind is dropped, as in Premiere
delete_items(&mut work, &moves.iter().map(|m| m.0).collect::<Vec<_>>());
if insert_mode {
insert(&mut work, placed, ctx)?;
} else {
overwrite(&mut work, placed, ctx)?;
}
for (dest, tr) in carried {
let t = track_mut(&mut work, dest)?;
if [tr.from, tr.to].into_iter().flatten().all(|c| t.item(c).is_some()) {
t.transitions.push(tr);
t.sort();
}
}
*seq = work;
Ok(())
}
/// The transitions that travel with a move (#227, #374): those whose clips all move, onto one
/// track and by one offset (a fade at a moved clip's edge, or the cut between two clips that move
/// together), shifted to where those clips land.
fn carried_transitions(seq: &Sequence, placed: &[(TrackId, TrackItem)]) -> Vec<(TrackId, Transition)> {
let landing = |c: ClipId| {
let (dest, it) = placed.iter().find(|(_, it)| it.id == c)?;
let (_, old) = seq.find_item(c)?;
Some((*dest, it.start - old.start))
};
let mut carried = Vec::new();
for tr in seq.all_tracks().filter(|t| !t.locked).flat_map(|t| t.transitions.iter()) {
let lands: Option<Vec<(TrackId, Tick)>> = [tr.from, tr.to].into_iter().flatten().map(landing).collect();
let Some((&(dest, delta), rest)) = lands.as_deref().and_then(<[_]>::split_first) else { continue };
if rest.iter().all(|l| *l == (dest, delta)) {
carried.push((dest, Transition { start: tr.start + delta, ..tr.clone() }));
}
}
carried
}
fn media_len(ctx: &EditCtx, item: &TrackItem) -> Option<Tick> {
if item.frame_hold.is_some() {
return None;
@ -876,6 +904,27 @@ pub fn slide(seq: &mut Sequence, clip: ClipId, delta: Tick, ctx: &mut EditCtx) -
Ok(d)
}
/// Slide several items (a linked picture/sound pair, a selection) by one common delta: the most
/// restrictive item's neighbour limit clamps them all, so their relative timing is kept.
/// Returns the delta applied.
pub fn slide_items(seq: &mut Sequence, clips: &[ClipId], delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
let mut d = delta;
for c in clips {
let x = slide(&mut seq.clone(), *c, d, ctx)?;
if x.0.signum() != d.0.signum() {
d = Tick::ZERO;
} else if x.abs() < d.abs() {
d = x;
}
}
if d != Tick::ZERO {
for c in clips {
slide(seq, *c, d, ctx)?;
}
}
Ok(d)
}
/// Rate stretch: change an item's duration by dragging an edge, adjusting speed to keep the same media.
pub fn rate_stretch(seq: &mut Sequence, clip: ClipId, edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<f64> {
let (tid, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;

33
crates/edit/src/tests.rs
View File

@ -464,6 +464,39 @@ fn transitions_stay_on_their_cuts() {
fx.seq.check().unwrap();
}
#[test]
fn transitions_travel_with_moved_clips() {
// V1: a [0,10) fading in, b [10,20), c [20,30); b→c crossfade, c fading out
let mut fx = Fx::new();
let (v1, v2) = (fx.v(0), fx.v(1));
let a = fx.put(v1, 0, 10, 10);
let b = fx.put(v1, 10, 10, 10);
let c = fx.put(v1, 20, 10, 10);
let tr = |id, start, from, to| Transition {
id: TransitionId(id),
effect: filmcraft_project::find_effect("cross_dissolve").unwrap().instance(),
start: f(start),
duration: f(4),
from,
to,
align: Default::default(),
reverse: false,
};
fx.seq.track_mut(v1).unwrap().transitions = vec![tr(1, 0, None, Some(a)), tr(2, 18, Some(b), Some(c)), tr(3, 26, Some(c), None)];
let at = |fx: &Fx, track: TrackId| -> Vec<(u64, i64)> { fx.seq.track(track).unwrap().transitions.iter().map(|t| (t.id.0, R.frame_at(t.start))).collect() };
let mut n = fx.next;
// a fade at a clip's edge goes with it, to another track too
move_items(&mut fx.seq, &[(a, v2, f(40))], false, &mut Fx::ctx(&mut n)).unwrap();
assert_eq!((at(&fx, v1), at(&fx, v2)), (vec![(2, 18), (3, 26)], vec![(1, 40)]));
// a cut whose two clips move together keeps its transition, in an insert move as well
move_items(&mut fx.seq, &[(b, v1, f(60)), (c, v1, f(70))], true, &mut Fx::ctx(&mut n)).unwrap();
assert_eq!(at(&fx, v1), vec![(2, 68), (3, 76)]);
// a crossfade whose partner stays behind is dropped; the other clip's fade stays with it
move_items(&mut fx.seq, &[(c, v1, f(90))], false, &mut Fx::ctx(&mut n)).unwrap();
assert_eq!(at(&fx, v1), vec![(3, 96)]);
fx.seq.check().unwrap();
}
#[test]
fn rate_stretch_and_speed() {
let mut fx = Fx::new();

3
crates/engine/Cargo.toml
View File

@ -15,6 +15,8 @@ default = []
whisper = ["filmcraft-speech/whisper"]
# Speech-model downloads over HTTPS.
speech-download = ["filmcraft-speech/download"]
# Natural voices for Text to Speech: Kokoro-82M on candle (CPU) and their download.
neural-voices = ["filmcraft-tts/kokoro", "speech-download"]
[dependencies]
filmcraft-time = { workspace = true }
@ -35,6 +37,7 @@ filmcraft-format = { workspace = true }
filmcraft-interchange = { workspace = true }
filmcraft-captions = { workspace = true }
filmcraft-speech = { workspace = true }
filmcraft-tts = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }

61
crates/engine/src/clip_ops_tests.rs
View File

@ -923,3 +923,64 @@ fn linked_slip_keeps_picture_and_sound_aligned_at_unequal_media_limits() {
s.execute("timeline.slip", json!({"clip": v, "deltaFrames": -10})).unwrap();
assert_eq!((clip(&s, v).source_in, clip(&s, a).source_in), (Tick(fr.0 * 10), Tick(fr.0 * 10)));
}
#[test]
fn reset_keeps_keyframes_and_resets_one_parameter() {
// #284: Reset Effect used to replace the effect with a fresh one, deleting every keyframe
let mut s = demo();
let a = v1(&s)[0].clone();
let sec = Tick(TICKS_PER_SECOND);
let (t0, t1, mid) = (a.start, a.start + sec, a.start + Tick(TICKS_PER_SECOND / 2));
for (t, v) in [(t0, 150.0), (t1, 50.0)] {
s.execute("effects.setParam", json!({"clip": a.id.0, "effect": "motion", "param": "scale", "value": v, "keyframe": true, "time": t.0})).unwrap();
}
s.execute("effects.setParam", json!({"clip": a.id.0, "effect": "motion", "param": "rotation", "value": 30.0})).unwrap();
let idx = clip(&s, a.id.0).effects.iter().position(|e| e.effect == "motion").unwrap();
s.execute("playhead.set", json!({"time": mid.0})).unwrap();
s.execute("effects.reset", json!({"clip": a.id.0, "index": idx})).unwrap();
let it = clip(&s, a.id.0);
let scale = it.effect("motion").unwrap().param("scale").unwrap();
// (the playhead snaps to a frame)
let mid_media = it.source_time_at(s.playhead());
let kf: Vec<(Tick, f64)> = scale.keyframes.iter().map(|k| (k.time, k.value.as_f64().unwrap())).collect();
assert_eq!(kf, vec![(it.source_time_at(t0), 150.0), (mid_media, 100.0), (it.source_time_at(t1), 50.0)], "keyframes kept, default at the playhead");
assert_eq!(f64_param(&it, "motion", "rotation"), 0.0);
// one parameter: only it goes back to its default
s.execute("effects.setParam", json!({"clip": a.id.0, "effect": "motion", "param": "rotation", "value": 45.0})).unwrap();
s.execute("effects.setParam", json!({"clip": a.id.0, "effect": "opacity", "param": "opacity", "value": 40.0})).unwrap();
s.execute("effects.resetParam", json!({"clip": a.id.0, "effect": "motion", "param": "rotation"})).unwrap();
let it = clip(&s, a.id.0);
assert_eq!(f64_param(&it, "motion", "rotation"), 0.0);
assert_eq!(f64_param(&it, "opacity", "opacity"), 40.0);
assert_eq!(it.effect("motion").unwrap().param("scale").unwrap().keyframes.len(), 3);
assert!(s.execute("effects.resetParam", json!({"clip": a.id.0, "effect": "motion", "param": "nope"})).is_err());
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(f64_param(&clip(&s, a.id.0), "motion", "rotation"), 45.0);
}
#[test]
fn keyboard_slip_keeps_linked_clips_in_sync_past_the_shorter_handle() {
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Slip"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 1, "width": 16, "height": 16})).unwrap();
let r = s.execute("file.newOfflineFile", json!({"name": "Media", "seconds": 4, "fps": 25, "video": true, "audio": true})).unwrap();
let item = r["item"].as_u64().unwrap();
let fr = s.sequence_rate().tick_of(1);
let r =
s.execute("timeline.place", json!({"item": item, "track": "V1", "audioTrack": "A1", "time": 0, "sourceIn": fr.0 * 20, "duration": fr.0 * 60})).unwrap();
let (v, a) = (r["clips"][0].as_u64().unwrap(), r["clips"][1].as_u64().unwrap());
// lengthen only the picture: it keeps 10 frames of tail handle, the sound keeps 20
s.execute("sequence.linkedSelection", json!({"on": false})).unwrap();
s.execute("timeline.trim", json!({"clip": v, "edge": "out", "deltaFrames": 10})).unwrap();
s.execute("sequence.linkedSelection", json!({"on": true})).unwrap();
s.execute("timeline.select", json!({"clips": [v, a]})).unwrap();
for _ in 0..3 {
s.execute("timeline.slipLeft5", json!({})).unwrap();
}
assert_eq!((clip(&s, v).source_in, clip(&s, a).source_in), (Tick(fr.0 * 30), Tick(fr.0 * 30)));
// with linked selection off each selected clip still slips on its own
s.execute("sequence.linkedSelection", json!({"on": false})).unwrap();
s.execute("timeline.select", json!({"clips": [v, a]})).unwrap();
s.execute("timeline.slipLeft5", json!({})).unwrap();
assert_eq!((clip(&s, v).source_in, clip(&s, a).source_in), (Tick(fr.0 * 30), Tick(fr.0 * 35)));
}

238
crates/engine/src/commands.rs
View File

@ -140,6 +140,29 @@ pub(crate) fn named_clips(s: &Session, spec: &CommandSpec, p: &Value) -> Option<
let clips: Vec<ClipId> = named_ids(spec, p, "clips", "clip")?.into_iter().map(ClipId).filter(|c| seq.find_item(*c).is_some()).collect();
(!clips.is_empty()).then_some(clips)
}
/// Why the clips a caller named in `clips` / `clip` stand in for no selection (#591): a clip of
/// another sequence, an id of no clip (a track, a project item…) or a value that is not an id.
/// `None` when the caller named nothing or no sequence is open, so the selection's reason stands.
pub(crate) fn named_clips_missing(s: &Session, spec: &CommandSpec, p: &Value) -> Option<String> {
s.active_sequence()?;
let first = if spec.params.contains("\"clips\":[id]")
&& let Some(a) = p.get("clips").and_then(Value::as_array)
{
a.first()?
} else if spec.params.contains("\"clip\":id") {
p.get("clip").filter(|v| !v.is_null())?
} else {
return None;
};
let Some(id) = first.as_u64() else {
return Some(format!("{first} is not a clip id (clip ids are the `clip` numbers of the items in sequence.inspect)"));
};
let elsewhere = s.project.sequences().find(|i| i.as_sequence().is_some_and(|q| q.find_item(ClipId(id)).is_some()));
Some(match elsewhere {
Some(i) => format!("clip {id} is in sequence \"{}\", not the active one: open it first with sequence.open {{\"item\":{}}}", i.name, i.id.0),
None => format!("no clip {id} in the active sequence (clip ids are the `clip` numbers of the items in sequence.inspect)"),
})
}
/// Project items named explicitly in `items` / `item`, likewise. The Project panel's selection
/// holds bins beside items (one id space, see `project.select`), so a bin named here counts too;
/// the project's own top bin does not, since no command acts on it (#244).
@ -349,6 +372,24 @@ fn move_clips(s: &mut Session, p: &Value) -> Result<Value> {
Ok(json!({"moved": moves.iter().map(|m| m.0.0).collect::<Vec<_>>(), "overwritten": overwritten}))
}
/// Slip clips together: the media delta of each clip is scaled down by the same factor so that the
/// most constrained clip (media end or start) still moves, and no linked partner slips further.
pub(crate) fn slip_together(q: &mut filmcraft_project::Sequence, ctx: &mut edit::EditCtx<'_>, deltas: &[(ClipId, Tick)]) -> Result<()> {
let mut scale = 1.0_f64;
for (c, d) in deltas {
if d.0 == 0 {
continue;
}
let x = edit::slip(&mut q.clone(), *c, *d, ctx)?;
scale = scale.min(x.0.abs() as f64 / d.0.abs() as f64);
}
for (c, d) in deltas {
let dd = if scale < 1.0 { Tick((d.0 as f64 * scale).round() as i64) } else { *d };
edit::slip(q, *c, dd, ctx)?;
}
Ok(())
}
/// Expand a clip selection with linked partners (when linked selection is on).
pub fn with_links(s: &Session, clips: &[ClipId]) -> Vec<ClipId> {
let Some(seq) = s.active_sequence() else { return clips.to_vec() };
@ -366,6 +407,31 @@ pub fn with_links(s: &Session, clips: &[ClipId]) -> Vec<ClipId> {
out
}
/// The delta a trim of `clips` (a clip and its linked partners) can apply together: `delta`
/// clamped by each one's media and neighbours.
fn clamp_trim_linked(q: &filmcraft_project::Sequence, clips: &[ClipId], edge: Edge, mode: TrimMode, delta: Tick, ctx: &edit::EditCtx) -> Result<Tick> {
let mut d = delta;
for c in clips {
let x = edit::clamp_trim(q, *c, edge, mode, d, ctx)?;
if x.abs() < d.abs() {
d = x;
}
}
Ok(d)
}
/// The delta `timeline.trim` would apply for `delta`, without editing anything: the Timeline's
/// trim drag shows this, so the edge stops at the end of the media or the neighbouring clip
/// instead of following the pointer past it (#653).
pub fn trim_delta(s: &Session, clip: ClipId, edge: Edge, mode: TrimMode, delta: Tick) -> Result<Tick> {
let q = s.active_sequence().ok_or(EngineError::NoSequence)?;
let durations = |id: ItemId| crate::media_duration(&s.project, &s.media, id);
let starts = |id: ItemId| crate::media_start(&s.project, id);
let mut next = s.project.next_id;
let ctx = edit::EditCtx { next_id: &mut next, media_duration: &durations, media_start: &starts, min_duration: q.settings.frame_rate.frame_duration() };
clamp_trim_linked(q, &with_links(s, &[clip]), edge, mode, delta, &ctx)
}
/// Sequence settings matching a media clip (New Sequence From Clip, New Sequence from an item):
/// its frame size, rate and pixel aspect ratio (Interpret Footage's, else the file's), so a
/// 1440 x 1080 clip with 4:3 pixels makes a 1440 x 1080 sequence that displays 16:9.
@ -688,6 +754,20 @@ pub(crate) fn place_item(
})
}
/// How long an item runs on the Timeline without In/Out marks: its duration, or for a still (and a
/// generator without a length) Settings ▸ Timeline ▸ Still Image Default Duration.
pub(crate) fn full_duration(s: &Session, pi: &filmcraft_project::ProjectItem) -> Tick {
match &pi.kind {
ItemKind::Media(m)
if matches!(m.info.kind, filmcraft_media::MediaKind::Still)
|| (matches!(m.info.kind, filmcraft_media::MediaKind::Synthetic) && m.info.duration.0 <= 0) =>
{
s.prefs.timeline.still_duration(s.sequence_rate())
}
_ => pi.duration(),
}
}
pub(crate) fn source_range(s: &Session) -> Option<(ItemId, TimeRange)> {
let item = s.state.source_item?;
let pi = s.project.item(item)?;
@ -913,7 +993,7 @@ fn build() -> Vec<CommandSpec> {
Ok(json!({"items": ids.iter().map(|i| i.0).collect::<Vec<_>>()}))
}
),
cmd!("file.import", "Import…", ["File"], Some("Cmd+I"), r#"{"paths":[str],"bin":binId?,"imageSequence":bool?}"#, always, |s, p| {
cmd!("file.import", "Import…", ["File"], Some("Cmd+I"), r#"{"paths":[str],"bin":binId?,"imageSequence":bool?,"replace":bool?}"#, always, |s, p| {
let bin = u64_p(p, "bin").map(filmcraft_project::BinId);
let paths: Vec<String> = match p.get("paths").and_then(Value::as_array) {
Some(a) => a.iter().filter_map(|v| v.as_str().map(str::to_string)).collect(),
@ -922,6 +1002,10 @@ fn build() -> Vec<CommandSpec> {
if paths.is_empty() {
return Err(bad("file.import", "need `paths`"));
}
if bool_p(p, "replace").unwrap_or(false) {
let name = s.project.name.clone();
s.execute("file.newProject", json!({ "name": name }))?;
}
let mut ids = Vec::new();
let mut errors = Vec::new();
let mut sequences = Vec::new();
@ -1180,7 +1264,7 @@ fn build() -> Vec<CommandSpec> {
"Media…",
["File", "Export"],
None,
r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass"?,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#,
r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass|crf"?,"crf":0..51=23,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#,
has_seq,
crate::export_tools::export_media
),
@ -2074,9 +2158,12 @@ fn build() -> Vec<CommandSpec> {
"Place Clip",
[],
None,
r#"{"item":id,"track":"V1"|id|"A1" (sound only)?,"audioTrack":"A1"|id?,"time":ticks|"frame":i64|"seconds":f64,"insert":bool,"sourceIn":ticks?,"duration":ticks?,"video":bool=true,"audio":bool=true}"#,
r#"{"item":id,"track":"V1"|id|"A1" (sound only)|"new" (a video track after the last)?,"audioTrack":"A1"|id|"new"?,"time":ticks|"frame":i64|"seconds":f64,"insert":bool,"sourceIn":ticks?,"duration":ticks?,"video":bool=true,"audio":bool=true}"#,
has_seq,
|s, p| {
if crate::sequence_tools::wants_new_track(p) {
return crate::sequence_tools::place_on_new_tracks(s, p);
}
let stream_flag = |key: &str| match p.get(key) {
None => Ok(true),
Some(Value::Bool(v)) => Ok(*v),
@ -2115,16 +2202,7 @@ fn build() -> Vec<CommandSpec> {
if p.get("audio").is_some() && audio && pi.has_audio() && a.is_none() {
return Err(bad("timeline.place", "no audio destination: add or enable an audio track"));
}
let full = match &pi.kind {
// Settings ▸ Timeline ▸ Still Image Default Duration
ItemKind::Media(m)
if matches!(m.info.kind, filmcraft_media::MediaKind::Still)
|| (matches!(m.info.kind, filmcraft_media::MediaKind::Synthetic) && m.info.duration.0 <= 0) =>
{
s.prefs.timeline.still_duration(s.sequence_rate())
}
_ => pi.duration(),
};
let full = full_duration(s, pi);
let (mi, mo) = match &pi.kind {
ItemKind::Media(m) => (m.mark_in, m.mark_out.map(|o| o + pi.frame_rate().frame_duration())),
ItemKind::Subclip { range, .. } => (Some(range.start), Some(range.end())),
@ -2198,13 +2276,7 @@ fn build() -> Vec<CommandSpec> {
let applied = s.edit_sequence(if mode == TrimMode::Ripple { "Ripple Trim" } else { "Trim" }, |q, ctx, st| {
let before = q.find_item(c).map(|(_, i)| i.range());
// clamp across all linked partners, then apply the common delta
let mut dd = d;
for c in &clips {
let x = edit::clamp_trim(q, *c, edge, mode, dd, ctx)?;
if x.abs() < dd.abs() {
dd = x;
}
}
let mut dd = clamp_trim_linked(q, &clips, edge, mode, d, ctx)?;
if mode == TrimMode::Ripple {
// the clip and its linked partners ripple as one edit
let mut group = vec![c];
@ -2326,17 +2398,8 @@ fn build() -> Vec<CommandSpec> {
let clips = with_links(s, &[c]);
s.edit_sequence("Slip", |q, ctx, _| {
// clamp across all linked partners, then slip them by the common delta
let mut dd = d;
for c in &clips {
let x = edit::slip(&mut q.clone(), *c, dd, ctx)?;
if x.abs() < dd.abs() {
dd = x;
}
}
for c in &clips {
edit::slip(q, *c, dd, ctx)?;
}
Ok(())
let deltas: Vec<(ClipId, Tick)> = clips.iter().map(|c| (*c, d)).collect();
slip_together(q, ctx, &deltas)
})?;
Ok(Value::Null)
}),
@ -2349,20 +2412,8 @@ fn build() -> Vec<CommandSpec> {
.or_else(|| p.get("deltaFrames").and_then(Value::as_i64).map(|f| s.sequence_rate().tick_of(f)))
.unwrap_or_default();
let clips = with_links(s, &[c]);
s.edit_sequence("Slide", |q, ctx, _| {
// clamp across all linked partners, then slide them by the common delta
let mut dd = d;
for c in &clips {
let x = edit::slide(&mut q.clone(), *c, dd, ctx)?;
if x.abs() < dd.abs() {
dd = x;
}
}
for c in &clips {
edit::slide(q, *c, dd, ctx)?;
}
Ok(())
})?;
// clamp across all linked partners, then slide them by the common delta
s.edit_sequence("Slide", |q, ctx, _| Ok(edit::slide_items(q, &clips, d, ctx)?))?;
Ok(Value::Null)
}),
cmd!("timeline.rateStretch", "Rate Stretch", [], None, r#"{"clip":id,"edge":"in|out","delta":ticks}"#, has_seq, |s, p| {
@ -2571,15 +2622,34 @@ fn build() -> Vec<CommandSpec> {
{
e.params.insert(pid.clone(), filmcraft_project::Param::new(d.default.clone()));
}
let prm = crate::masks::target_param(e, &pq, &pid).ok_or_else(|| bad("effects.setParam", format!("no param `{pid}`")))?;
let v = json_to_param(&prm.value, &val).ok_or_else(|| bad("effects.setParam", "value has the wrong type"))?;
if keyframe {
// the stopwatch and the value in one step
prm.put_keyframe(mt, v);
} else {
prm.set_at(mt, v);
let (keyframes, v) = {
let prm = crate::masks::target_param(e, &pq, &pid).ok_or_else(|| bad("effects.setParam", format!("no param `{pid}`")))?;
let v = json_to_param(&prm.value, &val).ok_or_else(|| bad("effects.setParam", "value has the wrong type"))?;
let keyframes = if keyframe {
// the stopwatch and the value in one step
prm.put_keyframe(mt, v.clone());
prm.keyframes.len()
} else {
prm.set_at(mt, v.clone());
prm.keyframes.len()
};
(keyframes, v)
};
if e.effect == "ultra_key"
&& pid == "setting"
&& let ParamValue::Choice(setting) = v
&& let Some(rows) = filmcraft_project::effect::ultra_key_setting(setting)
{
for (id, value) in rows {
if let Some(p) = e.param_mut(id) {
p.value = ParamValue::Float(*value);
p.keyframes.clear();
} else {
e.params.insert((*id).to_string(), filmcraft_project::Param::new(ParamValue::Float(*value)));
}
}
}
Ok(prm.keyframes.len())
Ok(keyframes)
})?;
// 0 keyframes: the value is static (`time` was not used)
Ok(json!({"keyframes": keyframes}))
@ -2619,15 +2689,35 @@ fn build() -> Vec<CommandSpec> {
cmd!("effects.reset", "Reset Effect", [], None, r#"{"clip":id,"index":n}"#, has_seq, |s, p| {
let c = clip_p(p, "clip").ok_or_else(|| bad("effects.reset", "need `clip`"))?;
let idx = u64_p(p, "index").unwrap_or(0) as usize;
let (fw, fh) = s.active_sequence().map(|q| (q.settings.width, q.settings.height)).unwrap_or((1920, 1080));
let frame = s.active_sequence().map(|q| (q.settings.width, q.settings.height)).unwrap_or((1920, 1080));
let ph = s.playhead();
s.edit_sequence("Reset Effect", |q, _, _| {
let (_, it) = q.find_item_mut(c).ok_or(filmcraft_edit::EditError::NoItem(c))?;
if let Some(e) = it.effects.get_mut(idx)
&& let Some(d) = e.def()
{
let mut fresh = d.instance();
resolve_auto_points(&mut fresh, (fw, fh), (fw, fh));
*e = fresh;
let mt = it.source_time_at(ph.clamp(it.start, (it.end() - Tick(1)).max(it.start)));
if let Some(e) = it.effects.get_mut(idx) {
reset_params(e, None, frame, mt);
}
Ok(())
})?;
Ok(Value::Null)
}),
cmd!("effects.resetParam", "Reset Parameter", [], None, r#"{"clip":id,"effect":str|index,"param":str}"#, has_seq, |s, p| {
let c = clip_p(p, "clip").ok_or_else(|| bad("effects.resetParam", "need `clip`"))?;
let pid = str_p(p, "param").ok_or_else(|| bad("effects.resetParam", "need `param`"))?.to_string();
let eff = p.get("effect").cloned().unwrap_or(json!("motion"));
let frame = s.active_sequence().map(|q| (q.settings.width, q.settings.height)).unwrap_or((1920, 1080));
let ph = s.playhead();
s.edit_sequence("Reset Parameter", |q, _, _| {
let (_, it) = q.find_item_mut(c).ok_or(filmcraft_edit::EditError::NoItem(c))?;
let mt = it.source_time_at(ph.clamp(it.start, (it.end() - Tick(1)).max(it.start)));
let e = match &eff {
Value::Number(n) => it.effects.get_mut(n.as_u64().unwrap_or(0) as usize),
Value::String(sid) => it.effects.iter_mut().find(|e| &e.effect == sid),
_ => None,
}
.ok_or_else(|| bad("effects.resetParam", "no such effect on clip"))?;
if !reset_params(e, Some(&pid), frame, mt) {
return Err(bad("effects.resetParam", format!("no param `{pid}`")));
}
Ok(())
})?;
@ -2729,7 +2819,7 @@ fn build() -> Vec<CommandSpec> {
s.fix_state();
Ok(json!({"items": items.len(), "bins": bins.len()}))
}),
cmd!("project.moveToBin", "Move to Bin", [], None, r#"{"items":[id]?,"bin":binId|null}"#, always, |s, p| {
cmd!("project.moveToBin", "Move to Bin", [], None, r#"{"items":[id|binId]?,"bin":binId|null}"#, always, |s, p| {
let items: Vec<ItemId> = match p.get("items").and_then(Value::as_array) {
Some(a) => a.iter().filter_map(|v| v.as_u64().map(ItemId)).collect(),
None => s.state.project_selection.clone(),
@ -2830,9 +2920,11 @@ fn build() -> Vec<CommandSpec> {
v.extend(crate::scopes::commands());
v.extend(crate::remix::commands());
v.extend(crate::voiceover::commands());
v.extend(crate::narration::commands());
v.extend(crate::keyboard::commands());
v.extend(crate::project_panel::commands());
v.extend(crate::media_browser::commands());
v.extend(crate::paste_media::commands());
// Edit ▸ Label ▸ <colour>, Paste Attributes, subclips, Video / Audio Options, Replace With Clip…
// and their menu order
crate::clip_ops::apply_layout(&mut v);
@ -2842,6 +2934,30 @@ fn build() -> Vec<CommandSpec> {
v
}
/// Put `e`'s parameters (all of them, or only `only`) back to their defaults, as Premiere's reset
/// buttons do: a parameter that is not animated gets its default as its static value, and an
/// animated one keeps its keyframes and gets the default as a keyframe at media time `mt`.
/// Masks are left alone. Returns whether any parameter was reset.
fn reset_params(e: &mut filmcraft_project::EffectInstance, only: Option<&str>, frame: (u32, u32), mt: Tick) -> bool {
let Some(d) = e.def() else { return false };
let mut fresh = d.instance();
resolve_auto_points(&mut fresh, frame, frame);
let mut any = false;
for (id, prm) in fresh.params {
if only.is_some_and(|o| o != id) {
continue;
}
any = true;
match e.params.get_mut(&id) {
Some(old) if old.is_animated() => old.set_at(mt, prm.value),
_ => {
e.params.insert(id, prm);
}
}
}
any
}
fn keyframe_op(s: &mut Session, p: &Value, op: &str) -> Result<Value> {
let c = clip_p(p, "clip").ok_or_else(|| bad("keyframe", "need `clip`"))?;
let pid = str_p(p, "param").ok_or_else(|| bad("keyframe", "need `param`"))?.to_string();

47
crates/engine/src/explicit_targets_tests.rs
View File

@ -119,10 +119,15 @@ fn clip_commands_take_explicit_clips_without_a_selection() {
fn naming_nothing_usable_does_not_enable_a_command() {
let mut s = demo();
let before = s.project.clone();
// no ids, ids of no clip, the wrong type, and a key the command does not take
for p in [json!({"clips": []}), json!({"clips": [987_654_321u64]}), json!({"clips": "all"}), json!({"clips": [-1, 1.5, null]}), json!({"items": [1]})] {
// no ids, the wrong type, and a key the command does not take
for p in [json!({"clips": []}), json!({"clips": "all"}), json!({"items": [1]})] {
assert_eq!(disabled(s.execute("edit.rippleDelete", p.clone())), "nothing selected", "{p}");
}
// ids of no clip and values that are not ids: still disabled, and the reason names them (#591)
let why = disabled(s.execute("edit.rippleDelete", json!({"clips": [987_654_321u64]})));
assert!(why.starts_with("no clip 987654321 in the active sequence"), "{why}");
let why = disabled(s.execute("edit.rippleDelete", json!({"clips": [-1, 1.5, null]})));
assert!(why.starts_with("-1 is not a clip id"), "{why}");
assert_eq!(disabled(s.execute("clip.replaceFromBin", json!({"clips": [v1(&s)[0].id.0], "item": 987_654_321u64}))), "select a clip in the Project panel");
// `edit.cut` takes no `clips`: it works on the selection only
assert_eq!(disabled(s.execute("edit.cut", json!({"clips": [v1(&s)[0].id.0]}))), "no clips selected");
@ -159,6 +164,44 @@ fn link_and_speed_take_explicit_clips_without_a_selection() {
assert!(s.state.selection.is_empty() && !s.is_enabled("clip.speedDuration"), "menu enablement still follows the selection");
}
/// #591: a caller that names clips which cannot stand in for the selection hears why, instead of
/// "no clips selected": a clip of another sequence, an id of something else, a string id.
#[test]
fn named_clips_that_are_not_usable_say_why() {
let mut s = demo();
let demo_seq = s.state.active_sequence.unwrap();
let demo_name = s.project.item(demo_seq).unwrap().name.clone();
let last = v1(&s).last().unwrap().clone();
// a track id and a project item id are not clip ids
let track = s.active_sequence().unwrap().video_tracks[0].id.0;
let why = disabled(s.execute("clip.speedDuration", json!({"clips": [track], "speed": 50})));
assert_eq!(why, format!("no clip {track} in the active sequence (clip ids are the `clip` numbers of the items in sequence.inspect)"));
let why = disabled(s.execute("clip.speedDuration", json!({"clips": [last.item.0], "speed": 50})));
assert!(why.starts_with(&format!("no clip {} in the active sequence", last.item.0)), "{why}");
// a string is not an id
let why = disabled(s.execute("clip.speedDuration", json!({"clips": [last.id.0.to_string()], "speed": 50})));
assert!(why.starts_with(&format!("\"{}\" is not a clip id", last.id.0)), "{why}");
// `clip`, where the command documents it
let why = disabled(s.execute("graphics.setText", json!({"clip": "x", "text": "x"})));
assert!(why.starts_with("\"x\" is not a clip id"), "{why}");
// a clip of a sequence that is not the active one
s.execute("file.newSequence", json!({"name": "Other"})).unwrap();
assert_ne!(s.state.active_sequence, Some(demo_seq));
let why = disabled(s.execute("clip.speedDuration", json!({"clips": [last.id.0], "speed": 200})));
assert_eq!(
why,
format!("clip {} is in sequence \"{demo_name}\", not the active one: open it first with sequence.open {{\"item\":{}}}", last.id.0, demo_seq.0)
);
// and doing what it says works
s.execute("sequence.open", json!({"item": demo_seq.0})).unwrap();
s.execute("clip.speedDuration", json!({"clips": [last.id.0], "speed": 200})).unwrap();
assert!(clip(&s, last.id).unwrap().duration < last.duration);
// naming nothing keeps the selection's reason, for menus and callers alike
s.state.selection.clear();
assert_eq!(disabled(s.execute("clip.speedDuration", json!({"speed": 50}))), "no clips selected");
assert_eq!(disabled(s.execute("clip.speedDuration", json!({"clips": [], "speed": 50}))), "no clips selected");
}
#[test]
fn naming_targets_lifts_only_the_selection_condition() {
// no sequence: named clips cannot stand in for one

29
crates/engine/src/export_tests.rs
View File

@ -64,6 +64,21 @@ fn export_integer_parameters_cannot_wrap_or_overflow() {
assert!(s.execute("export.resolve", json!({"bitrateKbps":8000.0, "keyframeDistance":48.0})).is_ok(), "integer-valued floats are integers");
}
#[test]
fn wav_resolve_honours_explicit_sample_rates() {
let mut s = demo();
for rate in [4000, 192_000, 192_001, 200_000, 352_800, 384_000] {
let result = s.execute("export.resolve", json!({"format":"wav", "settings":{"audio":{"sample_rate":rate}}})).unwrap();
assert_eq!(result["settings"]["audio"]["sample_rate"], rate);
assert_eq!(result["output"]["sampleRate"], rate);
}
let result = s.execute("export.resolve", json!({"format":"wav", "settings":{"audio":{"sample_rate":null}}})).unwrap();
assert_eq!(result["output"]["sampleRate"], 48_000);
for rate in [0, 384_001] {
assert!(s.execute("export.resolve", json!({"format":"wav", "settings":{"audio":{"sample_rate":rate}}})).is_err());
}
}
#[test]
fn nested_camel_case_settings_are_honoured() {
let mut s = demo();
@ -481,6 +496,20 @@ fn hardware_encoding_is_off_unless_asked_for() {
assert_eq!(old.hardware_encoding, Off);
}
#[test]
fn crf_param_picks_constant_quality() {
use filmcraft_export::BitrateMode::{Cbr, Crf, Vbr1Pass};
let s = demo();
let setting = |p: Value| crate::export_tools::settings_from_params(&s, &p, "file.exportMedia").map(|(_, st)| (st.bitrate_mode, st.crf));
assert_eq!(setting(json!({"path": "x.mp4"})).unwrap(), (Vbr1Pass, 23.0));
// a factor alone means CRF mode; the mode can also be named, or another one kept explicitly
assert_eq!(setting(json!({"path": "x.mp4", "crf": 18})).unwrap(), (Crf, 18.0));
assert_eq!(setting(json!({"path": "x.mp4", "bitrateMode": "crf"})).unwrap(), (Crf, 23.0));
assert_eq!(setting(json!({"path": "x.mp4", "bitrateMode": "cbr", "crf": 18})).unwrap(), (Cbr, 18.0));
let e = setting(json!({"path": "x.mp4", "bitrateMode": "abr"})).unwrap_err().to_string();
assert!(e.contains("crf"), "{e}");
}
#[test]
fn queue_exports_several_sequences_and_ranges() {
let mut s = demo();

22
crates/engine/src/export_tools.rs
View File

@ -26,7 +26,7 @@ use std::sync::Arc;
use std::sync::atomic::Ordering;
use filmcraft_export::presets::{DEFAULT_PRESET, preset_key};
use filmcraft_export::{ExportPreset, ExportSettings, Format, HardwareEncoding};
use filmcraft_export::{BitrateMode, ExportPreset, ExportSettings, Format, GpuRendering, HardwareEncoding};
use filmcraft_project::{ItemId, Project};
use filmcraft_time::{FrameRate, Tick, TimeRange};
use serde::{Deserialize, Serialize};
@ -213,7 +213,13 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
settings.max_bitrate_kbps = Some(v);
}
if let Some(v) = str_p(p, "bitrateMode") {
settings.bitrate_mode = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "bitrateMode: cbr | vbr1Pass | vbr2Pass"))?;
settings.bitrate_mode = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "bitrateMode: cbr | vbr1Pass | vbr2Pass | crf"))?;
}
if let Some(v) = f64_p(p, "crf") {
settings.crf = v as f32;
if p.get("bitrateMode").is_none() {
settings.bitrate_mode = BitrateMode::Crf;
}
}
if let Some(v) = crate::commands::checked_u32_p(p, "keyframeDistance", cmd)? {
settings.keyframe_distance = Some(v);
@ -230,6 +236,18 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
other => serde_json::from_value(other.clone()).map_err(|_| bad(cmd, "hardwareEncoding: off | auto"))?,
};
}
if let Some(v) = p.get("gpuRendering") {
settings.gpu_rendering = match v {
Value::Bool(on) => {
if *on {
GpuRendering::Auto
} else {
GpuRendering::Off
}
}
other => serde_json::from_value(other.clone()).map_err(|_| bad(cmd, "gpuRendering: off | auto"))?,
};
}
if let Some(v) = bool_p(p, "burnCaptions") {
settings.burn_captions = v;
}

18
crates/engine/src/image_sequence_tests.rs
View File

@ -11,6 +11,24 @@ use serde_json::{Value, json};
use crate::Session;
use crate::media_test_util::tmp_dir;
#[test]
fn import_replace_starts_from_an_empty_project() {
let dir = tmp_dir("import-replace");
let wav = dir.join("tone.wav");
std::fs::write(&wav, filmcraft_media::wav::write_wav16(&[0.1; 4800], 2, 48_000)).unwrap();
let mut s = demo();
let demo_items = s.project.items.len();
assert!(demo_items > 1);
assert!(!s.state.open_sequences.is_empty());
s.execute("file.import", json!({"paths": [wav.to_string_lossy()], "replace": true})).unwrap();
assert!(s.state.open_sequences.is_empty());
assert_eq!(s.project.items.len(), 1);
assert!(s.project.items.values().any(|it| it.name == "tone.wav"));
let mut merge = demo();
merge.execute("file.import", json!({"paths": [wav.to_string_lossy()]})).unwrap();
assert!(merge.project.items.len() > demo_items);
}
fn demo() -> Session {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();

9
crates/engine/src/interchange.rs
View File

@ -59,7 +59,8 @@ pub fn import(s: &mut Session, path: &str, bytes: &[u8], format: Format) -> Resu
proj.resolve_placed_auto_points(|seq| !before.contains(&seq), |source| !is_file_media(source));
Ok(seqs)
})?;
// Link media: probe each new file-backed item that exists.
// Link media: probe each new file-backed item that exists, through the host's reader as a
// direct import does (only the container index is read now, not the whole file: #461).
let new_media: Vec<(ItemId, String)> = s
.project
.items
@ -76,11 +77,7 @@ pub fn import(s: &mut Session, path: &str, bytes: &[u8], format: Format) -> Resu
let mut linked = 0;
let mut offline = Vec::new();
for (id, mpath) in new_media {
let opened = s.services.read_file(&mpath).ok().and_then(|b| {
let fname = std::path::Path::new(&mpath).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
s.media.open_bytes(&fname, b.into()).ok()
});
match opened {
match s.media.open_file(&mpath, &*s.services).ok() {
Some(src) => {
let info = src.info().clone();
let identity = crate::relink::identity_of(&*s.services, &mpath).ok();

42
crates/engine/src/interchange_auto_points_tests.rs
View File

@ -1,6 +1,6 @@
//! Interchange imports and the "auto" (NaN) point defaults of the effects they add: the points
//! are resolved as placing a clip resolves them, the anchor once the media's real size is known,
//! and a project saved after an import opens again.
//! and a project saved after an import opens again. Also how the media they name are linked.
use filmcraft_geom::Vec2;
use filmcraft_media::DemoScene;
@ -216,3 +216,43 @@ fn relinking_later_resolves_the_anchor_that_waited_for_the_real_size() {
assert_eq!(motion_point(&first_clip(&t, seq), "anchor"), Vec2::new(160.0, 90.0));
let _ = std::fs::remove_dir_all(&d);
}
/// Linking the media an interchange document names opens each file through the host's reader, as
/// a direct import does, instead of reading it whole into memory (#461: minutes for 4K files).
#[cfg(any(unix, windows))]
#[test]
fn an_interchange_import_links_media_without_reading_the_files_whole() {
/// The desktop filesystem, except that reading a media file whole fails.
struct NoWholeMediaReads;
impl crate::Services for NoWholeMediaReads {
fn read_file(&self, path: &str) -> std::io::Result<Vec<u8>> {
if path.ends_with(".mov") {
return Err(std::io::Error::other(format!("{path}: read whole")));
}
crate::FsServices.read_file(path)
}
fn write_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> {
crate::FsServices.write_file(path, data)
}
fn file_size(&self, path: &str) -> std::io::Result<u64> {
crate::FsServices.file_size(path)
}
fn read_range(&self, path: &str, offset: u64, len: usize) -> std::io::Result<Vec<u8>> {
crate::FsServices.read_range(path, offset, len)
}
fn reader(&self, path: &str) -> Option<std::io::Result<filmcraft_media::SharedReader>> {
crate::FsServices.reader(path)
}
}
let d = tmp_dir("ix-streamed-link");
std::fs::create_dir_all(d.join("media")).unwrap();
make_movie(&d.join("media/shot.mov"), DemoScene::OceanSunset, 320, 180, 24);
let doc = d.join("cut.xml");
std::fs::write(&doc, xml_with_a_scale_only_motion_filter()).unwrap();
let mut s = Session::new(std::sync::Arc::new(NoWholeMediaReads));
let (r, seq) = import(&mut s, &doc);
assert_eq!(r["linkedMedia"], 1, "{r}");
let clip = first_clip(&s, seq);
assert_eq!(s.project.source_size(clip.item), Some((320, 180)), "the file's real size, from its index");
let _ = std::fs::remove_dir_all(&d);
}

31
crates/engine/src/keyboard.rs
View File

@ -40,7 +40,7 @@ use filmcraft_project::{ClipId, ItemKind, ParamValue, Sequence, TrackId, TrackKi
use filmcraft_time::Tick;
use serde_json::{Value, json};
use crate::commands::{CommandSpec, always, bad, has_selection, has_seq, item_p, time_p, with_links};
use crate::commands::{CommandSpec, always, bad, has_selection, has_seq, item_p, slip_together, time_p, with_links};
use crate::{EngineError, Result, Session};
type Run = fn(&mut Session, &Value) -> Result<Value>;
@ -248,14 +248,21 @@ fn slip_selection(s: &mut Session, frames: i64) -> Result<Value> {
let rate = s.sequence_rate();
let clips = with_links(s, &s.state.selection.clone());
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
// media delta per clip (source frames run at the clip's speed)
let deltas: Vec<(ClipId, Tick)> = clips
.iter()
.filter_map(|c| seq.find_item(*c).map(|(_, i)| (*c, Tick((rate.tick_of(frames).0 as f64 * i.speed.abs().max(1e-9)).round() as i64))))
.collect();
let linked = s.state.linked_selection;
// media delta per clip (source frames run at the clip's speed); linked partners slip as one group
let mut groups: Vec<(Option<u64>, Vec<(ClipId, Tick)>)> = Vec::new();
for c in &clips {
let Some((_, i)) = seq.find_item(*c) else { continue };
let d = Tick((rate.tick_of(frames).0 as f64 * i.speed.abs().max(1e-9)).round() as i64);
let link = i.link.filter(|_| linked);
match groups.iter_mut().find(|(l, _)| link.is_some() && *l == link) {
Some((_, g)) => g.push((*c, d)),
None => groups.push((link, vec![(*c, d)])),
}
}
s.edit_sequence("Slip", |q, ctx, _| {
for (c, d) in &deltas {
edit::slip(q, *c, *d, ctx)?;
for (_, g) in &groups {
slip_together(q, ctx, g)?;
}
Ok(())
})?;
@ -265,12 +272,8 @@ fn slip_selection(s: &mut Session, frames: i64) -> Result<Value> {
fn slide_selection(s: &mut Session, frames: i64) -> Result<Value> {
let d = s.sequence_rate().tick_of(frames);
let clips = with_links(s, &s.state.selection.clone());
s.edit_sequence("Slide", |q, ctx, _| {
for c in &clips {
edit::slide(q, *c, d, ctx)?;
}
Ok(())
})?;
// one common delta, clamped by the most restrictive member, keeps linked partners in sync
s.edit_sequence("Slide", |q, ctx, _| Ok(edit::slide_items(q, &clips, d, ctx)?))?;
Ok(Value::Null)
}

26
crates/engine/src/keyboard_tests.rs
View File

@ -376,3 +376,29 @@ fn reveal_nested_sequence_opens_it_at_the_matching_frame() {
assert_ne!(s.state.active_sequence, Some(outer));
assert_eq!(s.playhead(), nest_clip.source_in + rate.tick_of(7), "{r}");
}
#[test]
fn keyboard_slide_keeps_linked_partners_aligned_at_a_neighbour_limit() {
// #712: a three-frame sound-only follower limits only the sound partner
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Slide"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 1, "width": 16, "height": 16})).unwrap();
let r = s.execute("file.newOfflineFile", json!({"name": "Media", "seconds": 4, "fps": 25, "video": true, "audio": true})).unwrap();
let item = r["item"].as_u64().unwrap();
let fr = s.sequence_rate().tick_of(1);
let r =
s.execute("timeline.place", json!({"item": item, "track": "V1", "audioTrack": "A1", "time": 0, "sourceIn": fr.0 * 20, "duration": fr.0 * 60})).unwrap();
let (v, a) = (r["clips"][0].as_u64().unwrap(), r["clips"][1].as_u64().unwrap());
let r = s.execute("timeline.place", json!({"item": item, "track": "A1", "time": fr.0 * 60, "sourceIn": 0, "duration": fr.0 * 3})).unwrap();
let f = r["clips"][0].as_u64().unwrap();
s.execute("sequence.linkedSelection", json!({"on": true})).unwrap();
s.execute("timeline.select", json!({"clips": [v]})).unwrap();
s.execute("timeline.slideRight5", json!({})).unwrap();
assert_eq!(clip(&s, v).start, Tick(fr.0 * 2), "the pair moves only as far as the sound may");
assert_eq!(clip(&s, a).start, clip(&s, v).start, "picture and sound stay aligned");
let fl = clip(&s, f);
assert_eq!((fl.start, fl.duration, fl.source_in), (Tick(fr.0 * 62), fr, Tick(fr.0 * 2)));
// nothing more to give: a further slide right moves neither partner
s.execute("timeline.slideRight", json!({})).unwrap();
assert_eq!((clip(&s, v).start, clip(&s, a).start), (Tick(fr.0 * 2), Tick(fr.0 * 2)));
}

19
crates/engine/src/lib.rs
View File

@ -29,7 +29,9 @@ pub mod media_browser;
pub mod media_pool;
pub mod mixer;
pub mod multicam;
pub mod narration;
pub mod panels;
pub mod paste_media;
pub mod perf;
pub mod presets;
pub mod previews;
@ -353,6 +355,10 @@ pub struct Session {
pub mcrec: multicam::Recorder,
/// Voice-over recording: the input device and the take in progress.
pub voiceover: voiceover::VoiceOver,
/// The last `tts.preview` result, for the host to play (Text to Speech ▸ Preview).
pub tts_preview: Option<Arc<filmcraft_tts::Audio>>,
/// Synthesized narrations (`tts.render` fills it from a background job).
pub tts_cache: narration::SynthCache,
/// Dynamic (J/K/L) trimming and trim-mode loop playback in progress.
pub trim_play: trim::TrimPlayback,
/// Keyboard shortcuts (active bindings, presets; `shortcuts.*` commands).
@ -455,6 +461,8 @@ impl Session {
mixrec: Default::default(),
mcrec: Default::default(),
voiceover: Default::default(),
tts_preview: None,
tts_cache: Default::default(),
trim_play: Default::default(),
shortcuts: shortcuts::Shortcuts::new(),
offline: Default::default(),
@ -687,8 +695,11 @@ impl Session {
}
let clips = commands::named_clips(self, spec, params);
let items = commands::named_items(self, spec, params);
// clips were named but none is one of the active sequence: say which and why, rather
// than "no clips selected" to a caller that did not mean the selection (#591)
let missing = if clips.is_none() { commands::named_clips_missing(self, spec, params) } else { None };
if clips.is_none() && items.is_none() {
return by_selection;
return by_selection.map_err(|why| missing.unwrap_or(why));
}
let mut clips = clips.unwrap_or_else(|| self.state.selection.clone());
let mut items = items.unwrap_or_else(|| self.state.project_selection.clone());
@ -697,7 +708,7 @@ impl Session {
let by_params = (spec.enabled)(self);
self.state.selection = clips;
self.state.project_selection = items;
by_params
by_params.map_err(|why| missing.unwrap_or(why))
}
/// Whether the command can run on the current selection (what the menus show).
@ -1069,6 +1080,8 @@ mod mixer_tests;
#[cfg(test)]
mod multicam_tests;
#[cfg(test)]
mod narration_tests;
#[cfg(test)]
mod nest_editing_tests;
#[cfg(test)]
mod nest_fidelity_tests;
@ -1079,6 +1092,8 @@ mod panels_tests;
#[cfg(test)]
mod par_tests;
#[cfg(test)]
mod paste_media_tests;
#[cfg(test)]
mod presets_tests;
#[cfg(test)]
mod previews_tests;

124
crates/engine/src/media_browser.rs
View File

@ -12,7 +12,7 @@
//! | `mediaBrowser.favorite` | Add to / Remove from Favorites |
//! | `mediaBrowser.clearRecent` | Clear Recent Directories |
//! | `mediaBrowser.settings` | file types, view (list / thumbnails), columns (Edit Columns…), Import as Image Sequence, Hover Scrub, thumbnail size |
//! | `mediaBrowser.import` | Import (the selection or `paths`; Project ▸ Ingest settings apply) |
//! | `mediaBrowser.import` | Import (the selection or `paths`; a folder becomes a bin, its sub-folders bins inside it; Project ▸ Ingest settings apply) |
//! | `mediaBrowser.openInSource` | Open In Source Monitor (Shift+O) |
//! | `mediaBrowser.probe` | a file's media properties (frame rate, duration, video / audio info), cached |
//!
@ -421,32 +421,134 @@ fn paths_p(s: &Session, p: &Value) -> Vec<String> {
}
}
/// `mediaBrowser.import {paths?, bin?, imageSequence?}`: imports files (directories import their
/// supported files, like dragging a folder in Premiere).
/// Most folders one import walks, and how deep it goes: a folder import mirrors the tree as bins,
/// and symlinked folders can loop.
const FOLDER_IMPORT_MAX_FOLDERS: usize = 4096;
const FOLDER_IMPORT_MAX_DEPTH: usize = 32;
/// A folder to import as a bin: its name, its files and its sub-folders that hold files.
struct Folder {
name: String,
files: Vec<String>,
subs: Vec<Folder>,
}
/// The files the browser shows in `dir` and its sub-folders, as a tree; `None` when the tree
/// holds none (or `budget` folders were walked already).
fn walk_folder(services: &dyn Services, dir: &str, depth: usize, budget: &mut usize) -> Option<Folder> {
*budget = budget.checked_sub(1)?;
let entries = list(services, dir, "all").ok()?;
let mut folder = Folder { name: base_name(dir), files: Vec::new(), subs: Vec::new() };
for e in entries {
if !e.is_dir {
folder.files.push(e.path);
} else if depth < FOLDER_IMPORT_MAX_DEPTH
&& let Some(sub) = walk_folder(services, &e.path, depth + 1, budget)
{
folder.subs.push(sub);
}
}
(!folder.files.is_empty() || !folder.subs.is_empty()).then_some(folder)
}
/// Add a `file.import` result to the merged one: arrays add up, other values (the Ingest result)
/// are listed.
fn merge_import(out: &mut serde_json::Map<String, Value>, r: Value) {
let Value::Object(r) = r else { return };
for (k, v) in r {
let slot = out.entry(k).or_insert_with(|| json!([]));
let Value::Array(all) = slot else { continue };
match v {
Value::Null => {}
Value::Array(a) => all.extend(a),
other => all.push(other),
}
}
}
/// Run `file.import` and merge its result; a failure becomes an error of the merged result.
fn import_into(s: &mut Session, files: &[String], bin: Option<u64>, out: &mut serde_json::Map<String, Value>) {
let mut q = json!({"paths": files, "imageSequence": false});
if let Some(b) = bin {
q["bin"] = json!(b);
}
match s.execute("file.import", q) {
Ok(r) => merge_import(out, r),
Err(e) => merge_import(out, json!({"errors": [e.to_string()]})),
}
}
/// Import a folder as a bin named after it, in `parent`: its files go into the bin, each
/// sub-folder becomes a bin inside it (Premiere's Media Browser keeps the folder structure).
fn import_folder(s: &mut Session, folder: &Folder, parent: Option<u64>, out: &mut serde_json::Map<String, Value>) -> Result<()> {
let name = folder.name.clone();
let bin = s.edit("New Bin", |pr, _| Ok(pr.add_bin(&name, parent.map(filmcraft_project::BinId))))?;
merge_import(out, json!({"bins": [bin.0]}));
if !folder.files.is_empty() {
import_into(s, &folder.files, Some(bin.0), out);
}
// recursion as deep as the walk went, which is bounded
for sub in &folder.subs {
import_folder(s, sub, Some(bin.0), out)?;
}
Ok(())
}
/// `mediaBrowser.import {paths?, bin?, imageSequence?}`: imports files. A folder becomes a bin
/// named after it (in `bin`) holding its files, and each sub-folder a bin inside that, as in
/// Premiere; the result then also lists the new `bins`. Import as Image Sequence takes the first
/// file instead.
fn import(s: &mut Session, p: &Value) -> Result<Value> {
let paths = paths_p(s, p);
if paths.is_empty() {
return Err(bad("mediaBrowser.import", "select files in the Media Browser"));
}
let seq = bool_p(p, "imageSequence").unwrap_or(s.prefs.media_browser.import_as_image_sequence);
let bin = u64_p(p, "bin");
let mut files = Vec::new();
let mut folders = Vec::new();
let mut empty = Vec::new();
let mut budget = FOLDER_IMPORT_MAX_FOLDERS;
for path in paths {
// a directory imports its media files (not recursive)
let is_dir = matches!(s.services.list_entries(&path), Some(Ok(_))) && kind_of(&path).is_none();
match list(&*s.services, &path, "all") {
Ok(entries) if is_dir => files.extend(entries.into_iter().filter(|e| !e.is_dir).map(|e| e.path)),
_ => files.push(path),
if !is_dir {
files.push(path);
} else if seq {
// an image sequence: the folder's frames
match list(&*s.services, &path, "all") {
Ok(entries) => files.extend(entries.into_iter().filter(|e| !e.is_dir).map(|e| e.path)),
Err(_) => files.push(path),
}
} else {
match walk_folder(&*s.services, &path, 0, &mut budget) {
Some(f) => folders.push(f),
None => empty.push(path),
}
}
}
// an image sequence imports from its first selected frame only
if seq {
files.truncate(1);
}
let mut q = json!({"paths": files, "imageSequence": seq});
if let Some(b) = u64_p(p, "bin") {
q["bin"] = json!(b);
if folders.is_empty() {
if files.is_empty() && !empty.is_empty() {
return Err(bad("mediaBrowser.import", format!("no media files in {}", empty.join(", "))));
}
let mut q = json!({"paths": files, "imageSequence": seq});
if let Some(b) = bin {
q["bin"] = json!(b);
}
return s.execute("file.import", q);
}
s.execute("file.import", q)
let mut out = serde_json::Map::new();
merge_import(&mut out, json!({"items": [], "errors": empty.iter().map(|e| format!("{e}: no media files")).collect::<Vec<_>>(), "bins": []}));
if !files.is_empty() {
import_into(s, &files, bin, &mut out);
}
for f in &folders {
import_folder(s, f, bin, &mut out)?;
}
Ok(Value::Object(out))
}
/// The project item that already references a file.

41
crates/engine/src/media_browser_tests.rs
View File

@ -210,6 +210,47 @@ fn import_the_selection_and_open_in_source() {
assert_eq!(r["items"].as_array().unwrap().len(), 1);
}
/// Importing a folder keeps its structure (#411): the folder becomes a bin named after it, each
/// sub-folder with media a bin inside that, and every file lands in its folder's bin.
#[test]
fn importing_a_folder_mirrors_its_sub_folders_as_bins() {
let mut fs = FakeFs::new();
let wav = filmcraft_media::wav::write_wav16(&[0.0; 4800], 2, 48_000);
fs.dirs.extend(["/Shoot", "/Shoot/Day 1", "/Shoot/Day 1/Cam A", "/Shoot/Empty"].map(String::from));
for f in ["/Shoot/slate.wav", "/Shoot/Day 1/room.wav", "/Shoot/Day 1/Cam A/take1.wav", "/Shoot/Day 1/Cam A/take2.wav"] {
fs.files.get_mut().unwrap().insert(f.into(), wav.clone());
}
let mut s = Session::new(Arc::new(fs));
let r = s.execute("mediaBrowser.import", json!({"paths": ["/Shoot"]})).unwrap();
assert_eq!(r["items"].as_array().unwrap().len(), 4, "{r}");
assert_eq!(r["bins"].as_array().unwrap().len(), 3, "{r}");
let tree = |b: &filmcraft_project::Bin| -> Vec<String> {
b.children
.iter()
.map(|c| match c {
filmcraft_project::BinEntry::Bin(b) => format!("bin {}", b.name),
filmcraft_project::BinEntry::Item(i) => s.project.item(*i).unwrap().name.clone(),
})
.collect()
};
let sub = |b: &filmcraft_project::Bin, name: &str| -> filmcraft_project::Bin {
b.children
.iter()
.find_map(|c| match c {
filmcraft_project::BinEntry::Bin(b) if b.name == name => Some(b.clone()),
_ => None,
})
.unwrap()
};
assert_eq!(tree(&s.project.root), ["bin Shoot"]);
let shoot = sub(&s.project.root, "Shoot");
// a folder without media gets no bin
assert_eq!(tree(&shoot), ["slate.wav", "bin Day 1"]);
let day = sub(&shoot, "Day 1");
assert_eq!(tree(&day), ["room.wav", "bin Cam A"]);
assert_eq!(tree(&sub(&day, "Cam A")), ["take1.wav", "take2.wav"]);
}
#[test]
fn import_with_ingest_settings() {
let mut s = session();

533
crates/engine/src/narration.rs Normal file
View File

@ -0,0 +1,533 @@
//! Text to Speech (narrations): write what the narrator says, pick a voice, and get an audio clip.
//!
//! | command | does |
//! |---|---|
//! | `tts.voices` | languages, voices, vocal pitches, pace limits and the pause marker |
//! | `tts.preview` | synthesize without changing the project (`sample: true` speaks the voice's sample sentence); the host plays [`Session::tts_preview`]; `path` also writes a WAV |
//! | `tts.create` | synthesize, write `Narration <n>.wav`, import it and place it at the playhead (one undo step) |
//! | `tts.edit` | change a narration clip's script or voice settings and regenerate it in place (one undo step) |
//! | `tts.inspect` | the narration settings of a clip or item |
//! | `tts.render` | synthesize in a background job into the cache, so the UI's create / edit / preview return at once |
//! | `tts.downloadVoices` | download the natural voices (Kokoro-82M, 9 US English voices, CMUdict; 336 MB, SHA-256 checked) as a background job |
//!
//! **Where a new narration goes.** At `time` (default: the playhead) on `track` if given (it must
//! be free there), else on the first targeted audio track that is free for the narration's length,
//! else the first free audio track, else on a new audio track. Existing clips are never overwritten.
//!
//! **Editing keeps the clip's length** (user decision, 2026-10-07). The clip keeps its start,
//! duration and speed. When the new speech is longer than the clip, the clip still ends where it
//! did but the file holds the whole speech, so trimming the clip's end out reveals the rest. When it
//! is shorter, the file is padded with silence to the clip's length. The clip's source In is reset
//! to the start of the new speech. Other clips (including other clips of the old narration) are not
//! changed. Each edit writes a new file; the old item and its narration stay, so undo is exact.
//!
//! Files are named `Narration <n>.wav` (mono 32-bit float at the voice's rate, 24 kHz) and saved
//! where voice-over recordings go (Scratch Disks ▸ Captured, else next to the project, else
//! `Narrations` in the data or temporary directory); an edit writes next to the previous file.
use std::sync::{Arc, Mutex, PoisonError};
use serde_json::{Value, json};
use filmcraft_project::{ClipId, ItemId, ItemKind, MediaRef, Narration, TrackId, TrackKind, VocalPitch};
use filmcraft_time::{Tick, TimeRange};
use filmcraft_tts as tts;
use crate::commands::{CommandSpec, always, bad, has_seq, str_p, time_p, u64_p};
use crate::{EngineError, Result, Session};
/// Item name prefix of generated narrations.
const BASE_NAME: &str = "Narration";
/// The narration of a project item.
pub fn narration_of(s: &Session, item: ItemId) -> Option<&Narration> {
s.project.narrations.get(&item)
}
/// The narration clip a command works on: `clip`, else the first selected clip whose item is a
/// narration.
pub fn target_clip(s: &Session, p: &Value) -> Option<ClipId> {
let seq = s.active_sequence()?;
let is_narration = |c: ClipId| seq.find_item(c).is_some_and(|(_, it)| s.project.narrations.contains_key(&it.item));
match u64_p(p, "clip") {
Some(c) => Some(ClipId(c)).filter(|c| is_narration(*c)),
None => s.state.selection.iter().copied().find(|c| is_narration(*c)),
}
}
fn has_narration_clip(s: &Session) -> std::result::Result<(), String> {
has_seq(s)?;
target_clip(s, &Value::Null).map(|_| ()).ok_or_else(|| "select a narration clip".into())
}
/// Script and settings from params, on top of `base`.
fn settings(cmd: &str, p: &Value, base: Option<&Narration>) -> Result<Narration> {
// a setting of the wrong type is an error, never silently replaced by a default
for key in ["text", "voice", "pitch"] {
if p.get(key).is_some_and(|v| !v.is_string()) {
return Err(bad(cmd, format!("`{key}` must be a string")));
}
}
let text = match str_p(p, "text") {
Some(t) => t.to_string(),
None => base.map(|b| b.text.clone()).ok_or_else(|| bad(cmd, "`text` is required"))?,
};
let voice = str_p(p, "voice").map(str::to_string).or_else(|| base.map(|b| b.voice.clone())).unwrap_or_else(|| tts::default_voice_id().to_string());
let info = tts::voices_in(None).into_iter().find(|v| v.id == voice).ok_or_else(|| bad(cmd, format!("unknown voice `{voice}` (see tts.voices)")))?;
let pitch = match str_p(p, "pitch") {
Some(v) => VocalPitch::from_id(v).ok_or_else(|| bad(cmd, format!("unknown pitch `{v}` (extraLow, low, default, high, extraHigh)")))?,
None => base.map(|b| b.pitch).unwrap_or_default(),
};
let pace = match p.get("pace") {
Some(v) => v.as_f64().ok_or_else(|| bad(cmd, "`pace` must be a number"))?,
None => base.map_or(1.0, |b| b.pace),
};
let n = Narration { text, language: info.language.to_string(), voice, pitch, pace, speech_duration: Tick::ZERO };
n.check().map_err(|e| bad(cmd, e))?;
Ok(n)
}
/// Synthesized narrations by settings, newest last (`tts.render` fills it in the background so
/// the UI's create / edit / preview don't block).
pub type SynthCache = Arc<Mutex<Vec<(String, Arc<tts::Audio>)>>>;
const CACHE_ENTRIES: usize = 6;
fn cache_key(n: &Narration) -> String {
format!("{}\u{1f}{}\u{1f}{}\u{1f}{}", n.voice, n.pitch.id(), n.pace.to_bits(), n.text)
}
fn tts_err(cmd: &str, e: tts::TtsError) -> EngineError {
match e {
tts::TtsError::NotInstalled => bad(cmd, "the natural voices are not downloaded yet: Text to Speech ▸ Download natural voices (tts.downloadVoices)"),
e => bad(cmd, e.to_string()),
}
}
/// Synthesize `n` (no cache).
fn render(n: &Narration, models_dir: Option<&std::path::Path>) -> std::result::Result<tts::Audio, tts::TtsError> {
let voice = tts::voice_in(&n.voice, models_dir)?;
voice.synthesize(&n.text, &tts::Params { semitones: n.pitch.semitones(), pace: n.pace })
}
fn synthesize(s: &Session, cmd: &str, n: &Narration) -> Result<Arc<tts::Audio>> {
let key = cache_key(n);
if let Some(a) = s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(k, _)| *k == key).map(|(_, a)| a.clone()) {
return Ok(a);
}
let a = Arc::new(render(n, crate::transcript::models_dir().as_deref()).map_err(|e| tts_err(cmd, e))?);
cache_put(&s.tts_cache, key, a.clone());
Ok(a)
}
fn cache_put(c: &SynthCache, key: String, a: Arc<tts::Audio>) {
let mut c = c.lock().unwrap_or_else(PoisonError::into_inner);
c.retain(|(k, _)| *k != key);
c.push((key, a));
let extra = c.len().saturating_sub(CACHE_ENTRIES);
c.drain(..extra);
}
fn package_json() -> Value {
let dir = crate::transcript::models_dir();
json!({
"id": tts::catalog::PACKAGE_ID,
"name": tts::catalog::PACKAGE_NAME,
"installed": dir.as_deref().is_some_and(tts::catalog::installed),
"size": tts::catalog::size(),
"missingBytes": dir.as_deref().map_or(tts::catalog::size(), tts::catalog::missing_bytes),
"license": tts::catalog::LICENSE,
"source": tts::catalog::SOURCE,
"available": cfg!(feature = "neural-voices"),
})
}
fn voices(_s: &mut Session, _p: &Value) -> Result<Value> {
let dir = crate::transcript::models_dir();
Ok(json!({
"package": package_json(),
"languages": tts::LANGUAGES.iter().map(|(id, name)| json!({"id": id, "name": name})).collect::<Vec<_>>(),
"voices": tts::voices_in(dir.as_deref()),
"defaultVoice": tts::default_voice_id(),
"pitches": VocalPitch::ALL.iter().map(|p| json!({"id": p.id(), "label": p.label(), "semitones": p.semitones()})).collect::<Vec<_>>(),
"pace": {"min": tts::MIN_PACE, "max": tts::MAX_PACE, "default": 1.0},
"pauseMarker": tts::script::PAUSE_MARKER,
"maxTextBytes": tts::MAX_TEXT_BYTES,
"sampleSentence": tts::SAMPLE_SENTENCE,
}))
}
fn preview(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.preview";
let base = target_clip(s, p).and_then(|c| s.active_sequence()?.find_item(c).map(|(_, it)| it.item)).and_then(|i| s.project.narrations.get(&i).cloned());
let mut q = p.clone();
if q.get("sample").and_then(Value::as_bool).unwrap_or(false) {
q["text"] = json!(tts::SAMPLE_SENTENCE);
}
let n = settings(cmd, &q, base.as_ref())?;
let audio = synthesize(s, cmd, &n)?;
let seconds = audio.seconds();
let samples = audio.samples.len();
let rate = audio.sample_rate;
let mut path = Value::Null;
if let Some(dest) = str_p(p, "path") {
let bytes = crate::voiceover::write_wav_f32(&audio.samples, rate);
s.services.write_file(dest, &bytes).map_err(|e| EngineError::Other(format!("{dest}: {e}")))?;
path = json!(dest);
}
s.tts_preview = Some(audio);
Ok(json!({"seconds": seconds, "samples": samples, "sampleRate": rate, "voice": n.voice, "path": path}))
}
/// Write `samples` as a new WAV and import it (its own undo step; callers collapse history).
/// `near`: an edit writes next to the narration's previous file unless `dir` is given.
fn write_and_import(s: &mut Session, p: &Value, samples: &[f32], rate: u32, near: Option<ItemId>) -> Result<(ItemId, String)> {
let beside = near.filter(|_| p.get("dir").is_none()).and_then(|i| match &s.project.items.get(&i)?.kind {
ItemKind::Media(m) => match &m.media {
MediaRef::File { path } => std::path::Path::new(path).parent().filter(|d| !d.as_os_str().is_empty()).map(|d| d.to_string_lossy().into_owned()),
MediaRef::Generator(_) => None,
},
_ => None,
});
let dir = beside.unwrap_or_else(|| crate::voiceover::media_dir(s, p, "Narrations"));
let path = crate::voiceover::unique_wav_path(s, &dir, BASE_NAME)?;
if !cfg!(target_arch = "wasm32") {
std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?;
}
let bytes = crate::voiceover::write_wav_f32(samples, rate);
s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
let item = crate::commands::import_bytes(s, &path, bytes.into(), None)?;
Ok((item, path))
}
fn track_is_free(s: &Session, track: TrackId, range: TimeRange) -> bool {
s.active_sequence()
.and_then(|q| q.audio_tracks.iter().find(|t| t.id == track))
.is_some_and(|t| !t.locked && !t.items.iter().any(|it| it.start < range.end() && range.start < it.end()))
}
/// The track a new narration of `range` goes to (None: add a track).
fn placement(s: &Session, p: &Value, range: TimeRange) -> Result<Option<TrackId>> {
let cmd = "tts.create";
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
if let Some(v) = p.get("track") {
let id =
crate::mixer::strip_ref(seq, v).filter(|id| seq.audio_tracks.iter().any(|t| t.id == *id)).ok_or_else(|| bad(cmd, format!("no audio track {v}")))?;
if !track_is_free(s, id, range) {
return Err(bad(cmd, format!("audio track {v} is locked or has clips there")));
}
return Ok(Some(id));
}
let targeted = s.targeting().targeted;
let order = seq.audio_tracks.iter().filter(|t| targeted.contains(&t.id)).chain(seq.audio_tracks.iter().filter(|t| !targeted.contains(&t.id)));
Ok(order.map(|t| t.id).find(|id| track_is_free(s, *id, range)))
}
fn create(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.create";
let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?;
let mut n = settings(cmd, p, None)?;
let at = time_p(s, p, "").unwrap_or(s.playhead());
if at < Tick::ZERO {
return Err(bad(cmd, "the start time cannot be negative"));
}
let audio = synthesize(s, cmd, &n)?;
let rate = audio.sample_rate.max(1);
let dur = Tick::from_units(audio.samples.len() as i64, i64::from(rate));
n.speech_duration = dur;
let range = TimeRange::new(at, dur);
let track = placement(s, p, range)?;
let n0 = s.history.undo.len();
let result = (|| -> Result<Value> {
let track = match track {
Some(t) => t,
None => {
let before: Vec<TrackId> = s.active_sequence().map(|q| q.audio_tracks.iter().map(|t| t.id).collect()).unwrap_or_default();
s.execute("sequence.addTracks", json!({"video": 0, "audio": 1, "audioAfter": "last"}))?;
s.active_sequence()
.and_then(|q| q.audio_tracks.iter().map(|t| t.id).find(|id| !before.contains(id)))
.ok_or_else(|| EngineError::Other("could not add an audio track".into()))?
}
};
let (item, path) = write_and_import(s, p, &audio.samples, rate, None)?;
let seq_rate = s.project.sequence(seq_id).map(|q| q.settings.frame_rate).unwrap_or_default();
let narration = n.clone();
let clip = s.edit("New Narration", |pr, st| {
let mut it = pr
.make_track_item(item, TrackKind::Audio, at, TimeRange::new(Tick::ZERO, dur), seq_rate)
.ok_or(EngineError::Other("the narration is not importable".into()))?;
it.duration = dur;
let id = it.id;
let seq = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?;
let t = seq.audio_tracks.iter_mut().find(|t| t.id == track).ok_or_else(|| bad(cmd, "the narration track was deleted"))?;
let pos = t.items.partition_point(|x| x.start <= at);
t.items.insert(pos, it);
seq.check().map_err(EngineError::Other)?;
pr.narrations.insert(item, narration);
st.selection = vec![id];
Ok(id)
})?;
let seq = s.project.sequence(seq_id).ok_or(EngineError::NoSequence)?;
Ok(json!({
"item": item.0,
"clip": clip.0,
"track": crate::mixer::strip_label(seq, track),
"start": at.0,
"duration": dur.0,
"speechDuration": dur.0,
"path": path,
"sampleRate": rate,
}))
})();
crate::clip_ops::collapse_history(s, n0, "New Narration");
if result.is_err() && s.history.undo.len() > n0 {
// a step failed after the track was added or the file imported: roll back to before
let _ = s.undo();
s.history.redo.clear();
}
result
}
/// Samples of media a clip of `duration` at `speed` plays, rounded up (exact for 100 %).
fn needed_samples(duration: Tick, speed: f64, rate: u32) -> usize {
let rate = i64::from(rate.max(1));
let media = if speed.is_finite() && speed != 0.0 && (speed.abs() - 1.0).abs() > 1e-12 {
// speed is a ratio, not edit time: scale the exact tick count
Tick::from_seconds_f64(duration.seconds() * speed.abs())
} else {
duration
};
let mut n = media.to_units_floor(rate);
if Tick::from_units(n, rate) < media {
n = n.saturating_add(1);
}
let cap = (tts::MAX_SECONDS * 2.0) as i64 * rate;
n.clamp(0, cap) as usize
}
fn edit(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.edit";
let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?;
let clip = target_clip(s, p).ok_or_else(|| bad(cmd, "select a narration clip (or pass `clip`)"))?;
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
let (track, it) = seq.find_item(clip).ok_or_else(|| bad(cmd, "no such clip"))?;
if seq.audio_tracks.iter().any(|t| t.id == track && t.locked) {
return Err(bad(cmd, "the clip's track is locked"));
}
let (old_item, clip_dur, speed) = (it.item, it.duration, it.speed);
let base = s.project.narrations.get(&old_item).cloned().ok_or_else(|| bad(cmd, "the clip is not a narration"))?;
let mut n = settings(cmd, p, Some(&base))?;
let audio = synthesize(s, cmd, &n)?;
let rate = audio.sample_rate.max(1);
let speech = audio.samples.len();
n.speech_duration = Tick::from_units(speech as i64, i64::from(rate));
// the clip keeps its length: pad the file with silence to what the clip plays
let needed = needed_samples(clip_dur, speed, rate);
let mut samples = audio.samples.clone();
if needed > samples.len() {
samples.try_reserve(needed - samples.len()).map_err(|e| bad(cmd, format!("unable to allocate the narration: {e}")))?;
samples.resize(needed, 0.0);
}
let file_len = samples.len();
let n0 = s.history.undo.len();
let result = (|| -> Result<Value> {
let (item, path) = write_and_import(s, p, &samples, rate, Some(old_item))?;
let name = s.project.items.get(&item).map(|i| i.name.clone()).unwrap_or_default();
let narration = n.clone();
s.edit("Edit Narration", |pr, _| {
let seq = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?;
let (_, it) = seq.find_item_mut(clip).ok_or_else(|| bad(cmd, "the clip was deleted"))?;
it.item = item;
it.name = name;
it.source_in = Tick::ZERO;
seq.check().map_err(EngineError::Other)?;
pr.narrations.insert(item, narration);
Ok(())
})?;
Ok(json!({
"clip": clip.0,
"item": item.0,
"previousItem": old_item.0,
"duration": clip_dur.0,
"speechDuration": n.speech_duration.0,
"fileDuration": Tick::from_units(file_len as i64, i64::from(rate)).0,
"path": path,
"sampleRate": rate,
}))
})();
crate::clip_ops::collapse_history(s, n0, "Edit Narration");
if result.is_err() && s.history.undo.len() > n0 {
let _ = s.undo();
s.history.redo.clear();
}
result
}
fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.inspect";
let (item, clip) = match u64_p(p, "item") {
Some(i) => (ItemId(i), None),
None => {
let c = target_clip(s, p).ok_or_else(|| bad(cmd, "pass `item` or `clip`, or select a narration clip"))?;
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
(seq.find_item(c).map(|(_, it)| it.item).ok_or_else(|| bad(cmd, "no such clip"))?, Some(c.0))
}
};
let n = s.project.narrations.get(&item).ok_or_else(|| bad(cmd, "not a narration"))?;
Ok(json!({"item": item.0, "clip": clip, "narration": n}))
}
fn new_job(s: &mut Session, label: String) -> crate::Job {
let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1;
crate::Job { id, label, progress: Default::default(), result: Default::default() }
}
fn spawn(s: &mut Session, job: crate::Job, wait: bool, name: &str, run: impl FnOnce() + Send + 'static) -> Result<u64> {
let id = job.id;
let run = crate::export_tools::guard_job(job.progress.clone(), job.result.clone(), run);
s.jobs.push(job);
if wait || cfg!(target_arch = "wasm32") {
run();
} else {
std::thread::Builder::new().name(name.into()).spawn(run).map_err(|e| EngineError::Other(e.to_string()))?;
}
Ok(id)
}
type JobResult = std::sync::Mutex<Option<std::result::Result<filmcraft_export::Report, String>>>;
fn finish(prog: &filmcraft_export::Progress, res: &JobResult, r: std::result::Result<String, String>, t0: web_time::Instant) {
use std::sync::atomic::Ordering;
let secs = t0.elapsed().as_secs_f64();
let r = match r {
Ok(status) => {
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = status;
Ok(filmcraft_export::Report { path: String::new(), frames: 0, seconds: secs, bytes: 0, render_fps: 0.0, extra_files: Vec::new() })
}
Err(e) => {
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = e.clone();
*prog.error.lock().unwrap_or_else(PoisonError::into_inner) = Some(e.clone());
Err(e)
}
};
prog.finished.store(true, Ordering::Relaxed);
*res.lock().unwrap_or_else(PoisonError::into_inner) = Some(r);
}
/// Synthesize in the background into the cache, so `tts.create` / `tts.edit` / `tts.preview` with
/// the same settings return at once.
fn render_job(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.render";
let base = target_clip(s, p).and_then(|c| s.active_sequence()?.find_item(c).map(|(_, it)| it.item)).and_then(|i| s.project.narrations.get(&i).cloned());
let mut q = p.clone();
if q.get("sample").and_then(Value::as_bool).unwrap_or(false) {
q["text"] = json!(tts::SAMPLE_SENTENCE);
}
let n = settings(cmd, &q, base.as_ref())?;
let key = cache_key(&n);
if s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|(k, _)| *k == key) {
return Ok(json!({"job": null, "cached": true}));
}
let job = new_job(s, "Synthesizing narration".into());
let (prog, res, cache) = (job.progress.clone(), job.result.clone(), s.tts_cache.clone());
let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
let id = spawn(s, job, wait, "filmcraft-tts", move || {
let t0 = web_time::Instant::now();
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = "Synthesizing…".into();
let r = match render(&n, crate::transcript::models_dir().as_deref()) {
Ok(a) => {
let secs = a.seconds();
cache_put(&cache, key, Arc::new(a));
Ok(format!("{secs:.1} s of speech"))
}
Err(tts::TtsError::NotInstalled) => Err("the natural voices are not downloaded yet".to_string()),
Err(e) => Err(e.to_string()),
};
finish(&prog, &res, r, t0);
})?;
Ok(json!({"job": id, "cached": false}))
}
/// Download the natural voices (Kokoro-82M, its voices and CMUdict) in the background.
fn download_voices(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "tts.downloadVoices";
let dir = crate::transcript::models_dir().ok_or_else(|| bad(cmd, "no data directory for voice models"))?;
if tts::catalog::installed(&dir) {
return Ok(json!({"job": null, "installed": true}));
}
if !cfg!(feature = "neural-voices") {
return Err(bad(cmd, "natural voices are not available in this build (built without the `neural-voices` feature)"));
}
let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
let job = new_job(s, format!("Downloading {}", tts::catalog::PACKAGE_NAME));
job.progress.total.store(tts::catalog::missing_bytes(&dir), std::sync::atomic::Ordering::Relaxed);
let (prog, res) = (job.progress.clone(), job.result.clone());
let id = spawn(s, job, wait, "filmcraft-tts-download", move || {
let t0 = web_time::Instant::now();
let r = download_package(&dir, &prog);
finish(&prog, &res, r.map(|()| "Natural voices installed".to_string()), t0);
})?;
Ok(json!({"job": id, "installed": false}))
}
#[cfg(feature = "neural-voices")]
fn download_package(dir: &std::path::Path, prog: &filmcraft_export::Progress) -> std::result::Result<(), String> {
use std::sync::atomic::Ordering;
let files: Vec<filmcraft_speech::models::ModelFile> =
tts::catalog::FILES.iter().map(|f| filmcraft_speech::models::ModelFile { name: f.name, url: f.url, sha256: f.sha256, size: f.size }).collect();
let mut progress = |done: u64, total: u64, file: &str| {
prog.done.store(done, Ordering::Relaxed);
prog.total.store(total, Ordering::Relaxed);
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = format!("Downloading {file}");
!prog.cancel.load(Ordering::Relaxed)
};
filmcraft_speech::models::download_files(&tts::catalog::package_dir(dir), &files, &mut progress).map_err(|e| e.to_string())
}
#[cfg(not(feature = "neural-voices"))]
fn download_package(_: &std::path::Path, _: &filmcraft_export::Progress) -> std::result::Result<(), String> {
Err("natural voices are not available in this build".into())
}
fn spec(
id: &'static str,
label: &'static str,
params: &'static str,
enabled: fn(&Session) -> std::result::Result<(), String>,
run: fn(&mut Session, &Value) -> Result<Value>,
journal: bool,
) -> CommandSpec {
CommandSpec { id, label, menu: &[], shortcut: None, params, enabled, run, journal }
}
pub fn commands() -> Vec<CommandSpec> {
vec![
spec("tts.voices", "List Narration Voices", "{}", always, voices, false),
spec(
"tts.preview",
"Preview Narration",
r#"{"text":str?,"voice":str?,"pitch":"default"?,"pace":f64?,"sample":bool?,"clip":id?,"path":str?}"#,
always,
preview,
false,
),
spec(
"tts.create",
"New Narration",
r#"{"text":str,"voice":str?,"pitch":"extraLow|low|default|high|extraHigh"?,"pace":0.5..2?,"time":ticks?,"track":"A1"|id?,"dir":str?}"#,
has_seq,
create,
true,
),
spec("tts.edit", "Edit Narration", r#"{"clip":id?,"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"dir":str?}"#, has_narration_clip, edit, true),
spec("tts.inspect", "Inspect Narration", r#"{"clip":id?,"item":id?}"#, always, inspect, false),
spec(
"tts.render",
"Synthesize Narration",
r#"{"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"sample":bool?,"clip":id?,"wait":bool?}"#,
always,
render_job,
false,
),
spec("tts.downloadVoices", "Download Natural Voices", r#"{"wait":bool?}"#, always, download_voices, false),
]
}

425
crates/engine/src/narration_tests.rs Normal file
View File

@ -0,0 +1,425 @@
//! Tests of [`crate::narration`]: Text to Speech narrations are placed without overwriting, keep
//! their clip length when edited, and are one exact undo step.
use super::*;
use filmcraft_project::{ClipId, ItemId};
use serde_json::json;
const RATE: i64 = filmcraft_tts::SAMPLE_RATE as i64;
fn tmp(name: &str) -> String {
let d = std::env::temp_dir().join(format!("filmcraft-tts-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
d.to_string_lossy().into_owned()
}
fn demo() -> Session {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();
s
}
/// The 32-bit float samples of a WAV written by `write_wav_f32`.
fn wav_samples(path: &str) -> Vec<f32> {
let b = std::fs::read(path).unwrap();
b[44..].as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect()
}
fn clip(s: &Session, id: u64) -> filmcraft_project::TrackItem {
s.active_sequence().unwrap().find_item(ClipId(id)).unwrap().1.clone()
}
fn all_clips(s: &Session) -> Vec<filmcraft_project::TrackItem> {
s.active_sequence().unwrap().all_tracks().flat_map(|t| t.items.clone()).collect()
}
#[test]
fn voices_lists_the_built_in_voices() {
let mut s = Session::default();
let r = s.execute("tts.voices", json!({})).unwrap();
let ids: Vec<&str> = r["voices"].as_array().unwrap().iter().map(|v| v["id"].as_str().unwrap()).collect();
assert!(ids.contains(&"basic-female") && ids.contains(&"basic-male"), "{ids:?}");
assert_eq!(r["languages"][0]["id"], "en-US");
assert_eq!(r["pitches"].as_array().unwrap().len(), 5);
assert_eq!(r["pauseMarker"], "[pause 1s]");
}
#[test]
fn create_places_the_narration_at_the_playhead_as_one_undo_step() {
let mut s = demo();
let dir = tmp("create");
s.execute("playhead.set", json!({"seconds": 2.0})).unwrap();
let t0 = s.playhead();
assert!(t0 > Tick::ZERO);
let before = all_clips(&s);
let undo0 = s.history.undo.len();
let r = s.execute("tts.create", json!({"text": "Welcome to FilmCraft.", "voice": "basic-male", "pitch": "low", "pace": 1.2, "dir": dir})).unwrap();
let c = clip(&s, r["clip"].as_u64().unwrap());
assert_eq!(c.start, t0);
// the clip is exactly as long as the speech (sample-exact) and plays it from the start
let samples = wav_samples(r["path"].as_str().unwrap());
assert_eq!(c.duration, Tick::from_units(samples.len() as i64, RATE));
assert_eq!(c.source_in, Tick::ZERO);
assert_eq!(r["speechDuration"], c.duration.0);
let n = s.project.narrations.get(&ItemId(r["item"].as_u64().unwrap())).unwrap();
assert_eq!((n.text.as_str(), n.voice.as_str(), n.pitch, n.pace), ("Welcome to FilmCraft.", "basic-male", VocalPitch::Low, 1.2));
assert_eq!(n.speech_duration, c.duration);
assert_eq!(s.state.selection, vec![c.id]);
// nothing that was there before moved or changed
let after = all_clips(&s);
for b in &before {
assert!(after.contains(b), "clip {:?} changed", b.id);
}
assert_eq!(after.len(), before.len() + 1);
// one undo step that removes the clip and the narration; redo brings both back
assert_eq!(s.history.undo.len(), undo0 + 1);
assert_eq!(s.history.undo.last().unwrap().0, "New Narration");
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(all_clips(&s), before);
assert!(s.project.narrations.is_empty());
s.execute("edit.redo", json!({})).unwrap();
assert_eq!(clip(&s, c.id.0), c);
assert_eq!(s.project.narrations.len(), 1);
let _ = std::fs::remove_dir_all(&dir);
}
use filmcraft_project::VocalPitch;
use filmcraft_time::Tick;
#[test]
fn create_never_overwrites_and_adds_a_track_when_every_track_is_busy() {
let mut s = demo();
let dir = tmp("busy");
// fill every audio track over 0–60 s with a narration-length clip so none is free at 0
let tracks = s.active_sequence().unwrap().audio_tracks.len();
let first = s.execute("tts.create", json!({"text": "One two three.", "time": 0, "dir": dir})).unwrap();
let busy_len = Tick(first["duration"].as_i64().unwrap());
for _ in 1..tracks + 2 {
s.execute("tts.create", json!({"text": "One two three.", "time": 0, "dir": dir})).unwrap();
}
// every create landed on its own track at 0: no two narrations overlap on one track
let seq = s.active_sequence().unwrap();
for t in &seq.audio_tracks {
let mut items: Vec<_> = t.items.iter().map(|i| (i.start, i.end())).collect();
items.sort();
for w in items.windows(2) {
assert!(w[0].1 <= w[1].0, "overlap on {:?}", t.name);
}
}
assert!(seq.audio_tracks.len() > tracks, "a new audio track was added");
assert!(busy_len > Tick::ZERO);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn create_on_an_occupied_or_unknown_track_is_refused_and_changes_nothing() {
let mut s = demo();
let dir = tmp("occupied");
let r = s.execute("tts.create", json!({"text": "Hello.", "time": 0, "track": "A1", "dir": dir}));
if s.active_sequence().unwrap().audio_tracks[0].items.iter().any(|i| i.start <= Tick::ZERO && Tick::ZERO < i.end()) {
assert!(r.is_err(), "A1 is busy at 0");
}
let undo0 = s.history.undo.len();
let items0 = s.project.items.len();
assert!(s.execute("tts.create", json!({"text": "Hello.", "track": "A99", "dir": dir})).is_err());
s.execute("timeline.setTrack", json!({"track": "A2", "locked": true})).unwrap();
let undo1 = s.history.undo.len();
assert!(s.execute("tts.create", json!({"text": "Hello.", "track": "A2", "time": 0, "dir": dir})).is_err());
assert_eq!(s.history.undo.len(), undo1);
assert!(undo1 >= undo0);
assert_eq!(s.project.items.len(), items0, "no file was imported");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn editing_to_longer_speech_keeps_the_clip_length_and_the_whole_speech() {
let mut s = demo();
let dir = tmp("longer");
let r = s.execute("tts.create", json!({"text": "Hi.", "time": 0, "dir": dir})).unwrap();
let c0 = clip(&s, r["clip"].as_u64().unwrap());
let undo0 = s.history.undo.len();
let e = s.execute("tts.edit", json!({"clip": c0.id.0, "text": "Hi. This narration is now a great deal longer than it was before."})).unwrap();
let c1 = clip(&s, c0.id.0);
assert_eq!((c1.id, c1.start, c1.duration, c1.speed), (c0.id, c0.start, c0.duration, c0.speed), "position and length kept");
assert_ne!(c1.item, c0.item, "a new file");
assert_eq!(c1.source_in, Tick::ZERO);
let n = s.project.narrations.get(&c1.item).unwrap();
assert!(n.speech_duration > c1.duration, "the speech is longer than the clip");
// the file holds the whole speech, nothing padded
let samples = wav_samples(e["path"].as_str().unwrap());
assert_eq!(Tick::from_units(samples.len() as i64, RATE), n.speech_duration);
assert_eq!(e["fileDuration"], n.speech_duration.0);
// the old item and its narration are untouched (other clips of it, undo)
assert_eq!(s.project.narrations.get(&c0.item).unwrap().text, "Hi.");
assert_eq!(s.history.undo.len(), undo0 + 1);
assert_eq!(s.history.undo.last().unwrap().0, "Edit Narration");
// the edited file sits next to the first one
assert!(e["path"].as_str().unwrap().starts_with(&dir), "{}", e["path"]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn editing_to_shorter_speech_pads_the_file_with_exact_silence() {
let mut s = demo();
let dir = tmp("shorter");
let r = s.execute("tts.create", json!({"text": "This first version of the narration is fairly long, with several words.", "time": 0, "dir": dir})).unwrap();
let id = r["clip"].as_u64().unwrap();
let c0 = clip(&s, id);
let e = s.execute("tts.edit", json!({"clip": id, "text": "Short."})).unwrap();
let c1 = clip(&s, id);
assert_eq!((c1.start, c1.duration), (c0.start, c0.duration));
let n = s.project.narrations.get(&c1.item).unwrap().clone();
assert!(n.speech_duration < c1.duration);
let samples = wav_samples(e["path"].as_str().unwrap());
let speech = n.speech_duration.to_units_floor(RATE) as usize;
// the file covers the clip, and everything after the speech is exactly zero
assert!(Tick::from_units(samples.len() as i64, RATE) >= c1.duration);
assert!(samples.len() > speech);
assert!(samples[speech..].iter().all(|&x| x == 0.0));
assert!(samples[..speech].iter().any(|&x| x != 0.0));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn undo_and_redo_of_an_edit_are_exact() {
let mut s = demo();
let dir = tmp("undo");
let r = s.execute("tts.create", json!({"text": "Before.", "time": 0, "dir": dir})).unwrap();
let id = r["clip"].as_u64().unwrap();
let before = (*s.project).clone();
s.execute("tts.edit", json!({"clip": id, "text": "After.", "voice": "basic-male", "pitch": "extraHigh", "pace": 0.7})).unwrap();
let after = (*s.project).clone();
assert_ne!(before, after);
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(*s.project, before);
let n = s.project.narrations.get(&clip(&s, id).item).unwrap();
assert_eq!((n.text.as_str(), n.voice.as_str()), ("Before.", "basic-female"));
s.execute("edit.redo", json!({})).unwrap();
assert_eq!(*s.project, after);
let n = s.project.narrations.get(&clip(&s, id).item).unwrap();
assert_eq!((n.text.as_str(), n.pitch, n.pace), ("After.", VocalPitch::ExtraHigh, 0.7));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn edit_without_changes_keeps_settings_and_settings_inherit() {
let mut s = demo();
let dir = tmp("inherit");
let r = s.execute("tts.create", json!({"text": "Keep me.", "voice": "basic-male", "pitch": "high", "pace": 1.5, "time": 0, "dir": dir})).unwrap();
let id = r["clip"].as_u64().unwrap();
s.execute("tts.edit", json!({"clip": id, "pace": 0.8})).unwrap();
let n = s.project.narrations.get(&clip(&s, id).item).unwrap();
assert_eq!((n.text.as_str(), n.voice.as_str(), n.pitch, n.pace), ("Keep me.", "basic-male", VocalPitch::High, 0.8));
let i = s.execute("tts.inspect", json!({"clip": id})).unwrap();
assert_eq!(i["narration"]["pace"], 0.8);
assert_eq!(i["narration"]["pitch"], "high");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn edit_is_disabled_without_a_narration_clip() {
let mut s = demo();
let dir = tmp("enabled");
s.state.selection.clear();
assert!(!s.is_enabled("tts.edit"));
assert!(s.execute("tts.edit", json!({"text": "x"})).is_err());
// a non-narration clip is refused
let other = s.active_sequence().unwrap().all_tracks().find_map(|t| t.items.first().map(|i| i.id)).unwrap();
assert!(s.execute("tts.edit", json!({"clip": other.0, "text": "x"})).is_err());
let r = s.execute("tts.create", json!({"text": "Enable me.", "time": 0, "dir": dir})).unwrap();
assert!(s.is_enabled("tts.edit"), "the new narration is selected");
s.state.selection.clear();
assert!(s.execute("tts.edit", json!({"clip": r["clip"], "text": "By id."})).is_ok());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn edit_on_a_locked_track_is_refused() {
let mut s = demo();
let dir = tmp("locked");
let r = s.execute("tts.create", json!({"text": "Locked.", "time": 0, "dir": dir})).unwrap();
let track = r["track"].as_str().unwrap().to_string();
s.execute("timeline.setTrack", json!({"track": track, "locked": true})).unwrap();
let snap = (*s.project).clone();
assert!(s.execute("tts.edit", json!({"clip": r["clip"], "text": "Changed."})).is_err());
assert_eq!(*s.project, snap);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn hostile_parameters_are_refused_without_changes() {
let mut s = demo();
let dir = tmp("hostile");
let snap = (*s.project).clone();
let undo0 = s.history.undo.len();
let big = "word ".repeat(300_000);
let cases = [
json!({}),
json!({"text": ""}),
json!({"text": " [pause 2s] "}),
json!({"text": "👍"}),
json!({"text": 42}),
json!({"text": big}),
json!({"text": "hi", "voice": "nope"}),
json!({"text": "hi", "voice": 7}),
json!({"text": "hi", "pitch": "loud"}),
json!({"text": "hi", "pace": "fast"}),
json!({"text": "hi", "pace": 5.0}),
json!({"text": "hi", "pace": -1}),
json!({"text": "hi", "pace": 1e308}),
json!({"text": "hi", "time": -5}),
json!({"text": "hi", "track": {"x": 1}}),
json!({"text": "hi", "track": "V1"}),
];
for p in cases {
let mut q = p.clone();
q["dir"] = json!(dir);
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| s.execute("tts.create", q.clone())));
assert!(r.is_ok(), "panicked on {p}");
assert!(r.unwrap().is_err(), "accepted {p}");
assert_eq!(*s.project, snap, "changed by {p}");
assert_eq!(s.history.undo.len(), undo0);
}
for p in [json!({"clip": 999_999_999}), json!({"clip": "x"}), json!({"clip": -1})] {
assert!(s.execute("tts.edit", p.clone()).is_err(), "{p}");
assert!(s.execute("tts.inspect", p.clone()).is_err(), "{p}");
}
assert!(s.execute("tts.preview", json!({"text": "hi", "pace": 9})).is_err());
// no sequence open
let mut empty = Session::default();
assert!(empty.execute("tts.create", json!({"text": "hi", "dir": dir})).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn preview_changes_nothing_and_hands_the_audio_to_the_host() {
let mut s = demo();
let dir = tmp("preview");
std::fs::create_dir_all(&dir).unwrap();
let snap = (*s.project).clone();
let undo0 = s.history.undo.len();
let path = format!("{dir}/p.wav");
let r = s.execute("tts.preview", json!({"text": "Preview me.", "path": path})).unwrap();
assert_eq!(*s.project, snap);
assert_eq!(s.history.undo.len(), undo0);
let a = s.tts_preview.clone().unwrap();
assert_eq!(r["samples"], a.samples.len());
assert_eq!(wav_samples(&path), a.samples);
// the sample sentence for "Hear this voice"
let r = s.execute("tts.preview", json!({"sample": true, "voice": "basic-male"})).unwrap();
assert_eq!(r["voice"], "basic-male");
assert!(r["seconds"].as_f64().unwrap() > 1.0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn narrations_survive_save_and_reload() {
let mut s = demo();
let dir = tmp("save");
let r = s.execute("tts.create", json!({"text": "Saved [pause 500ms] and loaded.", "pitch": "extraLow", "time": 0, "dir": dir})).unwrap();
let bytes = filmcraft_format::encode(&s.project, false);
let loaded = filmcraft_format::decode(&bytes).unwrap();
assert_eq!(loaded.project.narrations, s.project.narrations);
let n = loaded.project.narrations.get(&ItemId(r["item"].as_u64().unwrap())).unwrap();
assert_eq!((n.text.as_str(), n.pitch), ("Saved [pause 500ms] and loaded.", VocalPitch::ExtraLow));
// a project without narrations writes no `narrations` key (older files are unchanged)
let plain = String::from_utf8(filmcraft_format::encode(&demo().project, false)).unwrap();
assert!(!plain.contains("narrations"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_narration_is_heard_in_the_mix() {
let mut s = demo();
let dir = tmp("mix");
let r = s.execute("tts.create", json!({"text": "Can you hear me now?", "time": 0, "dir": dir})).unwrap();
let track = r["track"].as_str().unwrap().to_string();
let n = s.active_sequence().unwrap().audio_tracks.len();
for i in 1..=n {
if format!("A{i}") != track {
s.execute("mixer.setStrip", json!({"strip": format!("A{i}"), "muted": true})).unwrap();
}
}
let provider = s.media.provider(s.project.clone(), s.services.clone());
let sr = s.active_sequence().unwrap().settings.sample_rate as usize;
let mix = filmcraft_render::audio::mix_sequence(&s.project, s.active_sequence().unwrap(), 0, sr, &provider).channels;
let rms = (mix[0].iter().map(|x| f64::from(x * x)).sum::<f64>() / sr as f64).sqrt();
assert!(rms > 0.01, "narration audible, rms {rms}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn render_fills_the_cache_that_create_and_preview_use() {
let mut s = demo();
let dir = tmp("render");
let p = json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1});
let r = s.execute("tts.render", json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1, "wait": true})).unwrap();
let job = r["job"].as_u64().expect("a job");
let j = s.jobs.iter().find(|j| j.id == job).unwrap();
assert!(j.progress.finished.load(std::sync::atomic::Ordering::Relaxed));
assert!(matches!(&*j.result.lock().unwrap(), Some(Ok(_))));
// the same settings are now cached: render again is a no-op, preview hands back the same audio
assert_eq!(s.execute("tts.render", p.clone()).unwrap()["cached"], true);
s.execute("tts.preview", p.clone()).unwrap();
let cached = s.tts_cache.lock().unwrap().last().unwrap().1.clone();
assert!(std::sync::Arc::ptr_eq(s.tts_preview.as_ref().unwrap(), &cached));
let mut q = p.clone();
q["dir"] = json!(dir);
q["time"] = json!(0);
let c = s.execute("tts.create", q).unwrap();
assert_eq!(c["speechDuration"], Tick::from_units(cached.samples.len() as i64, RATE).0);
// a failing render reports the error in the job
let r = s.execute("tts.render", json!({"text": "x", "voice": "kokoro-heart", "wait": true}));
if let Ok(r) = r {
let job = r["job"].as_u64().unwrap();
let j = s.jobs.iter().find(|j| j.id == job).unwrap();
assert!(matches!(&*j.result.lock().unwrap(), Some(Err(_))) || cfg!(feature = "neural-voices"));
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn natural_voices_are_listed_and_explain_what_is_missing() {
let mut s = demo();
let v = s.execute("tts.voices", json!({})).unwrap();
let ids: Vec<&str> = v["voices"].as_array().unwrap().iter().map(|x| x["id"].as_str().unwrap()).collect();
assert!(ids.contains(&"kokoro-heart") && ids.contains(&"kokoro-michael"), "{ids:?}");
assert_eq!(v["package"]["id"], "kokoro-82m");
assert!(v["package"]["size"].as_u64().unwrap() > 300_000_000);
assert_eq!(v["package"]["available"], cfg!(feature = "neural-voices"));
if !v["package"]["installed"].as_bool().unwrap() {
let e = s.execute("tts.create", json!({"text": "Hi.", "voice": "kokoro-heart"})).unwrap_err().to_string();
assert!(e.contains("download") || e.contains("not available"), "{e}");
}
if !cfg!(feature = "neural-voices") && !v["package"]["installed"].as_bool().unwrap() {
let e = s.execute("tts.downloadVoices", json!({})).unwrap_err().to_string();
assert!(e.contains("not available in this build"), "{e}");
}
}
#[test]
fn a_project_subset_keeps_only_the_narrations_of_kept_items() {
let mut s = demo();
let dir = tmp("subset");
let r = s.execute("tts.create", json!({"text": "Hello.", "voice": "basic-male", "dir": dir})).unwrap();
let item = ItemId(r["item"].as_u64().unwrap());
let others: std::collections::BTreeSet<ItemId> = s.project.items.keys().copied().filter(|i| *i != item).collect();
assert!(crate::project_tools::project_subset(&s.project, &others).narrations.is_empty());
let with: std::collections::BTreeSet<ItemId> = [item].into();
assert!(crate::project_tools::project_subset(&s.project, &with).narrations.contains_key(&item));
}
#[test]
fn unique_wav_path_gives_up_instead_of_looping_forever() {
let mut s = demo();
let first = crate::voiceover::unique_wav_path(&s, "/nonexistent-dir", "Narration").unwrap();
assert!(first.ends_with("Narration 1.wav"), "{first}");
// every candidate name taken by a project item: an error, not an endless loop
let template = s.project.items.values().next().cloned().unwrap();
for k in 1..=99_999u32 {
let mut it = template.clone();
it.name = format!("Narration {k}.wav");
std::sync::Arc::make_mut(&mut s.project).items.insert(ItemId(1_000_000 + u64::from(k)), it);
}
assert!(crate::voiceover::unique_wav_path(&s, "/nonexistent-dir", "Narration").is_err());
}

44
crates/engine/src/panels.rs
View File

@ -4,6 +4,7 @@
//! |---|---|
//! | `events.list` / `events.clear` | the Events panel: warnings and errors of commands, background jobs and the auto-save worker ([`EventLog`]) |
//! | `metadata.get` / `metadata.set` | the Metadata panel: an item's clip and file properties; editable log fields (undoable) |
//! | `project.notes` / `project.setNotes` | the Project Notes panel: free plain text saved in the project (undoable) |
//!
//! The Progress panel uses `jobs.list` / `jobs.cancel`; the Timecode panel and the Reference
//! Monitor are frontend state only.
@ -339,6 +340,38 @@ fn metadata_set(s: &mut Session, p: &Value) -> Result<Value> {
Ok(json!({"item": id.0, "name": it.name, "label": it.label.name(), "metadata": it.metadata, "changed": !unchanged}))
}
// ------------------------------------------------------------------------------------ project notes
/// Longest project notes accepted (bytes of UTF-8), so a script or a pasted file cannot grow the
/// project (and every undo step) without bound.
pub const MAX_NOTES_BYTES: usize = 1 << 20;
/// `project.notes`: the project notes.
fn notes_get(s: &mut Session, _: &Value) -> Result<Value> {
Ok(json!({"text": s.project.notes}))
}
/// `project.setNotes {text, merge?}`: replace the project notes as one undo step. Calls with the
/// same `merge` key and no other edit in between share that step (typing in the panel).
fn notes_set(s: &mut Session, p: &Value) -> Result<Value> {
let text = p.get("text").and_then(Value::as_str).ok_or_else(|| bad("project.setNotes", "need `text` (a string)"))?;
if text.len() > MAX_NOTES_BYTES {
return Err(bad("project.setNotes", format!("the notes are too long ({} bytes; at most {MAX_NOTES_BYTES})", text.len())));
}
let changed = s.project.notes != text;
if changed {
let body = |pr: &mut Project, _: &mut crate::EditorState| {
pr.notes = text.to_string();
Ok(())
};
match str_p(p, "merge") {
Some(key) => s.edit_merged("Edit Project Notes", &format!("project.notes:{key}"), body)?,
None => s.edit("Edit Project Notes", body)?,
}
}
Ok(json!({"text": s.project.notes, "changed": changed}))
}
// ------------------------------------------------------------------------------------ registry
pub(crate) fn commands() -> Vec<CommandSpec> {
@ -370,5 +403,16 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
run: metadata_set,
journal: true,
},
q("project.notes", "Get Project Notes", "{}", notes_get),
CommandSpec {
id: "project.setNotes",
label: "Edit Project Notes",
menu: &[],
shortcut: None,
params: r#"{"text":str,"merge":str?}"#,
enabled: always,
run: notes_set,
journal: true,
},
]
}

37
crates/engine/src/panels_tests.rs
View File

@ -156,3 +156,40 @@ fn events_report_background_jobs() {
s.poll_persistence();
assert_eq!(s.log.entries.len(), 6);
}
/// #620: project notes are saved in the project file; typing (one `merge` key) is one undo step.
#[test]
fn project_notes_are_saved_and_typing_is_one_undo_step() {
let mut s = demo();
assert_eq!(s.execute("project.notes", json!({})).unwrap()["text"], "");
let undo0 = s.history.undo.len();
for text in ["T", "To", "To do: fix the logo at 00:01:02:03"] {
s.execute("project.setNotes", json!({"text": text, "merge": "1"})).unwrap();
}
assert_eq!(s.history.undo.len(), undo0 + 1, "one typing session, one undo step");
assert_eq!(s.history.undo.last().unwrap().0, "Edit Project Notes");
// the same text again changes nothing and adds no step
let r = s.execute("project.setNotes", json!({"text": "To do: fix the logo at 00:01:02:03"})).unwrap();
assert_eq!(r["changed"], json!(false));
assert_eq!(s.history.undo.len(), undo0 + 1);
let loaded = filmcraft_format::decode(&filmcraft_format::encode(&s.project, false)).unwrap();
assert_eq!(loaded.project.notes, "To do: fix the logo at 00:01:02:03");
// an older file without notes still loads, with none
let mut doc: Value = serde_json::from_slice(&filmcraft_format::encode(&filmcraft_project::Project::new("old"), false)).unwrap();
doc["schema_version"] = json!(14);
assert_eq!(filmcraft_format::decode(&serde_json::to_vec(&doc).unwrap()).unwrap().project.notes, "");
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(s.project.notes, "");
}
#[test]
fn project_notes_refuse_bad_input() {
let mut s = demo();
assert!(s.execute("project.setNotes", json!({})).is_err());
assert!(s.execute("project.setNotes", json!({"text": 5})).is_err());
let huge = "x".repeat(crate::panels::MAX_NOTES_BYTES + 1);
assert!(s.execute("project.setNotes", json!({"text": huge})).is_err());
assert_eq!(s.project.notes, "");
}

123
crates/engine/src/paste_media.rs Normal file
View File

@ -0,0 +1,123 @@
//! Paste media from the system clipboard (#611). The desktop app reads the clipboard: copied files
//! paste by path, and an image is first saved by [`save_pasted_image`] as `Pasted Image <n>.png`
//! where voice-over takes and narrations go (Scratch Disks ▸ Captured Audio and Video, else next
//! to the project).
//!
//! `edit.pasteMedia` imports the files and puts them on the Timeline at the playhead, end to end,
//! without overwriting anything: each clip goes on the first free track from the targeted one
//! outward (up for video, down for audio), and a track is added when every track is busy there.
//! With `insert` they are inserted on the targeted tracks instead, as Paste Insert does.
use filmcraft_project::{BinId, ClipId, ItemId, TrackId, TrackKind};
use filmcraft_time::Tick;
use serde_json::{Value, json};
use crate::commands::{CommandSpec, always, bad, bool_p, u64_p};
use crate::{EngineError, Result, Session};
/// Write a pasted image (PNG bytes) to a new `Pasted Image <n>.png`; returns its path.
pub fn save_pasted_image(s: &mut Session, png: &[u8]) -> Result<String> {
let dir = crate::voiceover::media_dir(s, &Value::Null, "Pasted Images");
let names: std::collections::HashSet<String> = s.project.items.values().map(|i| i.name.clone()).collect();
// bounded like voiceover::unique_wav_path: `file_size` is a filesystem call per try
let path = (1..=99_999u32)
.map(|k| format!("Pasted Image {k}.png"))
.map(|file| (std::path::Path::new(&dir).join(&file).to_string_lossy().into_owned(), file))
.find(|(path, file)| !names.contains(file) && s.services.file_size(path).is_err())
.map(|(path, _)| path)
.ok_or_else(|| EngineError::Other(format!("{dir}: no free file name for \"Pasted Image <n>.png\"")))?;
if !cfg!(target_arch = "wasm32") {
std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?;
}
s.services.write_file(&path, png).map_err(|e| EngineError::Other(format!("{path}: {e}")))?;
Ok(path)
}
fn paste_media(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "edit.pasteMedia";
let paths: Vec<String> =
p.get("paths").and_then(Value::as_array).map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_string)).collect()).unwrap_or_default();
if paths.is_empty() {
return Err(bad(cmd, "need `paths`"));
}
let place = bool_p(p, "place").unwrap_or(true);
if place && s.active_sequence().is_none() {
return Err(EngineError::NoSequence);
}
let insert = bool_p(p, "insert").unwrap_or(false);
let n0 = s.history.undo.len();
let r = import_and_place(s, &paths, u64_p(p, "bin").map(BinId), place, insert);
crate::clip_ops::collapse_history(s, n0, if insert { "Paste Insert" } else { "Paste" });
r
}
fn import_and_place(s: &mut Session, paths: &[String], bin: Option<BinId>, place: bool, insert: bool) -> Result<Value> {
let mut import = json!({"paths": paths});
if let Some(b) = bin {
import["bin"] = json!(b.0);
}
let imported = s.execute("file.import", import)?;
let items: Vec<ItemId> = imported["items"].as_array().map(|a| a.iter().filter_map(Value::as_u64).map(ItemId).collect()).unwrap_or_default();
if !place {
return Ok(json!({"items": items.iter().map(|i| i.0).collect::<Vec<_>>(), "clips": []}));
}
let mut at = s.playhead();
let mut clips: Vec<ClipId> = Vec::new();
for item in &items {
let Some(pi) = s.project.item(*item) else { continue };
let (dur, video, audio) = (crate::commands::full_duration(s, pi), pi.has_video(), pi.has_audio());
let mut params = json!({"item": item.0, "time": at.0, "insert": insert});
if !insert {
if video {
params["track"] = json!(free_track(s, TrackKind::Video, at, at + dur)?.0);
}
if audio {
params["audioTrack"] = json!(free_track(s, TrackKind::Audio, at, at + dur)?.0);
}
}
let placed = s.execute("timeline.place", params)?;
let ids: Vec<ClipId> = placed["clips"].as_array().map(|a| a.iter().filter_map(Value::as_u64).map(ClipId).collect()).unwrap_or_default();
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
at = ids.iter().filter_map(|c| seq.find_item(*c)).map(|(_, it)| it.end()).max().unwrap_or(at + dur);
clips.extend(ids);
}
// the playhead goes to the end of what was pasted, as after Paste
s.set_playhead(at);
s.state.selection = clips.clone();
Ok(json!({"items": items.iter().map(|i| i.0).collect::<Vec<_>>(), "clips": clips.iter().map(|c| c.0).collect::<Vec<_>>()}))
}
/// The first unlocked track of `kind` from the targeted one outward with nothing in `start..end`.
/// When every track is busy there, a new one is added (on top for video, at the bottom for audio).
fn free_track(s: &mut Session, kind: TrackKind, start: Tick, end: Tick) -> Result<TrackId> {
let seq = s.active_sequence().ok_or(EngineError::NoSequence)?;
let tg = s.targeting();
let (tracks, dest) = match kind {
TrackKind::Video => (&seq.video_tracks, tg.video_dest),
TrackKind::Audio => (&seq.audio_tracks, tg.audio_dest),
};
let from = dest.and_then(|d| tracks.iter().position(|t| t.id == d)).unwrap_or(0);
let free = tracks.iter().skip(from).find(|t| !t.locked && !t.items.iter().any(|i| i.start < end && i.end() > start)).map(|t| t.id);
if let Some(t) = free {
return Ok(t);
}
let (params, key) = match kind {
TrackKind::Video => (json!({"video": 1}), "video"),
TrackKind::Audio => (json!({"video": 0, "audio": 1}), "audio"),
};
let added = s.execute("sequence.addTracks", params)?;
added[key][0].as_u64().map(TrackId).ok_or_else(|| EngineError::Other("no track was added".into()))
}
pub(crate) fn commands() -> Vec<CommandSpec> {
vec![CommandSpec {
id: "edit.pasteMedia",
label: "Paste Media",
menu: &[],
shortcut: None,
params: r#"{"paths":[str],"insert":bool?,"place":bool=true,"bin":binId?}"#,
enabled: always,
run: paste_media,
journal: true,
}]
}

173
crates/engine/src/paste_media_tests.rs Normal file
View File

@ -0,0 +1,173 @@
//! Pasting files and images from the system clipboard (#611).
use std::path::Path;
use filmcraft_project::{BinEntry, BinId, ClipId, ItemId, TrackKind};
use filmcraft_time::Tick;
use serde_json::{Value, json};
use crate::Session;
use crate::media_test_util::tmp_dir;
fn demo() -> Session {
let mut s = Session::default();
s.execute("file.openDemoProject", json!({})).unwrap();
s
}
fn write_png(path: &Path) -> String {
image::RgbaImage::from_pixel(64, 36, image::Rgba([40, 120, 200, 255])).save(path).unwrap();
path.to_string_lossy().into_owned()
}
fn write_wav(path: &Path, seconds: f32) -> String {
std::fs::write(path, crate::voiceover::write_wav_f32(&vec![0.0; (48_000.0 * seconds) as usize], 48_000)).unwrap();
path.to_string_lossy().into_owned()
}
/// Every clip of the active sequence: (track kind, track index, clip id, start, end).
fn clips(s: &Session) -> Vec<(TrackKind, usize, u64, Tick, Tick)> {
let q = s.active_sequence().unwrap();
let mut v = Vec::new();
for (kind, tracks) in [(TrackKind::Video, &q.video_tracks), (TrackKind::Audio, &q.audio_tracks)] {
for (i, t) in tracks.iter().enumerate() {
v.extend(t.items.iter().map(|c| (kind, i, c.id.0, c.start, c.end())));
}
}
v
}
fn pasted(r: &Value) -> Vec<u64> {
r["clips"].as_array().unwrap().iter().map(|c| c.as_u64().unwrap()).collect()
}
/// An image pasted over footage goes on the first video track that is free there, from the
/// targeted one up; every clip already on the Timeline stays as it was. One undo step takes the
/// clip and the imported item back; the playhead moves to its end, as after Paste.
#[test]
fn a_pasted_image_sits_above_the_footage_without_overwriting_it() {
let dir = tmp_dir("paste-media-image");
let mut s = demo();
let png = write_png(&dir.join("shot.png"));
let at = s.playhead();
let (before, items, undo) = (clips(&s), s.project.items.len(), s.history.undo.len());
assert!(before.iter().any(|c| c.0 == TrackKind::Video && c.1 == 0 && c.3 <= at && c.4 > at), "the playhead is over a V1 clip");
let r = s.execute("edit.pasteMedia", json!({"paths": [png]})).unwrap();
let ids = pasted(&r);
assert_eq!(ids.len(), 1, "a still has picture only");
let after = clips(&s);
for c in &before {
assert!(after.contains(c), "{c:?} is untouched");
}
let new = *after.iter().find(|c| c.2 == ids[0]).unwrap();
assert_eq!((new.0, new.3), (TrackKind::Video, at));
assert!(new.1 > 0, "not on V1, where the footage is");
assert!(!before.iter().any(|c| c.0 == TrackKind::Video && c.1 == new.1 && c.3 < new.4 && c.4 > new.3), "on a track that was free there");
assert!(before.iter().any(|c| c.0 == TrackKind::Video && c.1 < new.1 && c.3 < new.4 && c.4 > new.3), "so it sits above a clip");
assert_eq!(new.4 - new.3, s.prefs.timeline.still_duration(s.sequence_rate()), "for the still image default duration");
assert_eq!(s.playhead(), new.4);
assert_eq!(s.state.selection, vec![ClipId(ids[0])]);
assert_eq!(s.history.undo.len(), undo + 1);
assert_eq!(s.undo().as_deref(), Some("Paste"));
assert_eq!((clips(&s), s.project.items.len()), (before, items), "undo takes back the clip and the import");
let _ = std::fs::remove_dir_all(&dir);
}
/// With every video track busy at the playhead a new one is added on top; undo removes it too.
/// Audio goes on the first free audio track from the targeted one down, or a new one at the bottom.
#[test]
fn a_track_is_added_when_every_track_is_busy() {
let dir = tmp_dir("paste-media-tracks");
let mut s = demo();
let png = write_png(&dir.join("a.png"));
let wav = write_wav(&dir.join("b.wav"), 2.0);
let at = s.playhead();
// fill every track at the playhead
for _ in 0..8 {
let q = s.active_sequence().unwrap();
let busy = |ts: &[filmcraft_project::Track]| ts.iter().all(|t| t.items.iter().any(|c| c.start <= at && c.end() > at + Tick::from_seconds_f64(5.0)));
if busy(&q.video_tracks) && busy(&q.audio_tracks) {
break;
}
s.set_playhead(at);
s.execute("edit.pasteMedia", json!({"paths": [png.clone(), wav.clone()]})).unwrap();
}
let (nv, na) = {
let q = s.active_sequence().unwrap();
(q.video_tracks.len(), q.audio_tracks.len())
};
let before = clips(&s);
s.set_playhead(at);
let r = s.execute("edit.pasteMedia", json!({"paths": [png, wav]})).unwrap();
let ids = pasted(&r);
let q = s.active_sequence().unwrap();
assert_eq!((q.video_tracks.len(), q.audio_tracks.len()), (nv + 1, na + 1), "one new track of each kind");
let after = clips(&s);
let image = after.iter().find(|c| c.2 == ids[0]).unwrap();
assert_eq!((image.0, image.1), (TrackKind::Video, nv), "the image on the new top track");
let sound = after.iter().find(|c| c.2 == ids[1]).unwrap();
assert_eq!((sound.0, sound.1, sound.3), (TrackKind::Audio, na, image.4), "the sound on the new bottom track, after the image");
for c in &before {
assert!(after.contains(c), "{c:?} is untouched");
}
s.undo();
let q = s.active_sequence().unwrap();
assert_eq!((q.video_tracks.len(), q.audio_tracks.len()), (nv, na), "undo removes the tracks it added");
let _ = std::fs::remove_dir_all(&dir);
}
/// `insert` is Paste Insert: on the targeted video track, pushing later clips right.
#[test]
fn paste_insert_inserts_on_the_targeted_track() {
let dir = tmp_dir("paste-media-insert");
let mut s = demo();
for t in ["V2", "V3", "A2", "A3"] {
let _ = s.execute("timeline.setTrack", json!({"track": t, "syncLock": false}));
}
let png = write_png(&dir.join("i.png"));
let at = s.playhead();
let last = s.active_sequence().unwrap().video_tracks[0].items.iter().map(|c| c.end()).max().unwrap();
let r = s.execute("edit.pasteMedia", json!({"paths": [png], "insert": true})).unwrap();
let new = *clips(&s).iter().find(|c| c.2 == pasted(&r)[0]).unwrap();
let still = s.prefs.timeline.still_duration(s.sequence_rate());
assert_eq!((new.0, new.1, new.3), (TrackKind::Video, 0, at), "on the targeted track, V1");
assert_eq!(s.active_sequence().unwrap().video_tracks[0].items.iter().map(|c| c.end()).max().unwrap(), last + still, "later clips moved right");
assert_eq!(s.undo().as_deref(), Some("Paste Insert"));
let _ = std::fs::remove_dir_all(&dir);
}
/// Without `place` (the Project panel) the files are only imported, into `bin`.
#[test]
fn without_place_files_are_only_imported() {
let dir = tmp_dir("paste-media-import");
let mut s = demo();
let png = write_png(&dir.join("p.png"));
let bin = s.execute("file.newBin", json!({"name": "Pasted"})).unwrap()["bin"].as_u64().unwrap();
let before = clips(&s);
let r = s.execute("edit.pasteMedia", json!({"paths": [png], "place": false, "bin": bin})).unwrap();
assert_eq!(clips(&s), before);
let item = ItemId(r["items"][0].as_u64().unwrap());
let pasted_bin = s.project.root.find_bin(BinId(bin)).unwrap();
assert!(pasted_bin.children.iter().any(|e| matches!(e, BinEntry::Item(i) if *i == item)), "imported into the bin");
assert!(s.execute("edit.pasteMedia", json!({"paths": []})).is_err(), "nothing to paste");
let _ = std::fs::remove_dir_all(&dir);
}
/// A pasted image is written as `Pasted Image <n>.png` next to the saved project, with the first
/// number no file or project item uses.
#[test]
fn pasted_images_are_saved_next_to_the_project() {
let dir = tmp_dir("paste-media-save");
let mut s = demo();
s.path = Some(dir.join("film.fcproj").to_string_lossy().into_owned());
let mut png = Vec::new();
image::RgbaImage::from_pixel(8, 8, image::Rgba([0, 0, 0, 255])).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap();
let first = crate::paste_media::save_pasted_image(&mut s, &png).unwrap();
assert_eq!(Path::new(&first), dir.join("Pasted Image 1.png"));
assert_eq!(std::fs::read(&first).unwrap(), png);
let second = crate::paste_media::save_pasted_image(&mut s, &png).unwrap();
assert_eq!(Path::new(&second), dir.join("Pasted Image 2.png"));
s.execute("edit.pasteMedia", json!({"paths": [second]})).unwrap();
assert!(s.project.items.values().any(|i| i.name == "Pasted Image 2.png"));
let _ = std::fs::remove_dir_all(&dir);
}

18
crates/engine/src/presets.rs
View File

@ -365,12 +365,24 @@ pub fn instantiate(p: &EffectPreset, target: &TrackItem, size: (u32, u32)) -> Ve
out
}
/// Put preset effects on a clip: intrinsic ones replace the clip's instance, others are inserted
/// before the intrinsic effects.
/// Keep `old`'s keyframes that lie outside the span `new` animates, so presets keyframing the same
/// parameter at different times (Fade In, then Fade Out) add up instead of replacing each other.
fn merge_keyframes(new: &mut EffectInstance, old: &EffectInstance) {
for (id, p) in new.params.iter_mut() {
let (Some(first), Some(last)) = (p.keyframes.first().map(|k| k.time), p.keyframes.last().map(|k| k.time)) else { continue };
let Some(o) = old.params.get(id) else { continue };
p.keyframes.extend(o.keyframes.iter().filter(|k| k.time < first || k.time > last).cloned());
p.keyframes.sort_by_key(|k| k.time);
}
}
/// Put preset effects on a clip: intrinsic ones replace the clip's instance (keeping its keyframes
/// outside the span the preset animates), others are inserted before the intrinsic effects.
pub fn put_effects(it: &mut TrackItem, effects: Vec<EffectInstance>) {
for e in effects {
for mut e in effects {
let intrinsic = e.def().is_some_and(|d| d.intrinsic);
if intrinsic && let Some(slot) = it.effects.iter_mut().find(|x| x.effect == e.effect) {
merge_keyframes(&mut e, slot);
*slot = e;
continue;
}

19
crates/engine/src/presets_tests.rs
View File

@ -194,6 +194,25 @@ fn builtins_apply_and_user_presets_persist() {
let _ = std::fs::remove_dir_all(&dir);
}
/// #333: Fade In then Fade Out on one clip keeps both ramps (they keyframe the same Opacity).
#[test]
fn fade_in_and_fade_out_add_up_on_one_clip() {
let (mut s, clips) = demo();
let c = clips[0];
s.execute("presets.apply", json!({"preset": "Fade In", "clips": [c.0]})).unwrap();
s.execute("presets.apply", json!({"preset": "Fade Out", "clips": [c.0]})).unwrap();
let it = item(&s, c);
let op = it.effect("opacity").unwrap().param("opacity").unwrap();
let keys: Vec<(Tick, ParamValue)> = op.keyframes.iter().map(|k| (k.time, k.value.clone())).collect();
let (tin, tout) = (it.source_in, it.source_in + media_len(&it));
let f = ParamValue::Float;
assert_eq!(keys, vec![(tin, f(0.0)), (tin + secs(1.0), f(100.0)), (tout - secs(1.0), f(100.0)), (tout, f(0.0))], "both fades survive");
// re-applying Fade In replaces its own ramp, not the Fade Out one
s.execute("presets.apply", json!({"preset": "Fade In", "clips": [c.0]})).unwrap();
let it = item(&s, c);
assert_eq!(it.effect("opacity").unwrap().param("opacity").unwrap().keyframes.len(), 4);
}
#[test]
fn mask_geometry_scales_to_the_target_frame_size() {
let (s, clips) = demo();

46
crates/engine/src/project_panel.rs
View File

@ -17,9 +17,9 @@
//! | `project.freeform.saveArrangement` / `.restoreArrangement` / `.arrangements` / `.deleteArrangement` | saved arrangements |
//! | `project.freeform.options` | Freeform View Options… (grid, snap, names, durations) |
//! | `project.renameBin` | rename a bin (inline rename, undoable) |
//! | `project.selectAll` / `project.deselectAll` | select every item in the project / clear the Project panel selection (Cmd+A / Cmd+Shift+A with the panel focused) |
//! | `project.selectAll` / `project.deselectAll` | select the items a bin shows (`bin`, plus the sub-bins open in List view in `expanded`; without `bin` every item in the project) / clear the Project panel selection (Cmd+A / Cmd+Shift+A with the panel focused) |
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use filmcraft_project::{Bin, BinEntry, BinId, ItemId, ItemKind, MediaRef, Project, ProjectItem, TrackKind};
use filmcraft_time::{Tick, TimeDisplay, format_time};
@ -1171,6 +1171,43 @@ fn rename_bin(s: &mut Session, p: &Value) -> Result<Value> {
Ok(json!({"bin": bin.0, "name": name}))
}
/// Select All in the Project panel. With `bin`, only what a panel showing that bin lists: its
/// items, plus the items of the sub-bins named in `expanded` (bins twirled open in List view),
/// never those of a closed bin. Without `bin`, every item in the project.
fn select_all(s: &mut Session, p: &Value) -> Result<Value> {
const CMD: &str = "project.selectAll";
// strict: a negative or fractional `bin` is an error, not bin 0
let id = match p.get("bin") {
None | Some(Value::Null) => {
s.state.project_selection = s.project.items.keys().copied().collect();
return Ok(json!({"selected": s.state.project_selection.len()}));
}
Some(b) => BinId(b.as_u64().ok_or_else(|| bad(CMD, "`bin` must be a bin id"))?),
};
let bin = s.project.root.find_bin(id).ok_or_else(|| bad(CMD, format!("no bin {}", id.0)))?;
let expanded: BTreeSet<BinId> = match p.get("expanded") {
None | Some(Value::Null) => Default::default(),
Some(Value::Array(a)) => {
a.iter().map(|v| v.as_u64().map(BinId).ok_or_else(|| bad(CMD, "`expanded` must be a list of bin ids"))).collect::<Result<_>>()?
}
Some(_) => return Err(bad(CMD, "`expanded` must be a list of bin ids")),
};
// iterative walk: a hostile project file cannot overflow the stack with deeply nested bins
let mut sel = Vec::new();
let mut stack = vec![bin];
while let Some(b) = stack.pop() {
for e in &b.children {
match e {
BinEntry::Item(i) if s.project.item(*i).is_some_and(|it| listed(&s.project, it)) => sel.push(*i),
BinEntry::Bin(sub) if expanded.contains(&sub.id) => stack.push(sub),
_ => {}
}
}
}
s.state.project_selection = sel;
Ok(json!({"bin": id.0, "selected": s.state.project_selection.len()}))
}
// ------------------------------------------------------------------------------------ registry
pub(crate) fn commands() -> Vec<CommandSpec> {
@ -1223,10 +1260,7 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
freeform_options,
),
c("project.renameBin", "Rename Bin", r#"{"bin":binId,"name":str}"#, always, rename_bin),
c("project.selectAll", "Select All Project Items", "{}", always, |s, _| {
s.state.project_selection = s.project.items.keys().copied().collect();
Ok(json!({"selected": s.state.project_selection.len()}))
}),
c("project.selectAll", "Select All Project Items", r#"{"bin":binId?,"expanded":[binId]?}"#, always, select_all),
c("project.deselectAll", "Deselect All Project Items", "{}", always, |s, _| {
s.state.project_selection.clear();
Ok(Value::Null)

74
crates/engine/src/project_panel_tests.rs
View File

@ -2,7 +2,7 @@
//! sorting, view presets, Freeform positions / stacks / arrangements (undoable, saved with the
//! project) and bin renames.
use filmcraft_project::{ItemId, ItemKind};
use filmcraft_project::{BinEntry, ItemId, ItemKind};
use serde_json::{Value, json};
use crate::Session;
@ -305,3 +305,75 @@ fn select_all_selects_every_project_item() {
assert!(!s.state.selection.is_empty());
assert_eq!(s.state.project_selection, before);
}
#[test]
fn select_all_in_a_bin_skips_closed_bins_and_other_bins() {
// #456: Cmd+A in the Project panel selected every item in the project, closed bins included
let mut s = demo();
let outer = s.execute("file.newBin", json!({"name": "Footage A"})).unwrap()["bin"].as_u64().unwrap();
let inner = s.execute("file.newBin", json!({"name": "Day 1", "parent": outer})).unwrap()["bin"].as_u64().unwrap();
let ids = media(&s);
assert!(ids.len() >= 2);
s.execute("project.moveToBin", json!({"items": [ids[0].0], "bin": outer})).unwrap();
s.execute("project.moveToBin", json!({"items": [ids[1].0], "bin": inner})).unwrap();
s.state.project_selection.clear();
// only the bin's own items: the closed sub-bin's item stays unselected
let r = s.execute("project.selectAll", json!({"bin": outer})).unwrap();
assert_eq!(r["selected"].as_u64(), Some(1));
assert_eq!(s.state.project_selection, vec![ids[0]]);
// a sub-bin twirled open in List view counts
s.execute("project.selectAll", json!({"bin": outer, "expanded": [inner]})).unwrap();
let mut sel = s.state.project_selection.clone();
sel.sort();
assert_eq!(sel, vec![ids[0], ids[1]]);
// the root: its direct items only, nothing from the (closed) bins
let root = s.project.root.id;
let mut want: Vec<ItemId> = s
.project
.root
.children
.iter()
.filter_map(|e| if let BinEntry::Item(i) = e { Some(*i) } else { None })
.filter(|i| s.project.item(*i).is_some_and(|it| project_panel::listed(&s.project, it)))
.collect();
want.sort();
s.execute("project.selectAll", json!({"bin": root.0})).unwrap();
let mut sel = s.state.project_selection.clone();
sel.sort();
assert_eq!(sel, want);
assert!(!sel.contains(&ids[0]) && !sel.contains(&ids[1]));
assert!(sel.len() < s.project.items.len());
// an open bin inside a closed one is not shown, so it does not count either
s.execute("project.selectAll", json!({"bin": root.0, "expanded": [inner]})).unwrap();
assert!(!s.state.project_selection.contains(&ids[1]));
// ... but open all the way down, it does
s.execute("project.selectAll", json!({"bin": root.0, "expanded": [outer, inner]})).unwrap();
assert!(s.state.project_selection.contains(&ids[0]) && s.state.project_selection.contains(&ids[1]));
}
#[test]
fn select_all_rejects_bad_bins_without_touching_the_selection() {
let mut s = demo();
let ids = media(&s);
let root = s.project.root.id.0;
s.execute("project.select", json!({"items": [ids[0].0]})).unwrap();
for p in [
json!({"bin": 987_654_321u64}),
json!({"bin": "footage"}),
json!({"bin": -1}),
json!({"bin": root, "expanded": "all"}),
json!({"bin": root, "expanded": ["x"]}),
] {
assert!(s.execute("project.selectAll", p.clone()).is_err(), "{p}");
assert_eq!(s.state.project_selection, vec![ids[0]], "{p}");
}
// unknown ids in `expanded` are simply not open bins
assert!(s.execute("project.selectAll", json!({"bin": root, "expanded": [987_654_321u64]})).is_ok());
// `bin: null` is the same as no bin: the whole project
let r = s.execute("project.selectAll", json!({"bin": null})).unwrap();
assert_eq!(r["selected"].as_u64(), Some(s.project.items.len() as u64));
}

45
crates/engine/src/project_tools.rs
View File

@ -22,9 +22,9 @@
//! | `clip.automateToSequence` | Clip ▸ Automate to Sequence… |
//! | `help.systemReport` | query behind Help ▸ System Compatibility Report… |
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use filmcraft_project::{BinEntry, BinId, FindOp, FindQuery, FindRow, ItemId, ItemKind, MarkerKind, MediaRef, Project, SearchBin, TrackKind};
use filmcraft_project::{BinEntry, FindOp, FindQuery, FindRow, ItemId, ItemKind, MarkerKind, MediaRef, Project, SearchBin, TrackKind};
use filmcraft_time::{Tick, TimeDisplay, TimeRange};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
@ -704,18 +704,42 @@ fn new_from_template(s: &mut Session, p: &Value) -> Result<Value> {
/// everything those items need), as `project.delete` does. The project's own top bin does not,
/// since no command acts on it (#244), and only ids that name a real item end up in the result.
pub fn closure(p: &Project, roots: &[ItemId]) -> BTreeSet<ItemId> {
// Index the bin tree once. Told apart from items with a set membership test (rather than a
// full `find_bin` per queued id, which is O(n²) when exporting many items from a flat
// project) and expanded from the indexed children (rather than a second walk per bin).
let mut bins: BTreeSet<u64> = BTreeSet::new();
let mut children: BTreeMap<u64, (Vec<ItemId>, Vec<u64>)> = BTreeMap::new();
let mut bin_stack = vec![&p.root];
while let Some(b) = bin_stack.pop() {
bins.insert(b.id.0);
let mut direct = (Vec::new(), Vec::new());
for c in &b.children {
match c {
BinEntry::Item(i) => direct.0.push(*i),
BinEntry::Bin(x) => {
direct.1.push(x.id.0);
bin_stack.push(x);
}
}
}
children.insert(b.id.0, direct);
}
let mut keep: BTreeSet<ItemId> = BTreeSet::new();
let mut todo: Vec<ItemId> = roots.to_vec();
let mut seen_bins: BTreeSet<u64> = BTreeSet::new();
let mut bin_todo: Vec<u64> = Vec::new();
while let Some(id) = todo.pop() {
let bin = BinId(id.0);
if bin != p.root.id && p.root.find_bin(bin).is_some() {
if seen_bins.insert(id.0)
&& let Some(b) = p.root.find_bin(bin)
{
let mut items = Vec::new();
b.all_items(&mut items);
todo.extend(items);
if bins.contains(&id.0) && id.0 != p.root.id.0 {
bin_todo.push(id.0);
while let Some(bid) = bin_todo.pop() {
if !seen_bins.insert(bid) {
continue;
}
if let Some((items, subs)) = children.get(&bid) {
todo.extend(items.iter().copied());
bin_todo.extend(subs.iter().copied());
}
}
continue;
}
@ -756,6 +780,7 @@ pub fn project_subset(p: &Project, keep: &BTreeSet<ItemId>) -> Project {
out.items.retain(|id, _| keep.contains(id));
prune(&mut out.root, keep);
out.transcripts.retain(|id, _| keep.contains(id));
out.narrations.retain(|id, _| keep.contains(id));
out.search_bins.clear();
out
}

64
crates/engine/src/sequence_tools.rs
View File

@ -131,7 +131,7 @@ fn groups() -> Vec<(&'static str, Vec<CommandSpec>)> {
"Delete Tracks…",
&["Sequence"],
None,
r#"{"video":"empty"|"V2"|id?,"audio":"empty"|"A2"|id?}"#,
r#"{"video":"empty"|"V2"|id?,"audio":"empty"|"A2"|id?,"captions":"empty"|"C2"|id?}"#,
has_seq,
delete_tracks,
)],
@ -723,6 +723,54 @@ pub(crate) fn add_tracks(s: &mut Session, p: &Value) -> Result<Value> {
Ok(json!({"video": video, "audio": audio, "submix": submix}))
}
/// `timeline.place` names `"new"` as its `track` or `audioTrack`.
pub(crate) fn wants_new_track(p: &Value) -> bool {
["track", "audioTrack"].iter().any(|k| p.get(*k).and_then(Value::as_str).is_some_and(|v| v.eq_ignore_ascii_case("new")))
}
/// `timeline.place` with `"track":"new"` and/or `"audioTrack":"new"` (#483: media dropped in the
/// empty space above the video tracks or below the audio tracks): a track is added after the
/// last one of its kind for each stream the item carries (and the call enables), then the clip
/// goes on it. One undo step; when the placement fails, the new tracks go away again.
pub(crate) fn place_on_new_tracks(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "timeline.place";
let item = crate::commands::item_p(p, "item").ok_or_else(|| bad(cmd, "need `item`"))?;
let pi = s.project.item(item).ok_or_else(|| bad(cmd, "no such item"))?;
let is_new = |k: &str| p.get(k).and_then(Value::as_str).is_some_and(|v| v.eq_ignore_ascii_case("new"));
let enabled = |k: &str| p.get(k).and_then(Value::as_bool).unwrap_or(true);
let new_v = is_new("track") && pi.has_video() && enabled("video");
let new_a = is_new("audioTrack") && pi.has_audio() && enabled("audio");
if !new_v && !new_a {
return Err(bad(cmd, "no new track to add: the item has no picture or sound for it"));
}
let added = add_tracks(s, &json!({"video": u64::from(new_v), "audio": u64::from(new_a)}))?;
let mark = s.history.undo.len();
let mut q = p.clone();
for (key, kind) in [("track", "video"), ("audioTrack", "audio")] {
if is_new(key) {
q[key] = added.get(kind).and_then(|ids| ids.get(0)).cloned().unwrap_or(Value::Null);
}
}
let place = crate::commands::find(cmd).ok_or_else(|| EngineError::UnknownCommand(cmd.into()))?.run;
match place(s, &q) {
Ok(v) => {
// the new tracks and the clip are one undo step
if mark >= 1
&& s.history.undo.len() == mark + 1
&& let (Some((label, _)), Some((_, before))) = (s.history.undo.pop(), s.history.undo.pop())
{
s.history.undo.push((label, before));
}
Ok(v)
}
Err(e) => {
s.undo();
s.history.redo.pop();
Err(e)
}
}
}
fn delete_tracks(s: &mut Session, p: &Value) -> Result<Value> {
let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?;
let q = s.active_sequence().ok_or(EngineError::NoSequence)?;
@ -748,6 +796,18 @@ fn delete_tracks(s: &mut Session, p: &Value) -> Result<Value> {
}
doomed.extend(mine);
}
// caption tracks: unlike video/audio, a sequence may have none, so all of them can go
match p.get("captions") {
None | Some(Value::Null) => {}
Some(v) if v.as_str().is_some_and(|x| matches!(x, "empty" | "allEmpty" | "all-empty")) => {
doomed.extend(q.caption_tracks.iter().filter(|t| t.captions.is_empty()).map(|t| t.id));
}
Some(v @ (Value::String(_) | Value::Number(_))) => {
let id = crate::captions::track_param(s, &json!({"track": v})).ok_or_else(|| bad("sequence.deleteTracks", "unknown caption track"))?;
doomed.push(id);
}
Some(_) => return Err(bad("sequence.deleteTracks", "`captions` must be \"empty\", a name like \"C2\" or a track id")),
}
if doomed.is_empty() {
return Err(EngineError::Other("no tracks to delete".into()));
}
@ -755,7 +815,9 @@ fn delete_tracks(s: &mut Session, p: &Value) -> Result<Value> {
s.edit_sequence("Delete Tracks", |q, _, st| {
q.video_tracks.retain(|t| !doomed.contains(&t.id));
q.audio_tracks.retain(|t| !doomed.contains(&t.id));
q.caption_tracks.retain(|t| !doomed.contains(&t.id));
st.selection.retain(|c| q.find_item(*c).is_some());
st.caption_selection.retain(|c| q.find_caption(*c).is_some());
if let Some(tg) = st.targeting.get_mut(&seq_id) {
tg.targeted.retain(|t| !doomed.contains(t));
if tg.video_dest.is_some_and(|t| doomed.contains(&t)) {

68
crates/engine/src/sequence_tools_tests.rs
View File

@ -350,6 +350,42 @@ fn delete_tracks_empty_and_specific() {
assert_eq!(s.active_sequence().unwrap(), &q);
}
#[test]
fn delete_tracks_removes_caption_tracks() {
let mut s = demo();
let before = s.active_sequence().unwrap().clone();
let n0 = before.caption_tracks.len();
for _ in 0..3 {
s.execute("captions.newTrack", json!({"format": "Subtitle"})).unwrap();
}
// new tracks are inserted on top: C1 gets a caption, C2 and C3 stay empty
let cap = s.execute("captions.add", json!({"track": "C1", "text": "Hi", "seconds": 1.0})).unwrap()["caption"].as_u64().unwrap();
s.execute("captions.select", json!({"captions": [cap]})).unwrap();
let empty = s.active_sequence().unwrap().caption_tracks.iter().filter(|t| t.captions.is_empty()).count();
assert!(empty >= 2);
let r = s.execute("sequence.deleteTracks", json!({"captions": "empty"})).unwrap();
assert_eq!(r["deleted"], json!(empty));
let q = s.active_sequence().unwrap();
assert!(q.caption_tracks.iter().all(|t| !t.captions.is_empty()));
assert_eq!(q.video_tracks.len(), before.video_tracks.len(), "video tracks untouched");
assert!(s.execute("sequence.deleteTracks", json!({"captions": "empty"})).is_err(), "nothing left to delete");
// unknown names, ids and value types are errors, not panics
for bad in [json!("C99"), json!("C0"), json!("Cx"), json!(999_999), json!(true), json!(["C1"])] {
assert!(s.execute("sequence.deleteTracks", json!({"captions": bad})).is_err(), "{bad}");
}
// a specific track (with its captions); the last caption track may go too
let left = s.active_sequence().unwrap().caption_tracks.len();
for _ in 0..left {
s.execute("sequence.deleteTracks", json!({"captions": "C1"})).unwrap();
}
assert!(s.active_sequence().unwrap().caption_tracks.is_empty());
assert!(s.state.caption_selection.is_empty(), "selection of deleted captions cleared");
for _ in 0..left + 1 {
s.execute("edit.undo", json!({})).unwrap();
}
assert_eq!(s.active_sequence().unwrap().caption_tracks.len(), n0 + 3);
}
#[test]
fn split_points_on_the_sequence() {
let (mut s, bars) = fresh();
@ -539,3 +575,35 @@ fn go_to_in_out_without_marks_go_to_the_sequence_ends() {
s.execute("markers.goToIn", json!({})).unwrap();
assert_eq!(s.playhead(), s.active_sequence().unwrap().mark_in.unwrap());
}
/// #483: media dropped in the empty space past the last track goes on a new track (with
/// `"new"`), one undo step; a placement that fails leaves no new track behind.
#[test]
fn place_on_new_tracks_adds_them_in_one_undo_step() {
let (mut s, item) = fresh();
let before = s.active_sequence().unwrap().clone();
let (nv, na) = (before.video_tracks.len(), before.audio_tracks.len());
let undo = s.history.undo.len();
let r = s.execute("timeline.place", json!({"item": item, "track": "new", "audioTrack": "new", "time": 0})).unwrap();
let q = s.active_sequence().unwrap();
assert_eq!((q.video_tracks.len(), q.audio_tracks.len()), (nv + 1, na + 1));
let clips: Vec<ClipId> = r["clips"].as_array().unwrap().iter().map(|c| ClipId(c.as_u64().unwrap())).collect();
assert_eq!(clips.len(), 2);
assert_eq!(q.video_tracks[nv].items.iter().map(|i| i.id).collect::<Vec<_>>(), [clips[0]], "the picture is on the new top video track");
assert_eq!(q.audio_tracks[na].items.iter().map(|i| i.id).collect::<Vec<_>>(), [clips[1]], "the sound is on the new last audio track");
assert_eq!(s.history.undo.len(), undo + 1, "one undo step");
s.execute("edit.undo", json!({})).unwrap();
assert_eq!(*s.active_sequence().unwrap(), before);
// only the picture gets a new track; the sound follows the given track
s.execute("timeline.place", json!({"item": item, "track": "new", "audioTrack": "A1", "time": 0})).unwrap();
let q = s.active_sequence().unwrap();
assert_eq!((q.video_tracks.len(), q.audio_tracks.len()), (nv + 1, na));
assert_eq!((q.video_tracks[nv].items.len(), q.audio_tracks[0].items.len()), (1, 1));
s.execute("edit.undo", json!({})).unwrap();
// a placement that fails after the tracks were added: nothing changes
assert!(s.execute("timeline.place", json!({"item": item, "track": "new", "audioTrack": "A99", "time": 0})).is_err());
assert_eq!(*s.active_sequence().unwrap(), before);
assert_eq!(s.history.undo.len(), undo);
}

25
crates/engine/src/settings.rs
View File

@ -104,6 +104,9 @@ pub struct AppearancePrefs {
pub accessible_contrast: bool,
/// Highlight (accent) colour of selections, focus and primary buttons (`#rrggbb`).
pub highlight_color: String,
/// "UI Scale": `auto` (default: the scale the operating system reports) or a fixed
/// percentage of [`UI_SCALES`] (`100` = one point per physical pixel).
pub ui_scale: String,
}
impl Default for AppearancePrefs {
@ -115,11 +118,19 @@ impl Default for AppearancePrefs {
light_theme: "light".into(),
accessible_contrast: false,
highlight_color: DEFAULT_HIGHLIGHT.into(),
ui_scale: "auto".into(),
}
}
}
impl AppearancePrefs {
/// The fixed UI scale (physical pixels per point) "UI Scale" asks for, or `None` for `auto`
/// (and for a value outside [`UI_SCALES`]), which keeps the operating system's scale.
pub fn ui_scale_factor(&self) -> Option<f32> {
// `auto` doesn't parse; every other listed value is a positive whole percentage
UI_SCALES.iter().any(|(k, _)| *k == self.ui_scale).then(|| self.ui_scale.parse::<f32>().ok()).flatten().map(|pct| pct / 100.0)
}
/// The theme to show (`darkest` | `dark` | `light`): the light or dark theme of the mode. Auto
/// follows `system_light` (the operating system's appearance) and is dark without an answer.
pub fn shown_theme(&self, system_light: Option<bool>) -> &str {
@ -669,6 +680,17 @@ const BIN_OPEN: &[(&str, &str)] = &[("openInPlace", "Open in place"), ("openNewT
const PROJECT_OPEN: &[(&str, &str)] = &[("openNewTab", "Open new tab"), ("openNewWindow", "Open in new window")];
const THEMES: &[(&str, &str)] = &[("darkest", "Darkest"), ("dark", "Dark"), ("light", "Light")];
pub const APPEARANCE_MODES: &[(&str, &str)] = &[("auto", "Sync with system"), ("dark", "Dark"), ("light", "Light")];
pub const UI_SCALES: &[(&str, &str)] = &[
("auto", "Auto (system)"),
("75", "75%"),
("100", "100%"),
("125", "125%"),
("150", "150%"),
("175", "175%"),
("200", "200%"),
("250", "250%"),
("300", "300%"),
];
pub const DARK_THEMES: &[(&str, &str)] = &[("darkest", "Darkest"), ("dark", "Dark")];
pub const LIGHT_THEMES: &[(&str, &str)] = &[("light", "Light")];
const MIXDOWN: &[(&str, &str)] = &[("front", "Front Only"), ("frontRear", "Front + Rear"), ("frontLfe", "Front + LFE"), ("frontRearLfe", "Front + Rear + LFE")];
@ -795,6 +817,7 @@ static CATEGORIES: &[Category] = &[
f("appearance.colorTheme", "Color Theme", Kind::Choice(THEMES), true),
b("appearance.accessibleContrast", "Accessible color contrast", true),
f("appearance.highlightColor", "Highlight Color", Kind::Color, true),
f("appearance.uiScale", "UI Scale", Kind::Choice(UI_SCALES), true),
],
},
Category {
@ -1074,7 +1097,7 @@ static CATEGORIES: &[Category] = &[
f("timeline.multichannelMonoMediaTracks", "Multichannel Mono Media", Kind::Choice(TRACKS), false),
],
),
b("timeline.focusTimelineOnEdit", "Set focus on the Timeline when performing Insert/Overwrite edits", true),
b("timeline.focusTimelineOnEdit", "Set focus on the Timeline when performing Insert/Overwrite edits", false),
b("timeline.snapPlayhead", "Snap playhead in Timeline when Snap is enabled", true),
b("timeline.returnToBeginning", "At playback end, return to beginning when restarting playback", true),
b("timeline.outOfSyncUnlinked", "Display out of sync indicators for unlinked clips", false),

12
crates/engine/src/settings_tests.rs
View File

@ -251,6 +251,18 @@ fn schema_command_describes_every_category() {
assert_eq!(auto["value"], "pageScroll");
assert_eq!(auto["wired"], true);
assert!(auto["kind"]["choices"].as_array().unwrap().iter().any(|c| c["value"] == "smoothScroll"));
// This option is stored, but Insert/Overwrite does not yet implement focus transfer.
let focus = tl["fields"].as_array().unwrap().iter().find(|f| f["key"] == "timeline.focusTimelineOnEdit").unwrap();
assert_eq!(focus["kind"], json!({"type": "bool"}));
assert_eq!(focus["wired"], false);
assert_eq!(focus["value"], false);
set(&mut s, "timeline.focusTimelineOnEdit", json!(true));
let updated = s.execute("prefs.schema", json!({"category": "timeline"})).unwrap();
let focus = updated["categories"][0]["fields"].as_array().unwrap().iter().find(|f| f["key"] == "timeline.focusTimelineOnEdit").unwrap();
assert_eq!(focus["value"], true);
assert_eq!(focus["wired"], false);
let one = s.execute("prefs.schema", json!({"category": "trim"})).unwrap();
assert_eq!(one["categories"].as_array().unwrap().len(), 1);
assert!(s.execute("prefs.schema", json!({"category": "nope"})).is_err());

1
crates/engine/src/shortcuts.rs
View File

@ -426,6 +426,7 @@ fn category_of(id: &str, menu: &[&str]) -> String {
"view" => "View".into(),
"app" | "help" | "mode" => "Application".into(),
"captions" => "Captions".into(),
"tts" => "Text to Speech".into(),
_ => menu.first().map(|s| s.to_string()).unwrap_or_else(|| {
let mut c = prefix.chars();
c.next().map(|f| f.to_uppercase().collect::<String>() + c.as_str()).unwrap_or_default()

18
crates/engine/src/trim_tests.rs
View File

@ -236,3 +236,21 @@ fn extend_to_playhead_moves_each_unequal_out_point_to_the_playhead() {
assert_eq!(q.find_item(a).unwrap().1.end(), ph);
assert_eq!(q.find_item(b).unwrap().1.end(), ph);
}
/// #653: the delta the Timeline previews while an edge is dragged is the one `timeline.trim`
/// applies on release: dragging the last clip's Out edge far past the end of its media stops at
/// the media's end instead of following the pointer.
#[test]
fn trim_delta_stops_at_the_end_of_the_media() {
let mut s = demo();
let last = *v1(&s).last().unwrap();
let clip = ClipId(last.0);
let far = Tick(3600 * filmcraft_time::TICKS_PER_SECOND);
let shown = commands::trim_delta(&s, clip, filmcraft_edit::Edge::Out, filmcraft_edit::TrimMode::Regular, far).unwrap();
assert!(Tick::ZERO < shown && shown < far, "clamped to the media left after the clip: {shown:?}");
let r = s.execute("timeline.trim", json!({"clip": last.0, "edge": "out", "mode": "regular", "delta": far.0})).unwrap();
assert_eq!(r["delta"], json!(shown.0), "the preview and the trim agree");
let small = s.sequence_rate().frame_duration();
let back = commands::trim_delta(&s, clip, filmcraft_edit::Edge::Out, filmcraft_edit::TrimMode::Regular, -small).unwrap();
assert_eq!(back, -small, "a trim within the media is not clamped");
}

13
crates/engine/src/vfx_tests.rs
View File

@ -115,3 +115,16 @@ fn new_effect_params_are_settable_and_keyframable() {
let img = s.render_program(0.125).unwrap();
assert!(img.px.iter().all(|v| v.is_finite()));
}
#[test]
fn ultra_key_setting_writes_aggressive_parameters() {
let mut s = demo();
let clip = first_v1_clip(&s);
s.execute("effects.apply", json!({"clips": [clip], "effect": "ultra_key"})).unwrap();
s.execute("effects.setParam", json!({"clip": clip, "effect": "ultra_key", "param": "setting", "value": 2})).unwrap();
let e = s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(clip)).unwrap().1.effect("ultra_key").unwrap();
assert_eq!(e.param("tolerance").unwrap().value.as_f64(), Some(90.0));
assert_eq!(e.param("pedestal").unwrap().value.as_f64(), Some(50.0));
assert_eq!(e.param("choke").unwrap().value.as_f64(), Some(10.0));
assert_eq!(e.param("contrast").unwrap().value.as_f64(), Some(10.0));
}

34
crates/engine/src/voiceover.rs
View File

@ -386,6 +386,13 @@ fn sync(s: &mut Session, p: &Value) -> Result<Value> {
/// Directory recordings are written to.
fn record_dir(s: &Session, p: &Value) -> String {
media_dir(s, p, "Voice-over Recordings")
}
/// Directory generated media (recordings, narrations) is written to: `dir`, else Project
/// Settings ▸ Scratch Disks ▸ Captured, else next to the project, else `fallback` in the data or
/// temporary directory.
pub(crate) fn media_dir(s: &Session, p: &Value, fallback: &str) -> String {
if let Some(d) = str_p(p, "dir") {
return d.to_string();
}
@ -396,7 +403,21 @@ fn record_dir(s: &Session, p: &Value) -> String {
return d.to_string_lossy().into_owned();
}
let base = s.prefs_path.as_ref().and_then(|p| p.parent()).map(|d| d.to_path_buf()).unwrap_or_else(|| crate::temp_dir().join("FilmCraft"));
base.join("Voice-over Recordings").to_string_lossy().into_owned()
base.join(fallback).to_string_lossy().into_owned()
}
/// `<dir>/<base> <n>.wav` with the first `n` used by neither a project item nor an existing file;
/// an error once `n` passes 99 999 (bounded: `file_size` is a filesystem call per try).
pub(crate) fn unique_wav_path(s: &Session, dir: &str, base: &str) -> Result<String> {
let names: std::collections::HashSet<String> = s.project.items.values().map(|i| i.name.clone()).collect();
for k in 1..=99_999u32 {
let file = format!("{base} {k}.wav");
let path = std::path::Path::new(dir).join(&file).to_string_lossy().into_owned();
if !names.contains(&file) && s.services.file_size(&path).is_err() {
return Ok(path);
}
}
Err(EngineError::Other(format!("{dir}: no free file name for \"{base} <n>.wav\"")))
}
fn stop(s: &mut Session, p: &Value) -> Result<Value> {
@ -449,16 +470,7 @@ fn stop(s: &mut Session, p: &Value) -> Result<Value> {
// write the file
let dir = record_dir(s, p);
let base = s.prefs.voice_over.name.clone();
let names: std::collections::HashSet<String> = s.project.items.values().map(|i| i.name.clone()).collect();
let mut k = 1;
let path = loop {
let file = format!("{base} {k}.wav");
let path = std::path::Path::new(&dir).join(&file).to_string_lossy().into_owned();
if !names.contains(&file) && s.services.file_size(&path).is_err() {
break path;
}
k += 1;
};
let path = unique_wav_path(s, &dir, &base)?;
if !cfg!(target_arch = "wasm32") {
std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?;
}

5
crates/export/src/audio_out.rs
View File

@ -145,6 +145,11 @@ impl AudioOut {
Some(buf)
}
/// Output samples still to come (per channel).
pub fn remaining(&self) -> u64 {
u64::try_from(self.end.saturating_sub(self.out)).unwrap_or(0)
}
/// Everything still to come (to the end of the range).
pub fn rest(&mut self, sources: &dyn SourceProvider) -> Option<Vec<Vec<f32>>> {
self.pull(self.end, sources)

159
crates/export/src/lib.rs
View File

@ -274,11 +274,19 @@ pub struct ExportSettings {
/// stream needs is raised.
pub h264_level: Option<u8>,
pub bitrate_mode: BitrateMode,
/// Constant rate factor of [`BitrateMode::Crf`], 0 (best) to 51 (smallest); 23 by default,
/// 18 is visually near-lossless.
#[serde(default = "default_crf")]
pub crf: f32,
/// May H.264 be encoded by the system's hardware encoder (VideoToolbox on macOS, NVENC on
/// Windows)? Off unless asked for: hardware output depends on the machine, so it is not
/// byte-reproducible like the built-in encoder's (`determinism_tests`).
#[serde(default)]
pub hardware_encoding: HardwareEncoding,
/// Composite the exported frames on the GPU instead of the CPU reference renderer (Export ▸
/// GPU rendering: Auto / Off, Off by default). Off = the CPU reference renderer exactly.
#[serde(default)]
pub gpu_rendering: GpuRendering,
/// VBR maximum bitrate (None = 1.5 × target).
pub max_bitrate_kbps: Option<u32>,
/// Adaptive bitrate (the Match Source presets): bits per pixel per frame; replaces
@ -467,7 +475,9 @@ impl Default for ExportSettings {
h264_profile: H264Profile::High,
h264_level: None,
bitrate_mode: BitrateMode::default(),
crf: DEFAULT_CRF,
hardware_encoding: HardwareEncoding::default(),
gpu_rendering: GpuRendering::Off,
max_bitrate_kbps: None,
adaptive_bitrate: None,
keyframe_distance: None,
@ -485,6 +495,13 @@ impl Default for ExportSettings {
}
}
/// [`ExportSettings::crf`] of new settings and of settings saved before it existed.
pub const DEFAULT_CRF: f32 = 23.0;
fn default_crf() -> f32 {
DEFAULT_CRF
}
impl ExportSettings {
/// Reject settings the encoders cannot honour.
pub fn validate(&self) -> Result<()> {
@ -519,6 +536,14 @@ impl ExportSettings {
if self.format == Format::Hevc && self.bitrate_mode == BitrateMode::Vbr2Pass {
return Err(ExportError::Unsupported("H.265 export has no two-pass mode: choose CBR or VBR, 1 pass".into()));
}
if self.bitrate_mode == BitrateMode::Crf {
if self.format == Format::Hevc {
return Err(ExportError::Unsupported("H.265 export has no CRF mode: choose CBR or VBR, 1 pass".into()));
}
if !self.crf.is_finite() || !(0.0..=51.0).contains(&self.crf) {
return Err(ExportError::Unsupported("CRF must be between 0 and 51".into()));
}
}
Ok(())
}
}
@ -758,6 +783,99 @@ pub fn register_encoder(f: EncoderFactory) {
pub fn encoder_registered(f: EncoderFactory) -> bool {
video_factories().read().unwrap_or_else(|e| e.into_inner()).iter().any(|x| std::ptr::fn_addr_eq(*x, f))
}
/// Renders one export frame on something other than the CPU reference renderer — today the GPU
/// compositor (`filmcraft-gpu`), registered by the app or CLI at startup. `render` returns the
/// frame exactly as [`filmcraft_render::render_sequence`] would (same size, premultiplied
/// linear-light RGBA), or `None` to let the CPU render this frame (no adapter, a plan the GPU
/// cannot draw, any internal error): the CPU renderer is the reference and the fallback.
pub trait FrameRenderer: Send {
fn render(
&mut self,
project: &Project,
seq: ItemId,
t: Tick,
opts: filmcraft_render::RenderOptions,
sources: &dyn SourceProvider,
) -> Option<filmcraft_render::Image>;
}
/// Builds the frame renderer for one export, or `None` when this host cannot (no adapter).
pub type FrameRendererFactory = fn() -> Option<Box<dyn FrameRenderer>>;
fn frame_renderer_factories() -> &'static RwLock<Vec<FrameRendererFactory>> {
static F: OnceLock<RwLock<Vec<FrameRendererFactory>>> = OnceLock::new();
F.get_or_init(|| RwLock::new(Vec::new()))
}
/// Register a frame-renderer factory (tried before those registered earlier). Registering the
/// same factory twice is harmless.
pub fn register_frame_renderer(f: FrameRendererFactory) {
let mut g = frame_renderer_factories().write().unwrap_or_else(|e| e.into_inner());
if !g.iter().any(|x| std::ptr::fn_addr_eq(*x, f)) {
g.insert(0, f);
}
}
/// Whether `f` is among the registered frame-renderer factories (startup diagnostics, tests).
pub fn frame_renderer_registered(f: FrameRendererFactory) -> bool {
frame_renderer_factories().read().unwrap_or_else(|e| e.into_inner()).iter().any(|x| std::ptr::fn_addr_eq(*x, f))
}
/// Clear the registered frame-renderer factories. Test-only: registration is a process-wide
/// startup step and tests that register their own must not see each other's.
#[cfg(test)]
pub(crate) fn reset_frame_renderers_for_tests() {
frame_renderer_factories().write().unwrap_or_else(|e| e.into_inner()).clear();
GPU_FRAMES.store(0, Ordering::Relaxed);
GPU_FALLBACKS.store(0, Ordering::Relaxed);
}
/// Pick the first registered factory that can build a renderer for this host.
pub(crate) fn build_frame_renderer() -> Option<Box<dyn FrameRenderer>> {
let g = frame_renderer_factories().read().unwrap_or_else(|e| e.into_inner());
g.iter().filter_map(|f| f()).next()
}
static GPU_FRAMES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static GPU_FALLBACKS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static LOCK_WAIT_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
/// A frame was composited by the registered GPU renderer.
pub fn note_gpu_frame() {
GPU_FRAMES.fetch_add(1, Ordering::Relaxed);
}
/// The GPU renderer declined a frame (the CPU reference renderer took it).
pub fn note_gpu_fallback() {
GPU_FALLBACKS.fetch_add(1, Ordering::Relaxed);
}
/// Time an export worker waited for a free GPU renderer (process-wide, for the bench).
pub(crate) fn note_lock_wait(d: std::time::Duration) {
LOCK_WAIT_NS.fetch_add(u64::try_from(d.as_nanos()).unwrap_or(u64::MAX), Ordering::Relaxed);
}
/// Total time export workers waited for a GPU renderer since [`reset_export_lock_wait`].
pub fn export_lock_wait() -> std::time::Duration {
std::time::Duration::from_nanos(LOCK_WAIT_NS.load(Ordering::Relaxed))
}
/// Start [`export_lock_wait`] again from zero.
pub fn reset_export_lock_wait() {
LOCK_WAIT_NS.store(0, Ordering::Relaxed);
}
/// GPU-vs-CPU frame counters of the export renderer (per-process, like the hardware ones).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GpuRenderStats {
pub frames: u64,
pub fallbacks: u64,
}
pub fn gpu_render_stats() -> GpuRenderStats {
GpuRenderStats { frames: GPU_FRAMES.load(Ordering::Relaxed), fallbacks: GPU_FALLBACKS.load(Ordering::Relaxed) }
}
pub fn register_audio_encoder(f: AudioEncoderFactory) {
audio_factories().write().unwrap_or_else(|e| e.into_inner()).insert(0, f);
}
@ -1344,6 +1462,8 @@ fn h264_factory(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSetti
let max = s.max_bitrate_kbps.filter(|m| *m >= kbps).unwrap_or_else(|| (u64::from(kbps) * 3 / 2).min(u64::from(u32::MAX)) as u32);
cfg.rate = match s.bitrate_mode {
BitrateMode::Cbr => filmcraft_h264enc::RateControl::Cbr { kbps },
// the encoder clamps to 0..=51; a NaN from a hand-edited preset becomes the default
BitrateMode::Crf => filmcraft_h264enc::RateControl::Crf(if s.crf.is_finite() { s.crf } else { DEFAULT_CRF }),
_ => filmcraft_h264enc::RateControl::Vbr { target_kbps: kbps, max_kbps: max },
};
cfg.pass = match &s.h264_pass {
@ -1474,20 +1594,41 @@ pub fn export(project: &Arc<Project>, seq: ItemId, settings: &ExportSettings, so
let batch = rayon::current_num_threads().clamp(2, 16) as i64;
let (bytes, nframes) = match settings.format {
Format::Wav | Format::Aiff => {
let mut a = audio_out::AudioOut::new(project.clone(), seq, settings, range)?;
let (ch, sr, bits) = (a.channels as u16, a.sr, settings.audio.bits);
let container = if settings.format == Format::Wav { pcm::PcmContainer::Wav } else { pcm::PcmContainer::Aiff };
let frames = a.remaining();
// the sizes are known up front: refuse an AIFF that cannot hold them before measuring
let head = pcm::header(container, ch, sr, bits, frames).map_err(ExportError::Unsupported)?;
// one second per chunk: memory stays flat however long the range is
let chunk = i64::from(sr.max(1));
if !settings.part_of_batch {
progress.total.store(1, Ordering::Relaxed);
progress.total.store(frames.div_ceil(chunk as u64).max(1), Ordering::Relaxed);
progress.set_status(format!("Exporting audio ({})", settings.format.label()));
}
let mut a = audio_out::AudioOut::new(project.clone(), seq, settings, range)?;
a.measure(settings, sources, &cancelled)?;
*progress.loudness.lock().unwrap_or_else(|e| e.into_inner()) = a.loudness;
let planar = a.rest(sources).unwrap_or_else(|| vec![Vec::new(); a.channels]);
let inter = audio_out::interleave(&planar);
let (ch, sr, bits) = (a.channels as u16, a.sr, settings.audio.bits);
let data = if settings.format == Format::Wav { pcm::write_wav(&inter, ch, sr, bits) } else { pcm::write_aiff(&inter, ch, sr, bits) };
let n = write_output(settings, &settings.path, data)?;
progress.done.store(1, Ordering::Relaxed);
(n, planar.first().map_or(0, Vec::len) as u64)
let mut out = Out::create(settings)?;
let io = |e: std::io::Error| ExportError::Io(e.to_string());
out.write_all(&head).map_err(io)?;
let mut buf = Vec::new();
while let Some(planar) = a.pull(a.out.saturating_add(chunk), sources) {
if cancelled() {
return Err(ExportError::Cancelled);
}
buf.clear();
pcm::encode(container, &audio_out::interleave(&planar), bits, &mut buf);
out.write_all(&buf).map_err(io)?;
if !settings.part_of_batch {
progress.done.fetch_add(1, Ordering::Relaxed);
}
}
out.write_all(pcm::pad(frames, ch, bits)).map_err(io)?;
let n = out.finish(settings)?;
if settings.part_of_batch {
progress.done.store(1, Ordering::Relaxed);
}
(n, frames)
}
Format::PngSequence | Format::TiffSequence | Format::BmpSequence | Format::Gif => {
let pipe = pipeline::Pipeline::new(project.clone(), seq, settings, false)?;

286
crates/export/src/pcm.rs
View File

@ -1,5 +1,16 @@
//! Audio-only files: WAV (RIFF, little-endian PCM) and AIFF (big-endian PCM), 16 or 24 bit, plus
//! image-sequence file naming.
//! Audio-only files: WAV (RIFF, little-endian PCM; RF64 past 4 GiB) and AIFF (big-endian PCM),
//! 16 or 24 bit, plus image-sequence file naming.
//!
//! The header is written first from the known length ([`header`]), then the samples in chunks
//! ([`encode`]) and the pad byte ([`pad`]), so an export streams to disk instead of building the
//! file in memory.
/// The container of an audio-only export.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PcmContainer {
Wav,
Aiff,
}
/// Quantise one sample to a signed integer of `bits` (16 or 24).
fn quantise(s: f32, bits: u16) -> i32 {
@ -7,41 +18,128 @@ fn quantise(s: f32, bits: u16) -> i32 {
(s.clamp(-1.0, 1.0) * max).round() as i32
}
/// The sample size written for a requested `bits`: 24, else 16.
fn sample_bits(bits: u16) -> u16 {
if bits >= 24 { 24 } else { 16 }
}
/// Bytes of sample data for `frames` frames, or None when that does not fit 64 bits.
fn data_len(frames: u64, channels: u16, bits: u16) -> Option<u64> {
frames.checked_mul(u64::from(channels))?.checked_mul(u64::from(sample_bits(bits) / 8))
}
/// The pad byte after an odd-sized data chunk (RIFF and AIFF chunks are word-aligned).
pub fn pad(frames: u64, channels: u16, bits: u16) -> &'static [u8] {
if data_len(frames, channels, bits).is_some_and(|n| n & 1 == 1) { &[0] } else { &[] }
}
/// Everything before the samples of a file of `frames` frames. A WAV whose RIFF size would not
/// fit 32 bits (4 GiB: 24-bit 5.1 at 96 kHz after about 41 minutes) is written as RF64 (EBU Tech
/// 3306), with the 64-bit sizes in a `ds64` chunk. AIFF has no 64-bit form, so a longer AIFF is
/// an error naming WAV instead.
pub fn header(container: PcmContainer, channels: u16, sample_rate: u32, bits: u16, frames: u64) -> Result<Vec<u8>, String> {
let too_long = || "the audio is too long to write".to_string();
let bits = sample_bits(bits);
let bps = bits / 8;
let data = data_len(frames, channels, bits).ok_or_else(too_long)?;
let block_align = channels.checked_mul(bps).ok_or("too many audio channels")?;
match container {
PcmContainer::Wav => {
let byte_rate = u32::try_from(u64::from(sample_rate) * u64::from(block_align)).map_err(|_| "the audio sample rate is too high".to_string())?;
let ext = channels > 2;
let fmt_len: u32 = if ext { 40 } else { 16 };
let padded = data.checked_add(data & 1).ok_or_else(too_long)?;
// "WAVE" + fmt chunk + data chunk header + samples
let riff = (4 + 8 + u64::from(fmt_len) + 8).checked_add(padded).ok_or_else(too_long)?;
let rf64 = riff > u64::from(u32::MAX);
let mut v = Vec::with_capacity(80);
if rf64 {
const DS64: u64 = 8 + 28;
v.extend_from_slice(b"RF64");
v.extend_from_slice(&u32::MAX.to_le_bytes());
v.extend_from_slice(b"WAVEds64");
v.extend_from_slice(&28u32.to_le_bytes());
v.extend_from_slice(&riff.checked_add(DS64).ok_or_else(too_long)?.to_le_bytes());
v.extend_from_slice(&data.to_le_bytes());
v.extend_from_slice(&frames.to_le_bytes());
v.extend_from_slice(&0u32.to_le_bytes()); // no table entries
} else {
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(riff as u32).to_le_bytes());
v.extend_from_slice(b"WAVE");
}
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&fmt_len.to_le_bytes());
v.extend_from_slice(&(if ext { 0xFFFEu16 } else { 1 }).to_le_bytes());
v.extend_from_slice(&channels.to_le_bytes());
v.extend_from_slice(&sample_rate.to_le_bytes());
v.extend_from_slice(&byte_rate.to_le_bytes());
v.extend_from_slice(&block_align.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
if ext {
// cbSize, valid bits, channel mask, KSDATAFORMAT_SUBTYPE_PCM
v.extend_from_slice(&22u16.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
let mask: u32 = if channels == 6 { 0x3F } else { 0 };
v.extend_from_slice(&mask.to_le_bytes());
v.extend_from_slice(&[0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71]);
}
v.extend_from_slice(b"data");
v.extend_from_slice(&(if rf64 { u32::MAX } else { data as u32 }).to_le_bytes());
Ok(v)
}
PcmContainer::Aiff => {
let aiff_too_long = || "AIFF holds at most 4 GB of audio: export WAV, which has no such limit".to_string();
let frames32 = u32::try_from(frames).map_err(|_| aiff_too_long())?;
// SSND: offset + block size + samples
let ssnd = u32::try_from(data.checked_add(8).ok_or_else(aiff_too_long)?).map_err(|_| aiff_too_long())?;
let form = (4u32 + (8 + 18)).checked_add(8).and_then(|n| n.checked_add(ssnd)).and_then(|n| n.checked_add(ssnd & 1)).ok_or_else(aiff_too_long)?;
let mut v = Vec::with_capacity(54);
v.extend_from_slice(b"FORM");
v.extend_from_slice(&form.to_be_bytes());
v.extend_from_slice(b"AIFFCOMM");
v.extend_from_slice(&18u32.to_be_bytes());
v.extend_from_slice(&channels.to_be_bytes());
v.extend_from_slice(&frames32.to_be_bytes());
v.extend_from_slice(&bits.to_be_bytes());
v.extend_from_slice(&extended(sample_rate));
v.extend_from_slice(b"SSND");
v.extend_from_slice(&ssnd.to_be_bytes());
v.extend_from_slice(&0u32.to_be_bytes()); // offset
v.extend_from_slice(&0u32.to_be_bytes()); // block size
Ok(v)
}
}
}
/// Append interleaved samples as `bits`-bit PCM (little-endian for WAV, big-endian for AIFF).
pub fn encode(container: PcmContainer, interleaved: &[f32], bits: u16, out: &mut Vec<u8>) {
let bits = sample_bits(bits);
let bps = usize::from(bits / 8);
out.reserve(interleaved.len().saturating_mul(bps));
for &s in interleaved {
let q = quantise(s, bits);
match container {
PcmContainer::Wav => out.extend_from_slice(q.to_le_bytes().get(..bps).unwrap_or_default()),
PcmContainer::Aiff => out.extend_from_slice(q.to_be_bytes().get(4 - bps..).unwrap_or_default()),
}
}
}
/// A whole file in memory (short audio; exports stream with [`header`] and [`encode`]).
fn write_whole(container: PcmContainer, interleaved: &[f32], channels: u16, sample_rate: u32, bits: u16) -> Result<Vec<u8>, String> {
let frames = (interleaved.len() / usize::from(channels.max(1))) as u64;
let mut v = header(container, channels, sample_rate, bits, frames)?;
encode(container, interleaved, bits, &mut v);
v.extend_from_slice(pad(frames, channels, bits));
Ok(v)
}
/// A RIFF/WAVE file of interleaved samples. More than two channels are written as
/// `WAVE_FORMAT_EXTENSIBLE` with the speaker mask of the layout (6 channels: L, R, C, LFE, Ls, Rs
/// = `0x3F`).
pub fn write_wav(interleaved: &[f32], channels: u16, sample_rate: u32, bits: u16) -> Vec<u8> {
let bits: u16 = if bits >= 24 { 24 } else { 16 };
let bps = (bits / 8) as u32;
let data_len = interleaved.len() as u32 * bps;
let ext = channels > 2;
let fmt_len: u32 = if ext { 40 } else { 16 };
let mut v = Vec::with_capacity(28 + fmt_len as usize + data_len as usize);
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(20 + fmt_len + data_len).to_le_bytes());
v.extend_from_slice(b"WAVEfmt ");
v.extend_from_slice(&fmt_len.to_le_bytes());
v.extend_from_slice(&(if ext { 0xFFFEu16 } else { 1 }).to_le_bytes());
v.extend_from_slice(&channels.to_le_bytes());
v.extend_from_slice(&sample_rate.to_le_bytes());
v.extend_from_slice(&(sample_rate * channels as u32 * bps).to_le_bytes());
v.extend_from_slice(&(channels * bps as u16).to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
if ext {
// cbSize, valid bits, channel mask, KSDATAFORMAT_SUBTYPE_PCM
v.extend_from_slice(&22u16.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
let mask: u32 = if channels == 6 { 0x3F } else { 0 };
v.extend_from_slice(&mask.to_le_bytes());
v.extend_from_slice(&[0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71]);
}
v.extend_from_slice(b"data");
v.extend_from_slice(&data_len.to_le_bytes());
for &s in interleaved {
let q = quantise(s, bits).to_le_bytes();
v.extend_from_slice(&q[..bps as usize]);
}
v
pub fn write_wav(interleaved: &[f32], channels: u16, sample_rate: u32, bits: u16) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Wav, interleaved, channels, sample_rate, bits)
}
/// IEEE 754 80-bit extended encoding of a sample rate (AIFF `COMM`).
@ -59,34 +157,8 @@ fn extended(rate: u32) -> [u8; 10] {
}
/// An AIFF file (`FORM`/`AIFF`, `COMM` + `SSND`) of interleaved samples.
pub fn write_aiff(interleaved: &[f32], channels: u16, sample_rate: u32, bits: u16) -> Vec<u8> {
let bits: u16 = if bits >= 24 { 24 } else { 16 };
let bps = (bits / 8) as usize;
let frames = interleaved.len() / channels.max(1) as usize;
let data_len = (interleaved.len() * bps) as u32;
let ssnd_len = 8 + data_len;
let form_len = 4 + (8 + 18) + (8 + ssnd_len) + (ssnd_len & 1);
let mut v = Vec::with_capacity(form_len as usize + 8);
v.extend_from_slice(b"FORM");
v.extend_from_slice(&form_len.to_be_bytes());
v.extend_from_slice(b"AIFFCOMM");
v.extend_from_slice(&18u32.to_be_bytes());
v.extend_from_slice(&channels.to_be_bytes());
v.extend_from_slice(&(frames as u32).to_be_bytes());
v.extend_from_slice(&bits.to_be_bytes());
v.extend_from_slice(&extended(sample_rate));
v.extend_from_slice(b"SSND");
v.extend_from_slice(&ssnd_len.to_be_bytes());
v.extend_from_slice(&0u32.to_be_bytes()); // offset
v.extend_from_slice(&0u32.to_be_bytes()); // block size
for &s in interleaved {
let q = quantise(s, bits).to_be_bytes();
v.extend_from_slice(&q[4 - bps..]);
}
if ssnd_len & 1 == 1 {
v.push(0);
}
v
pub fn write_aiff(interleaved: &[f32], channels: u16, sample_rate: u32, bits: u16) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Aiff, interleaved, channels, sample_rate, bits)
}
/// File name of frame `index` of an image sequence of `count` frames exported to `path`:
@ -126,12 +198,98 @@ mod tests {
#[test]
fn wav_and_aiff_layout() {
let s = [0.5f32, -0.5, 1.0, -1.0];
let w = write_wav(&s, 2, 48_000, 24);
let w = write_wav(&s, 2, 48_000, 24).unwrap();
assert_eq!(w.len(), 44 + 12);
assert_eq!(&w[34..36], &24u16.to_le_bytes());
let a = write_aiff(&s, 2, 48_000, 16);
let a = write_aiff(&s, 2, 48_000, 16).unwrap();
assert_eq!(&a[..4], b"FORM");
assert_eq!(u32::from_be_bytes(a[4..8].try_into().unwrap()) as usize, a.len() - 8);
assert_eq!(&a[a.len() - 2..], &(-32767i16).to_be_bytes());
}
fn le32(b: &[u8], at: usize) -> u32 {
u32::from_le_bytes(b[at..at + 4].try_into().unwrap())
}
fn le64(b: &[u8], at: usize) -> u64 {
u64::from_le_bytes(b[at..at + 8].try_into().unwrap())
}
/// 24-bit 5.1 at 96 kHz passes 4 GiB after about 41 minutes; the sizes overflowed `u32`
/// (a panic in debug builds, a corrupt header in release).
#[test]
fn wav_past_4_gib_is_rf64() {
let frames = 96_000u64 * 60 * 60; // an hour: 6.2 GB
let h = header(PcmContainer::Wav, 6, 96_000, 24, frames).unwrap();
let data = frames * 6 * 3;
assert_eq!(&h[..4], b"RF64");
assert_eq!(le32(&h, 4), u32::MAX);
assert_eq!(&h[8..16], b"WAVEds64");
assert_eq!(le32(&h, 16), 28);
// the 64-bit RIFF size covers everything after the first 8 bytes
assert_eq!(le64(&h, 20), h.len() as u64 - 8 + data);
assert_eq!((le64(&h, 28), le64(&h, 36), le32(&h, 44)), (data, frames, 0));
assert_eq!(&h[48..52], b"fmt ");
assert_eq!(&h[h.len() - 8..h.len() - 4], b"data");
assert_eq!(le32(&h, h.len() - 4), u32::MAX);
}
#[test]
fn wav_up_to_4_gib_stays_riff() {
// the longest stereo 16-bit RIFF: 36 bytes of header after the size field
let max = (u64::from(u32::MAX) - 36) / 4;
let h = header(PcmContainer::Wav, 2, 48_000, 16, max).unwrap();
assert_eq!(&h[..4], b"RIFF");
assert_eq!(u64::from(le32(&h, 4)), 36 + max * 4);
assert_eq!(u64::from(le32(&h, 40)), max * 4);
assert_eq!(&header(PcmContainer::Wav, 2, 48_000, 16, max + 1).unwrap()[..4], b"RF64");
}
#[test]
fn aiff_past_4_gib_is_refused_not_wrapped() {
let e = header(PcmContainer::Aiff, 2, 48_000, 24, 48_000 * 60 * 60 * 8).unwrap_err();
assert!(e.contains("export WAV"), "{e}");
assert!(header(PcmContainer::Aiff, 2, 48_000, 24, 48_000 * 60 * 60).is_ok());
}
#[test]
fn hostile_sizes_are_errors() {
for c in [PcmContainer::Wav, PcmContainer::Aiff] {
assert!(header(c, 2, 48_000, 24, u64::MAX).is_err());
assert!(header(c, u16::MAX, 48_000, 24, 1).is_err());
}
assert!(header(PcmContainer::Wav, 6, u32::MAX, 24, 1).is_err());
}
#[test]
fn odd_data_chunks_are_padded() {
// one 24-bit mono frame: 3 bytes of data, one pad byte, counted in the RIFF size
let w = write_wav(&[0.25], 1, 48_000, 24).unwrap();
assert_eq!((w.len(), le32(&w, 4) as usize, le32(&w, 40)), (48, 40, 3));
let a = write_aiff(&[0.25], 1, 48_000, 24).unwrap();
assert_eq!(u32::from_be_bytes(a[4..8].try_into().unwrap()) as usize, a.len() - 8);
assert!(pad(2, 1, 24).is_empty());
}
/// A reader takes the RF64 sizes: ffprobe reports the duration from `ds64` (skipped without it).
#[test]
fn ffprobe_reads_rf64_duration() {
let dir = std::env::temp_dir().join(format!("filmcraft-rf64-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("long.wav");
let frames = 48_000u64 * 60 * 60 * 5; // 5 hours of 24-bit stereo: 5.2 GB
let mut f = header(PcmContainer::Wav, 2, 48_000, 24, frames).unwrap();
encode(PcmContainer::Wav, &[0.0; 9_600], 24, &mut f);
std::fs::write(&path, &f).unwrap();
let out = std::process::Command::new("ffprobe")
.args(["-v", "error", "-show_entries", "format=format_name,duration", "-of", "default=nw=1"])
.arg(&path)
.output();
let _ = std::fs::remove_dir_all(&dir);
let Ok(out) = out else { return };
let text = String::from_utf8_lossy(&out.stdout);
assert!(text.contains("format_name=wav"), "{text}");
let secs: f64 = text.lines().find_map(|l| l.strip_prefix("duration=")).and_then(|d| d.parse().ok()).unwrap_or(0.0);
assert!((secs - 18_000.0).abs() < 1.0, "{text}");
}
}

104
crates/export/src/pipeline.rs
View File

@ -12,7 +12,7 @@ use filmcraft_time::FrameRate;
use rayon::prelude::*;
use crate::settings::{ExportEffects, Placement, Scaling, TextOverlay};
use crate::{ExportError, ExportSettings, Format, Result};
use crate::{ExportError, ExportSettings, Format, FrameRenderer, GpuRendering, Result, build_frame_renderer, note_gpu_fallback, note_gpu_frame};
/// Where the rendered picture lands in the output frame.
#[derive(Clone, Copy, Debug, PartialEq)]
@ -49,9 +49,68 @@ impl Geometry {
}
}
/// GPU renderers shared by the export workers: a worker takes a free one for a frame and puts it
/// back (a single renderer behind a mutex made the workers queue, GPU export ~20 % slower than CPU).
struct RendererPool {
renderers: std::sync::Mutex<Vec<Box<dyn FrameRenderer>>>,
available: std::sync::Condvar,
/// A renderer panicked: the pool is retired and the rest of the export renders on the CPU.
failed: std::sync::atomic::AtomicBool,
}
impl RendererPool {
/// `None` for an empty list (a pool nobody could take from).
fn new(renderers: Vec<Box<dyn FrameRenderer>>) -> Option<Self> {
(!renderers.is_empty()).then(|| Self {
renderers: std::sync::Mutex::new(renderers),
available: std::sync::Condvar::new(),
failed: std::sync::atomic::AtomicBool::new(false),
})
}
/// Run `f` with a free renderer, waiting for one if all are busy. `None` once a renderer has
/// panicked: the panicking renderer is dropped (its state can't be trusted) and so are the
/// others as they come back, so the caller falls back to the CPU for the rest of the export.
fn with<R>(&self, f: impl FnOnce(&mut dyn FrameRenderer) -> R) -> Option<R> {
use std::sync::atomic::Ordering;
let mut free = self.renderers.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let mut r = loop {
if self.failed.load(Ordering::SeqCst) {
return None;
}
match free.pop() {
Some(r) => break r,
None => free = self.available.wait(free).unwrap_or_else(std::sync::PoisonError::into_inner),
}
};
drop(free);
let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(&mut *r)));
let mut free = self.renderers.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if res.is_err() {
// (the panic hook has logged the panic itself)
self.failed.store(true, Ordering::SeqCst);
free.clear();
drop(free);
drop(r);
} else if self.failed.load(Ordering::SeqCst) {
drop(free);
drop(r);
} else {
free.push(r);
drop(free);
}
// waiters re-check `failed` (or take the renderer that came back)
self.available.notify_all();
res.ok()
}
}
/// The immutable part of an export: everything needed to produce one output frame.
pub(crate) struct Pipeline {
pub project: Arc<Project>,
/// The registered frame renderers (GPU compositors) when the setting allows and factories
/// provide them; pooled because `frame` runs on several export threads at once.
renderer: Option<RendererPool>,
pub seq: ItemId,
/// Output frame rate (frame `f` is at `rate.tick_of(f)`).
pub rate: FrameRate,
@ -112,7 +171,24 @@ impl Pipeline {
} else {
None
};
let renderer = if settings.gpu_rendering == GpuRendering::Off {
None
} else {
// Four renderers: as fast as one per worker (measured on 16 threads, where more only
// contend for memory bandwidth in the read-back), with a quarter of the GPU memory.
let count = rayon::current_num_threads().clamp(2, 4);
let mut list = Vec::with_capacity(count);
for _ in 0..count {
if let Some(r) = build_frame_renderer() {
list.push(r);
} else {
break;
}
}
RendererPool::new(list)
};
Ok(Pipeline {
renderer,
seq_rate: q.settings.frame_rate,
start_tc: q.start_timecode,
drop_frame: q.settings.drop_frame,
@ -136,7 +212,31 @@ impl Pipeline {
/// R'G'B' floats (3 per pixel).
pub fn frame(&self, f: i64, sources: &dyn SourceProvider) -> (Vec<u8>, Vec<f32>) {
let t = self.rate.tick_of(f);
let img = filmcraft_render::render_sequence(&self.project, self.seq, t, self.opts, sources);
// The registered GPU renderer goes first; `None` (no adapter, a plan the GPU cannot
// draw, any internal error) falls back to the CPU reference renderer below.
let gpu = match self.renderer.as_ref() {
Some(pool) => {
let asked = web_time::Instant::now();
pool.with(|r| {
crate::note_lock_wait(asked.elapsed());
r.render(&self.project, self.seq, t, self.opts, sources)
})
.flatten()
}
None => None,
};
let img = match gpu {
Some(img) => {
note_gpu_frame();
img
}
None => {
if self.renderer.is_some() {
note_gpu_fallback();
}
filmcraft_render::render_sequence(&self.project, self.seq, t, self.opts, sources)
}
};
let mut img = self.place(img);
self.overlays(&mut img, t);
let lim = &self.effects.video_limiter;

32
crates/export/src/settings.rs
View File

@ -75,6 +75,22 @@ pub enum HardwareEncoding {
Auto,
}
/// Whether the picture of an export may be composited on the GPU (`filmcraft-gpu`'s off-screen
/// compositor) instead of the CPU reference renderer. Auto = use the GPU when the app registered
/// a GPU frame renderer and the machine has an adapter; the CPU result is the fallback either
/// way. On an effects-heavy edit it exports 2.4× (1080p) to 3.2× (4K) faster, and a single plain
/// clip takes the same time (`docs/performance.md`). The GPU matches the CPU within the
/// compositor's parity tolerance, not bit for bit, so an export's bytes can depend on the
/// machine's GPU. Off = the CPU reference renderer, byte-reproducible everywhere; Off is the
/// default (opt-in per export) until GPU export has been measured on Windows and macOS.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum GpuRendering {
Auto,
#[default]
Off,
}
/// Bitrate encoding of bitrate-driven codecs (H.264).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@ -86,6 +102,9 @@ pub enum BitrateMode {
Vbr1Pass,
/// Variable bitrate, two passes (the picture is rendered and analysed first).
Vbr2Pass,
/// Constant quality: every frame is coded at [`ExportSettings::crf`] and the bitrate follows the
/// picture (H.264, built-in encoder only).
Crf,
}
impl BitrateMode {
@ -94,6 +113,7 @@ impl BitrateMode {
BitrateMode::Cbr => "CBR",
BitrateMode::Vbr1Pass => "VBR, 1 pass",
BitrateMode::Vbr2Pass => "VBR, 2 pass",
BitrateMode::Crf => "CRF (constant quality)",
}
}
}
@ -390,16 +410,7 @@ impl ExportSettings {
6.. => 6,
_ => 2,
};
Resolved {
width: w,
height: h,
rate,
sample_rate: self.audio.sample_rate.filter(|r| (8000..=192_000).contains(r)).unwrap_or(seq_sr),
channels,
target_kbps: target,
max_kbps: max,
keyint,
}
Resolved { width: w, height: h, rate, sample_rate: self.audio.sample_rate.unwrap_or(seq_sr), channels, target_kbps: target, max_kbps: max, keyint }
}
/// The audio codec actually used.
@ -504,6 +515,7 @@ impl ExportSettings {
);
v += &match self.bitrate_mode {
BitrateMode::Cbr => format!(", {}", mbps(r.target_kbps)),
BitrateMode::Crf => format!(" {}", self.crf),
_ => format!(", Target {}, Max {}", mbps(r.target_kbps), mbps(r.max_kbps)),
};
v += &format!(", keyframe every {} frames", r.keyint);

90
crates/export/src/settings_tests.rs
View File

@ -167,6 +167,50 @@ fn wav_and_aiff_audio_only() {
}
}
#[test]
fn wav_export_honours_explicit_sample_rates() {
let (project, sequence, sources) = matte([0.0, 0.0, 0.0, 1.0], 64, 36, Some(-6.0));
let dir = Scratch::new("wav-sample-rates");
for rate in [1, 4000, 7350, 8000, 48_000, 192_000, 192_001, 200_000, 352_800, 384_000] {
let settings = ExportSettings {
format: Format::Wav,
path: dir.path(&format!("{rate}.wav")),
audio: AudioSettings { sample_rate: Some(rate), ..Default::default() },
range: Some(TimeRange::new(Tick::ZERO, Tick(TICKS_PER_SECOND / 4))),
..Default::default()
};
settings.validate().unwrap();
let resolved = settings.resolve(64, 36, FrameRate::FPS_24, 48_000);
assert_eq!(resolved.sample_rate, rate, "requested {rate} Hz");
export(&project, sequence, &settings, &sources, &Progress::default()).unwrap();
let bytes = std::fs::read(&settings.path).unwrap();
assert_eq!(u32::from_le_bytes(bytes[24..28].try_into().unwrap()), rate);
assert_eq!(u32::from_le_bytes(bytes[40..44].try_into().unwrap()), rate / 4 * 4, "quarter-second stereo 16-bit PCM");
if let Some(probe) = ffprobe_json(&["-show_streams"], &settings.path) {
assert_eq!(probe["streams"][0]["sample_rate"], rate.to_string());
}
}
let settings = ExportSettings { format: Format::Wav, ..Default::default() };
assert_eq!(settings.resolve(64, 36, FrameRate::FPS_24, 44_100).sample_rate, 44_100);
}
#[test]
fn aac_export_rejects_unsupported_explicit_sample_rates() {
let (project, sequence, sources) = matte([0.0, 0.0, 0.0, 1.0], 64, 36, Some(-6.0));
let dir = Scratch::new("aac-sample-rates");
for rate in [4000, 192_001, 200_000, 384_000] {
let settings = ExportSettings {
path: dir.path(&format!("{rate}.mp4")),
audio: AudioSettings { sample_rate: Some(rate), ..Default::default() },
range: Some(TimeRange::new(Tick::ZERO, Tick(TICKS_PER_SECOND / 4))),
..Default::default()
};
let error = export(&project, sequence, &settings, &sources, &Progress::default()).unwrap_err();
assert!(error.to_string().contains("sample rate"), "{error}");
assert!(!Path::new(&settings.path).exists(), "an unsupported rate must not produce a substitute file");
}
}
#[test]
fn invalid_export_allocations_are_rejected_before_rendering() {
let (project, sequence, _) = matte([1.0, 0.0, 0.0, 1.0], 64, 36, None);
@ -324,6 +368,40 @@ fn two_pass_and_cbr_h264() {
}
}
#[test]
fn crf_h264() {
// settings saved before CRF existed read as the default; the range is checked
let old: ExportSettings = serde_json::from_value(serde_json::json!({"format": "h264", "bitrateMode": "vbr1Pass"})).unwrap();
assert_eq!(old.crf, DEFAULT_CRF);
let c: ExportSettings = serde_json::from_value(serde_json::json!({"bitrateMode": "crf", "crf": 18.0})).unwrap();
assert_eq!((c.bitrate_mode, c.crf), (BitrateMode::Crf, 18.0));
for bad in [-1.0, 51.5, f32::NAN, f32::INFINITY] {
assert!(ExportSettings { bitrate_mode: BitrateMode::Crf, crf: bad, ..Default::default() }.validate().is_err(), "{bad}");
}
// the value only matters in CRF mode
assert!(ExportSettings { crf: f32::NAN, ..Default::default() }.validate().is_ok());
let summary = c.summary(1920, 1080, FrameRate::FPS_25, 48_000, Tick(10 * TICKS_PER_SECOND));
assert!(summary.video.contains("CRF (constant quality) 18"), "{}", summary.video);
// one pass at constant quality: a lower factor spends more bits, and both decode to the picture
let (p, seq, m) = matte([0.2, 0.5, 0.8, 1.0], 320, 180, None);
let dir = Scratch::new("crf");
let mut sizes = Vec::new();
for crf in [10.0, 40.0] {
let path = dir.path(&format!("crf{crf}.mp4"));
let s = ExportSettings { format: Format::H264, path: path.clone(), bitrate_mode: BitrateMode::Crf, crf, include_audio: false, ..Default::default() };
let prog = Progress::default();
let r = export(&p, seq, &s, &m, &prog).unwrap();
assert_eq!((r.frames, prog.total.load(Ordering::Relaxed)), (24, 24));
let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into();
sizes.push(bytes.len());
let src = filmcraft_codecs::open_bytes("x.mp4", bytes).unwrap();
let f = src.video_frame(FrameRequest::full(Tick(TICKS_PER_SECOND / 2))).unwrap().to_rgba8();
assert!((f[2] as i32 - 204).abs() < 12 && (f[1] as i32 - 127).abs() < 12, "CRF {crf}: {:?}", &f[..4]);
}
assert!(sizes[0] > sizes[1], "CRF 10 {} bytes, CRF 40 {} bytes", sizes[0], sizes[1]);
}
#[test]
fn h264_profile_level_and_keyframes() {
let (p, seq, m) = matte([0.5, 0.5, 0.5, 1.0], 320, 180, None);
@ -516,6 +594,8 @@ fn h265_is_a_format_that_needs_a_registered_encoder() {
let e = ExportSettings { bitrate_mode: BitrateMode::Vbr2Pass, ..h.clone() }.validate().unwrap_err();
assert!(e.to_string().contains("two-pass"), "{e}");
assert!(ExportSettings { bitrate_mode: BitrateMode::Cbr, ..h.clone() }.validate().is_ok());
let e = ExportSettings { bitrate_mode: BitrateMode::Crf, ..h.clone() }.validate().unwrap_err();
assert!(e.to_string().contains("CRF"), "{e}");
// formats with a built-in encoder are always available; this one only when a registered probe says so
assert!(Format::ALL.iter().filter(|f| f.has_builtin_encoder()).all(|f| available(*f)));
@ -556,3 +636,13 @@ fn extreme_bitrates_do_not_overflow_encoder_setup() {
assert!(result.unwrap().unwrap().is_err(), "an unrepresentable encoder buffer rate must be rejected");
}
}
#[test]
fn gpu_rendering_is_off_by_default() {
assert_eq!(ExportSettings::default().gpu_rendering, crate::GpuRendering::Off);
// settings saved before the field existed get the default too
let s: ExportSettings = serde_json::from_value(serde_json::json!({"format": "h264"})).unwrap();
assert_eq!(s.gpu_rendering, crate::GpuRendering::Off);
let auto: ExportSettings = serde_json::from_value(serde_json::json!({"format": "h264", "gpuRendering": "auto"})).unwrap();
assert_eq!(auto.gpu_rendering, crate::GpuRendering::Auto);
}

132
crates/export/src/tests.rs
View File

@ -260,3 +260,135 @@ fn hdr_exports_signal_pq_and_hlg() {
eprintln!("{path}:\n{txt}");
}
}
/// Scenario runner for the GPU hook: everything in one test because the renderer registry is
/// process-global and parallel tests would see each other's factories.
#[test]
fn gpu_hook_scenarios() {
// 1. Declining renderer: byte-equal to Off, and an empty registry too.
crate::reset_frame_renderers_for_tests();
crate::register_frame_renderer(|| None);
let (p, seq, m) = project();
let mk = |path: &String, gpu: Option<bool>| {
let mut s = ExportSettings { format: Format::PngSequence, path: path.clone(), ..Default::default() };
s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(2)));
s.gpu_rendering = match gpu {
Some(true) | None => crate::GpuRendering::Auto,
Some(false) => crate::GpuRendering::Off,
};
export(&p, seq, &s, &m, &Progress::default()).unwrap()
};
mk(&tmp("off.png"), Some(false));
mk(&tmp("auto-declined.png"), Some(true));
assert_eq!(std::fs::read(tmp("off000.png")).unwrap(), std::fs::read(tmp("auto-declined000.png")).unwrap(), "declined GPU export must be byte-equal to Off");
crate::reset_frame_renderers_for_tests();
mk(&tmp("auto-empty.png"), Some(true));
assert_eq!(std::fs::read(tmp("off000.png")).unwrap(), std::fs::read(tmp("auto-empty000.png")).unwrap(), "empty registry must behave like Off");
// 2. Producing renderer: counters up and its picture reaches the export.
crate::reset_frame_renderers_for_tests();
crate::register_frame_renderer(|| Some(Box::new(FakeRenderer) as Box<dyn FrameRenderer>));
mk(&tmp("fake-gpu.png"), Some(true));
assert!(gpu_render_stats().frames >= 2, "gpu frames counted: {:?}", gpu_render_stats());
assert_eq!(gpu_render_stats().fallbacks, 0);
mk(&tmp("fake-off.png"), Some(false));
assert_eq!(gpu_render_stats().fallbacks, 0, "Off must not count as fallback");
assert_ne!(std::fs::read(tmp("fake-gpu000.png")).unwrap(), std::fs::read(tmp("fake-off000.png")).unwrap(), "the fake GPU picture must reach the export");
// 3. Hostile and degenerate frame sizes must not panic with a renderer attached.
crate::reset_frame_renderers_for_tests();
crate::register_frame_renderer(|| None);
for (w, h) in [(1, 1), (17, 13), (4096, 1)] {
let mut p = Project::new("x");
let g = GeneratorSource::new(Generator::ColorMatte { color: [0.5, 0.5, 0.5, 1.0] }, 320, 180, FrameRate::FPS_24, Tick(2 * TICKS_PER_SECOND));
let info = g.info().clone();
let red = p.add_item(
&info.name.clone(),
Label::Iris,
ItemKind::Media(MediaClip {
media: MediaRef::Generator(g.generator.clone()),
info,
interpret: Default::default(),
mark_in: None,
mark_out: None,
markers: vec![],
offline: false,
proxy: None,
identity: None,
}),
None,
);
let seq = p.new_sequence("s", SequenceSettings { width: w, height: h, frame_rate: FrameRate::FPS_24, ..Default::default() }, 1, 1, None);
let v = p.make_track_item(red, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(2)), FrameRate::FPS_24).unwrap();
p.sequence_mut(seq).unwrap().video_tracks[0].items.push(v);
let mut m = SourceMap::default();
m.0.insert(red, Arc::new(g));
let mut s = ExportSettings { format: Format::PngSequence, path: tmp("hostile.png"), ..Default::default() };
s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(1)));
s.gpu_rendering = crate::GpuRendering::Auto;
let r = export(&Arc::new(p), seq, &s, &m, &Progress::default());
if r.is_err() {
continue; // sizes the exporter rejects are fine too — as long as nothing panics
}
let bytes = std::fs::read(tmp("hostile000.png")).unwrap_or_default();
assert!(!bytes.is_empty(), "{w}x{h}: PNG written");
}
// 4. A renderer that panics is dropped, not put back: the export still completes, every
// frame comes from the CPU (byte-equal to Off), and no renderer is used after its panic.
crate::reset_frame_renderers_for_tests();
PANICS.store(0, std::sync::atomic::Ordering::SeqCst);
crate::register_frame_renderer(|| Some(Box::new(PanickingRenderer) as Box<dyn FrameRenderer>));
let (p, seq, m) = project();
let mk12 = |path: &String, gpu: crate::GpuRendering| {
let mut s = ExportSettings { format: Format::PngSequence, path: path.clone(), ..Default::default() };
s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(12)));
s.gpu_rendering = gpu;
export(&p, seq, &s, &m, &Progress::default()).unwrap()
};
mk12(&tmp("panic-off.png"), crate::GpuRendering::Off);
mk12(&tmp("panic-auto.png"), crate::GpuRendering::Auto);
for f in ["000", "005", "011"] {
assert_eq!(
std::fs::read(tmp(&format!("panic-off{f}.png"))).unwrap(),
std::fs::read(tmp(&format!("panic-auto{f}.png"))).unwrap(),
"frame {f}: a panicking GPU renderer falls back to the CPU"
);
}
// at most one panic per pooled renderer (4): none is used again after it panicked
assert!(PANICS.load(std::sync::atomic::Ordering::SeqCst) <= 4, "renderer reused after a panic: {}", PANICS.load(std::sync::atomic::Ordering::SeqCst));
crate::reset_frame_renderers_for_tests();
}
static PANICS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
struct PanickingRenderer;
impl FrameRenderer for PanickingRenderer {
fn render(
&mut self,
_project: &Project,
_seq: ItemId,
_t: Tick,
_opts: filmcraft_render::RenderOptions,
_sources: &dyn SourceProvider,
) -> Option<filmcraft_render::Image> {
PANICS.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
panic!("GPU renderer failure (test)");
}
}
struct FakeRenderer;
impl FrameRenderer for FakeRenderer {
fn render(
&mut self,
_project: &Project,
_seq: ItemId,
_t: Tick,
_opts: filmcraft_render::RenderOptions,
_sources: &dyn SourceProvider,
) -> Option<filmcraft_render::Image> {
Some(filmcraft_render::Image::new(320, 180))
}
}

18
crates/format/src/lib.rs
View File

@ -35,6 +35,10 @@
//! (`Project::source_graphics`). No-op step; older builds would drop these fields when saving.
//! - **v13**: all container audio streams, and per-timeline-clip stream selection. Old single-stream
//! `audio` objects read as stream zero; older builds must refuse new multi-stream projects.
//! - **v14** (M7.9): narrations (`Project::narrations`, Text to Speech). No-op step; older builds
//! would drop them when saving.
//! - **v15**: project notes (`Project::notes`, Window ▸ Project Notes). No-op step; older builds
//! would drop them when saving.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
@ -56,7 +60,7 @@ pub type Migration = fn(Value) -> Result<Value, String>;
/// `MIGRATIONS[i]` upgrades schema `i + 1` to `i + 2`. Append one function per schema bump; never
/// edit a shipped one.
pub const MIGRATIONS: &[Migration] =
&[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13];
&[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13, v13_to_v14, v14_to_v15];
/// The schema version this build writes (and the newest it reads).
pub const SCHEMA_VERSION: u32 = MIGRATIONS.len() as u32 + 1;
@ -347,13 +351,23 @@ fn v12_to_v13(doc: Value) -> Result<Value, String> {
Ok(doc)
}
/// v13 → v14: narrations (Text to Speech). Existing data needs no change.
fn v13_to_v14(doc: Value) -> Result<Value, String> {
Ok(doc)
}
/// v14 → v15: project notes. Existing data needs no change.
fn v14_to_v15(doc: Value) -> Result<Value, String> {
Ok(doc)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn schema_version_matches_table() {
assert_eq!(SCHEMA_VERSION, 13);
assert_eq!(SCHEMA_VERSION, 15);
}
#[test]

6
crates/gpu/Cargo.toml
View File

@ -13,14 +13,18 @@ workspace = true
filmcraft-color = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-geom = { workspace = true }
filmcraft-render = { workspace = true }
filmcraft-project = { workspace = true }
filmcraft-render = { workspace = true }
filmcraft-time = { workspace = true }
wgpu = { version = "30", default-features = false, features = ["wgsl"] }
log = { workspace = true }
pollster = "0.4"
rayon = { workspace = true }
[dev-dependencies]
pollster = "0.4"
filmcraft-export = { workspace = true }
image = { version = "0.25", default-features = false, features = ["png"] }
wgpu = { version = "30", features = ["wgsl", "metal", "vulkan", "dx12", "gles"] }
filmcraft-media = { workspace = true }
filmcraft-time = { workspace = true }

5
crates/gpu/src/composite.wgsl
View File

@ -45,7 +45,10 @@ fn vs(@builtin(vertex_index) vi: u32) -> VOut {
fn load4(t: texture_2d<f32>, p: vec2<f32>) -> vec4<f32> {
let d = vec2<i32>(textureDimensions(t));
let q = p - 0.5;
var q = p - 0.5;
let rq = round(q);
if abs(q.x - rq.x) < 1e-4 { q.x = rq.x; }
if abs(q.y - rq.y) < 1e-4 { q.y = rq.y; }
let i = vec2<i32>(floor(q));
let f = q - floor(q);
let a = textureLoad(t, clamp(i, vec2(0), d - 1), 0);

306
crates/gpu/src/export_renderer.rs Normal file
View File

@ -0,0 +1,306 @@
//! Off-screen GPU export renderer: composite a frame with the same
//! [`filmcraft_render::plan::plan_frame`] + [`GpuCompositor`] path the monitors use, then read
//! the **linear float accumulator** back (not the sRGB display texture), so the result matches
//! `filmcraft_render::render_sequence`'s premultiplied linear-light RGBA and the export pipeline's
//! placement, overlays and video limiter see exactly what the CPU renderer would have produced.
//!
//! Renderers share one off-screen wgpu device (no window), created by the first and kept for the
//! process, so an export can run several (one per export worker; each has its own compositor and
//! textures) without opening a device each. With no adapter (headless
//! machine, unsupported GPU) [`ExportRenderer::new`] returns `None` and the export falls back to
//! the CPU renderer; a `FramePlan::Image` plan (HDR / wide gamut / anything the plan sends to the
//! CPU) is returned unchanged — it already is the CPU image.
use crate::{GpuCompositor, prepare};
use filmcraft_project::{ItemId, Project};
use filmcraft_render::plan::{FramePlan, plan_frame};
use filmcraft_render::{Image, RenderOptions, SourceProvider};
use filmcraft_time::Tick;
/// Where export rendering spends its time (process-wide totals, for the bench).
pub mod timing {
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
static SUBMIT_NS: AtomicU64 = AtomicU64::new(0);
static MAP_WAIT_NS: AtomicU64 = AtomicU64::new(0);
static CONVERT_NS: AtomicU64 = AtomicU64::new(0);
static FRAMES: AtomicU64 = AtomicU64::new(0);
/// Totals since the last [`reset`].
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuTimings {
/// Recording and submitting the frame's commands.
pub submit: Duration,
/// Waiting for the GPU and the staging buffer's mapping.
pub map_wait: Duration,
/// Half float → f32 conversion of the read-back pixels.
pub convert: Duration,
/// Frames composited by the GPU.
pub frames: u64,
}
fn add(counter: &AtomicU64, d: Duration) {
counter.fetch_add(u64::try_from(d.as_nanos()).unwrap_or(u64::MAX), Ordering::Relaxed);
}
pub(crate) fn add_submit(d: Duration) {
add(&SUBMIT_NS, d);
}
pub(crate) fn add_map_wait(d: Duration) {
add(&MAP_WAIT_NS, d);
}
pub(crate) fn add_convert(d: Duration) {
add(&CONVERT_NS, d);
}
pub(crate) fn add_frame() {
FRAMES.fetch_add(1, Ordering::Relaxed);
}
pub fn get() -> GpuTimings {
let d = |c: &AtomicU64| Duration::from_nanos(c.load(Ordering::Relaxed));
GpuTimings { submit: d(&SUBMIT_NS), map_wait: d(&MAP_WAIT_NS), convert: d(&CONVERT_NS), frames: FRAMES.load(Ordering::Relaxed) }
}
pub fn reset() {
for c in [&SUBMIT_NS, &MAP_WAIT_NS, &CONVERT_NS, &FRAMES] {
c.store(0, Ordering::Relaxed);
}
}
}
/// Set once the shared export device reported an uncaptured error (validation, out of memory)
/// or was lost: every export renders on the CPU from then on. The device is shared by every
/// renderer and lives in a `OnceLock`, so it is not recreated in this process.
static DEVICE_FAILED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
/// Retire the shared export device, logging the first reason only.
fn mark_device_failed(why: &str) {
if !DEVICE_FAILED.swap(true, std::sync::atomic::Ordering::SeqCst) {
log::error!("GPU export disabled for this session ({why}); exports render on the CPU");
}
}
fn device_failed() -> bool {
DEVICE_FAILED.load(std::sync::atomic::Ordering::SeqCst)
}
pub struct ExportRenderer {
_instance: wgpu::Instance,
#[expect(dead_code, reason = "kept for future frame-accuracy flows (readback fences)")]
device: wgpu::Device,
#[expect(dead_code, reason = "kept for future frame-accuracy flows (readback fences)")]
queue: wgpu::Queue,
compositor: GpuCompositor,
}
impl ExportRenderer {
/// The off-screen device, or `None` when this host has no usable adapter.
pub fn new() -> Option<Self> {
// `None` is kept too: a machine without an adapter does not search again on every export
static SHARED: std::sync::OnceLock<Option<(wgpu::Instance, wgpu::Device, wgpu::Queue)>> = std::sync::OnceLock::new();
let (instance, device, queue) = SHARED
.get_or_init(|| {
let instance = wgpu::Instance::default();
let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?;
let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()?;
// wgpu's default handler panics on a validation or out-of-memory error; log it and
// send the remaining frames to the CPU instead.
device.on_uncaptured_error(std::sync::Arc::new(|e: wgpu::Error| mark_device_failed(&format!("GPU error: {e}"))));
device.set_device_lost_callback(|reason, msg| mark_device_failed(&format!("GPU device lost ({reason:?}): {msg}")));
Some((instance, device, queue))
})
.as_ref()?
.clone();
if device_failed() {
return None;
}
let compositor = GpuCompositor::new(&device, &queue);
Some(Self { _instance: instance, device, queue, compositor })
}
/// One frame as `render_sequence` would return it (premultiplied linear-light RGBA at the
/// render size), or `None` when this frame must go to the CPU renderer.
pub fn render(&mut self, project: &Project, seq: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> Option<Image> {
let plan = plan_frame(project, seq, t, opts, sources);
match &plan {
// Already the CPU image: return it unchanged instead of rendering it twice.
FramePlan::Image(img) => Some(img.clone()),
FramePlan::Layers { .. } => {
if device_failed() {
return None;
}
let prep = prepare(&plan);
let (w, h, px) = self.compositor.render_export_prepared(&plan, Some(&prep))?;
// an error reported while this frame was drawn leaves its pixels untrusted
if device_failed() {
return None;
}
timing::add_frame();
Some(Image { w: w as usize, h: h as usize, px })
}
// Transitions: the export path leaves them to the CPU reference renderer.
FramePlan::Composite { .. } => None,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{f16_to_f32, f32_to_f16};
use filmcraft_export::FrameRenderer;
use filmcraft_media::generators::GeneratorSource;
use filmcraft_media::{Generator, MediaSource};
use filmcraft_project::{ItemKind, Label, MediaClip, MediaRef, Project, SequenceSettings, TrackKind};
use filmcraft_render::SourceMap;
use filmcraft_time::{TICKS_PER_SECOND, Tick, TimeRange};
use std::sync::Arc;
const EPS: f32 = 1e-4;
/// A solid colour generator at 320×180 plus a tone generator (mirrors the export tests'
/// fixture-free project helper).
fn project(transform: bool) -> (Arc<Project>, ItemId, SourceMap) {
let mut p = Project::new("x");
let g = GeneratorSource::new(
Generator::ColorMatte { color: [0.2, 0.5, 0.9, 1.0] },
320,
180,
filmcraft_time::FrameRate::FPS_24,
Tick(2 * TICKS_PER_SECOND),
);
let tone = GeneratorSource::new(Generator::Tone { hz: 440.0, db: -6.0 }, 320, 180, filmcraft_time::FrameRate::FPS_24, Tick(2 * TICKS_PER_SECOND));
let add = |p: &mut Project, g: &GeneratorSource| {
let info = g.info().clone();
p.add_item(
&info.name.clone(),
Label::Iris,
ItemKind::Media(MediaClip {
media: MediaRef::Generator(g.generator.clone()),
info,
interpret: Default::default(),
mark_in: None,
mark_out: None,
markers: vec![],
offline: false,
proxy: None,
identity: None,
}),
None,
)
};
let red = add(&mut p, &g);
let t = add(&mut p, &tone);
let seq =
p.new_sequence("s", SequenceSettings { width: 320, height: 180, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 1, 1, None);
let r = filmcraft_time::FrameRate::FPS_24;
let _v = p.make_track_item(red, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, r.tick_of(24)), r).unwrap();
let a = p.make_track_item(t, TrackKind::Audio, Tick::ZERO, TimeRange::new(Tick::ZERO, r.tick_of(24)), r).unwrap();
if transform {
// Motion + opacity as fixed effects (the way the golden transform scene does it).
if let Some(clip) = p.sequence_mut(seq).unwrap().video_tracks[0].items.last_mut() {
let m = clip.effect_mut("motion").unwrap();
if let Some(pm) = m.param_mut("scale") {
pm.value = filmcraft_project::ParamValue::Float(45.0);
}
if let Some(pm) = m.param_mut("rotation") {
pm.value = filmcraft_project::ParamValue::Float(20.0);
}
if let Some(pm) = m.param_mut("position") {
pm.value = filmcraft_project::ParamValue::Vec2(filmcraft_geom::Vec2::new(215.0, 70.0));
}
if let Some(o) = clip.effect_mut("opacity")
&& let Some(pm) = o.param_mut("opacity")
{
pm.value = filmcraft_project::ParamValue::Float(70.0);
}
}
}
p.sequence_mut(seq).unwrap().audio_tracks[0].items.push(a);
let mut m = SourceMap::default();
m.0.insert(red, Arc::new(g));
m.0.insert(t, Arc::new(tone));
(Arc::new(p), seq, m)
}
/// GPU parity: p99 of the per-pixel max RGB difference (over black, 8-bit) ≤ 6 and mean < 1.5
/// — the same criterion the golden GPU-vs-CPU parity tests use.
fn assert_parity(cpu: &Image, gpu: &Image, what: &str) {
assert_eq!((cpu.w, cpu.h), (gpu.w, gpu.h), "{what}: size mismatch");
let to8 = |img: &Image| -> Vec<u8> { img.over_black_rgba8() };
let (a, b) = (to8(cpu), to8(gpu));
let mut d: Vec<u32> = a
.as_chunks::<4>()
.0
.iter()
.zip(b.as_chunks::<4>().0)
.map(|(x, y)| (0..3).map(|k| (x[k] as u32).abs_diff(y[k] as u32)).max().unwrap_or(0))
.collect();
d.sort_unstable();
let p99 = d[d.len() * 99 / 100];
let mean = d.iter().sum::<u32>() as f64 / d.len() as f64;
eprintln!("{what}: GPU vs CPU over-black p99 {p99}, mean {mean:.3}");
assert!(p99 <= 6 && mean < 1.5, "{what}: GPU parity p99 {p99}, mean {mean:.3}");
}
#[test]
fn solid_frame_matches_cpu() {
let Some(mut r) = ExportRenderer::new() else {
eprintln!("SKIPPED (gpu export parity): no adapter");
return;
};
let (p, seq, m) = project(false);
let t = Tick::ZERO;
let cpu = filmcraft_render::render_sequence(&p, seq, t, RenderOptions::default(), &m);
let gpu = r.render(&p, seq, t, RenderOptions::default(), &m).expect("GPU rendered the frame");
// The linear premultiplied accumulator should be very close to the CPU float image.
assert_eq!((cpu.w, cpu.h), (gpu.w, gpu.h));
let max: f32 = cpu.px.iter().zip(&gpu.px).map(|(a, b)| (a - b).abs()).fold(0.0, f32::max);
eprintln!("solid: max |GPU-CPU| linear {max:.6}");
assert!(max < 0.02, "solid frame differs by {max}");
assert_parity(&cpu, &gpu, "solid");
}
#[test]
fn transformed_frame_matches_cpu() {
let Some(mut r) = ExportRenderer::new() else {
eprintln!("SKIPPED (gpu export parity): no adapter");
return;
};
let (p, seq, m) = project(true);
let t = filmcraft_time::FrameRate::FPS_24.tick_of(10);
let cpu = filmcraft_render::render_sequence(&p, seq, t, RenderOptions::default(), &m);
let gpu = r.render(&p, seq, t, RenderOptions::default(), &m).expect("GPU rendered the frame");
assert_parity(&cpu, &gpu, "transformed");
}
#[test]
fn gpu_render_impls_frame_renderer() {
// dev-dependency: the same wrapper the app and CLI register.
struct R(ExportRenderer);
impl FrameRenderer for R {
fn render(&mut self, project: &Project, seq: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> Option<Image> {
self.0.render(project, seq, t, opts, sources)
}
}
let Some(r) = ExportRenderer::new() else {
eprintln!("SKIPPED (gpu export parity): no adapter");
return;
};
let (p, seq, m) = project(false);
let gpu = R(r).render(&p, seq, Tick::ZERO, RenderOptions::default(), &m).expect("rendered");
let cpu = filmcraft_render::render_sequence(&p, seq, Tick::ZERO, RenderOptions::default(), &m);
assert_parity(&cpu, &gpu, "frame renderer");
}
#[test]
fn f16_roundtrip() {
for v in [0.0f32, 1.0, 0.5, 2.0, 1e-3, 65504.0, -1.0, -0.25] {
assert!((f16_to_f32(f32_to_f16(v)) - v).abs() <= EPS * v.abs().max(1.0), "f16 {v}");
}
}
}

76
crates/gpu/src/fx.rs
View File

@ -40,6 +40,7 @@ const OP_ASC_CDL: u32 = 15;
const OP_CHANNEL_MIX: u32 = 16;
const OP_COLOR_REPLACE: u32 = 17;
const OP_ALPHA_ADJUST: u32 = 18;
const OP_VIGNETTE: u32 = 19;
const OP_BOX: u32 = 20;
const OP_DIRECTIONAL: u32 = 22;
const OP_UNSHARP: u32 = 23;
@ -49,6 +50,8 @@ const OP_HFLIP: u32 = 26;
const OP_VFLIP: u32 = 27;
const OP_MIRROR: u32 = 28;
const OP_OFFSET: u32 = 29;
const OP_VIDEO_LIMITER: u32 = 30;
const OP_LUMETRI: u32 = 31;
type Target = (wgpu::Texture, wgpu::TextureView);
@ -85,7 +88,7 @@ struct Step {
run: bool,
code: u32,
i1: [u32; 4],
p: [f32; 16],
p: [f32; 24],
/// Unsharp combine: reads the blurred image and the original.
combine: bool,
/// A blur pass of Unsharp Mask (the original stays untouched until the combine).
@ -93,7 +96,7 @@ struct Step {
}
fn step(code: u32, i1: [u32; 4], p: &[f32]) -> Step {
let mut q = [0.0; 16];
let mut q = [0.0; 24];
for (d, s) in q.iter_mut().zip(p) {
*d = *s;
}
@ -175,6 +178,73 @@ fn steps(op: &FxOp) -> Vec<Step> {
FxOp::AlphaAdjust { opacity, ignore, invert, mask_only } => {
out.push(step(OP_ALPHA_ADJUST, [*ignore as u32, *invert as u32, *mask_only as u32, 0], &[*opacity]));
}
FxOp::Vignette { amount, midpoint, roundness, feather, target } => {
if amount.abs() >= 1e-5 {
out.push(step(OP_VIGNETTE, [0; 4], &[*amount, *midpoint, *roundness, *feather, target[0], target[1], target[2]]));
}
}
FxOp::VideoLimiter { max, comp, axis, warn, warning_color } => {
out.push(step(OP_VIDEO_LIMITER, [*axis, *warn as u32, 0, 0], &[*max, *comp, warning_color[0], warning_color[1], warning_color[2]]));
}
FxOp::Lumetri {
gains,
exposure,
contrast,
hl,
sh,
wh,
bl,
sat,
creative_on,
faded,
vib,
st,
ht,
vignette_on,
va,
vmid,
vround,
vfeather,
aspect,
..
} => {
let ge = [gains[0] * exposure, gains[1] * exposure, gains[2] * exposure];
let b0 = -bl * 0.15;
let w0 = 1.0 - wh * 0.15;
let sh_k = sh * 0.35;
let hl_k = hl * 0.35;
let vround_aspect = filmcraft_render::gpufx::lumetri_vignette_aspect(*aspect, *vround);
let st_k = [(st[0] - 0.5) * 0.3, (st[1] - 0.5) * 0.3, (st[2] - 0.5) * 0.3];
let ht_k = [(ht[0] - 0.5) * 0.3, (ht[1] - 0.5) * 0.3, (ht[2] - 0.5) * 0.3];
let p = [
ge[0],
ge[1],
ge[2],
b0,
w0,
sh_k,
hl_k,
*contrast,
*faded,
*sat,
*vib,
*va,
*vmid,
*vfeather,
vround_aspect,
0.0,
st_k[0],
st_k[1],
st_k[2],
0.0,
ht_k[0],
ht_k[1],
ht_k[2],
0.0,
];
out.push(step(OP_LUMETRI, [*creative_on as u32, *vignette_on as u32, 0, 0], &p));
}
}
out
}
@ -340,7 +410,7 @@ impl FxStage {
(true, Some(o)) => views.get(o)?,
_ => dummy,
};
let mut bytes = Vec::with_capacity(96);
let mut bytes = Vec::with_capacity(128);
for v in [s.code, w, h, 0].iter().chain(&s.i1) {
bytes.extend_from_slice(&v.to_le_bytes());
}

245
crates/gpu/src/fx.wgsl
View File

@ -10,6 +10,8 @@ struct U {
p1: vec4<f32>,
p2: vec4<f32>,
p3: vec4<f32>,
p4: vec4<f32>,
p5: vec4<f32>,
};
@group(0) @binding(0) var<uniform> u: U;
@ -36,6 +38,7 @@ const OP_ASC_CDL: u32 = 15u;
const OP_CHANNEL_MIX: u32 = 16u;
const OP_COLOR_REPLACE: u32 = 17u;
const OP_ALPHA_ADJUST: u32 = 18u;
const OP_VIGNETTE: u32 = 19u;
const OP_BOX: u32 = 20u;
const OP_DIRECTIONAL: u32 = 22u;
const OP_UNSHARP: u32 = 23u;
@ -45,6 +48,8 @@ const OP_HFLIP: u32 = 26u;
const OP_VFLIP: u32 = 27u;
const OP_MIRROR: u32 = 28u;
const OP_OFFSET: u32 = 29u;
const OP_VIDEO_LIMITER: u32 = 30u;
const OP_LUMETRI: u32 = 31u;
fn size() -> vec2<i32> {
return vec2<i32>(i32(u.i0.y), i32(u.i0.z));
@ -218,6 +223,88 @@ fn hsl_to_rgb(h: f32, s: f32, l: f32) -> vec3<f32> {
return vec3(hsl_channel(p, q, h + 1.0 / 3.0), hsl_channel(p, q, h), hsl_channel(p, q, h - 1.0 / 3.0));
}
fn linear_to_enc_limiter(v: f32) -> f32 {
if v <= 1.0 {
return linear_to_srgb1(max(v, 0.0));
}
return 1.0 + (v - 1.0) / 2.4;
}
fn enc_to_linear_limiter(v: f32) -> f32 {
if v <= 1.0 {
return srgb_to_linear1(max(v, 0.0));
}
return 1.0 + (v - 1.0) * 2.4;
}
fn knee_limiter(v: f32, max_val: f32, comp: f32) -> f32 {
let k = max_val * (1.0 - comp);
if comp <= 0.0 || v <= k {
return min(v, max_val);
}
let r = max_val - k;
return k + r * (1.0 - exp(-(v - k) / r));
}
fn to_rgb_709(y: f32, cb: f32, cr: f32) -> vec3<f32> {
let kr = 0.2126;
let kb = 0.0722;
let kg = 1.0 - kr - kb;
let r = y + 2.0 * (1.0 - kr) * cr;
let b = y + 2.0 * (1.0 - kb) * cb;
let g = (y - kr * r - kb * b) / kg;
return vec3(r, g, b);
}
fn limit_video(v: vec3<f32>, max_val: f32, comp: f32, axis: u32) -> vec3<f32> {
let kr = 0.2126;
let kb = 0.0722;
let kg = 1.0 - kr - kb;
var y = kr * v.x + kg * v.y + kb * v.z;
var cb = (v.z - y) / (2.0 * (1.0 - kb));
var cr = (v.x - y) / (2.0 * (1.0 - kr));
if axis != 1u {
y = knee_limiter(max(y, 0.0), max_val, comp);
}
if axis != 0u {
let rgb = to_rgb_709(y, cb, cr);
var s = 1.0;
let d = rgb - vec3(y);
if rgb.x > max_val && d.x > 1e-6 {
s = min(s, max((max_val - y) / d.x, 0.0));
}
if rgb.x < 0.0 && d.x < -1e-6 {
s = min(s, max(y / (-d.x), 0.0));
}
if rgb.y > max_val && d.y > 1e-6 {
s = min(s, max((max_val - y) / d.y, 0.0));
}
if rgb.y < 0.0 && d.y < -1e-6 {
s = min(s, max(y / (-d.y), 0.0));
}
if rgb.z > max_val && d.z > 1e-6 {
s = min(s, max((max_val - y) / d.z, 0.0));
}
if rgb.z < 0.0 && d.z < -1e-6 {
s = min(s, max(y / (-d.z), 0.0));
}
if axis == 3u && s < 1.0 - 1e-4 {
s = min(knee_limiter(s, 1.0, max(comp, 0.03)), 1.0);
}
cb *= s;
cr *= s;
}
let out_rgb = to_rgb_709(y, cb, cr);
return clamp(out_rgb, vec3(0.0), vec3(max_val));
}
// ---- per-pixel colour effects on straight colour (`Image::map_rgb`)
fn color_op(op: u32, c: vec3<f32>) -> vec3<f32> {
@ -349,6 +436,95 @@ fn color_op(op: u32, c: vec3<f32>) -> vec3<f32> {
}
}
fn lumetri_op(c: vec3<f32>, p: vec2<i32>) -> vec3<f32> {
let s = size();
let creative_on = u.i1.x != 0u;
let vignette_on = u.i1.y != 0u;
// white balance + exposure in linear light
let ge = u.p0.xyz;
let lin = c * ge;
var v = enc(lin);
// whites / blacks: endpoints
let b0 = u.p0.w;
let w0 = u.p1.x;
let denom = max(w0 - b0, 1e-3);
v = (v - vec3(b0)) / denom;
// highlights / shadows: luma-weighted lift/compress, hue preserving
let sh_k = u.p1.y;
let hl_k = u.p1.z;
let l = luma709(v);
let ws_val = clamp(1.0 - l, 0.0, 1.0);
let ws = ws_val * ws_val * ws_val;
let whl_val = clamp(l, 0.0, 1.0);
let whl = whl_val * whl_val * whl_val;
let nl = max(l + sh_k * ws + hl_k * whl, 0.0);
if l > 1e-5 {
let k = nl / l;
v = v * k;
}
// contrast: smooth S-curve around mid grey
let contrast = u.p1.w;
if abs(contrast) > 1e-4 {
let k = 1.0 + contrast;
let q = clamp(v, vec3(0.0), vec3(1.0));
let sc = q * q * (vec3(3.0) - 2.0 * q);
if k >= 1.0 {
v = q + (sc - q) * (k - 1.0);
} else {
v = vec3(0.5) + (q - vec3(0.5)) * k;
}
}
// faded film: lift blacks and compress
let faded = u.p2.x;
if faded > 0.0 {
v = v * (1.0 - 0.25 * faded) + vec3(0.12 * faded);
}
// split tone
if creative_on {
let st_k = u.p4.xyz;
let ht_k = u.p5.xyz;
let l2 = clamp(luma709(v), 0.0, 1.0);
v = v + st_k * (1.0 - l2) + ht_k * l2;
}
// saturation & vibrance
let sat = u.p2.y;
let vib = u.p2.z;
let l3 = luma709(v);
let cur_sat = max(max(v.x, v.y), v.z) - min(min(v.x, v.y), v.z);
let sat_mult = sat * (1.0 + vib * (1.0 - clamp(cur_sat, 0.0, 1.0)));
v = vec3(l3) + (v - vec3(l3)) * sat_mult;
// vignette
let va = u.p2.w;
if vignette_on && abs(va) > 1e-4 {
let vmid = u.p3.x;
let vfeather = u.p3.y;
let vround_aspect = u.p3.z;
let w = f32(s.x);
let h = f32(s.y);
let nx = (f32(p.x) / w - 0.5) * 2.0 * vround_aspect;
let ny = (f32(p.y) / h - 0.5) * 2.0;
let d = sqrt(nx * nx + ny * ny) / 1.4142135623730951;
let edge = clamp((d - vmid * 0.9) / (max(vfeather, 0.01) * 0.9), 0.0, 1.0);
let e2 = edge * edge * (3.0 - 2.0 * edge);
let k = 1.0 + va * 0.2 * e2;
if va < 0.0 {
v = v * max(k, 0.0);
} else {
v = v + (vec3(1.0) - v) * (k - 1.0);
}
}
return dec(v);
}
// One output pixel of every op but the running-sum box blur.
fn pixel(op: u32, p: vec2<i32>) -> vec4<f32> {
let s = size();
@ -358,7 +534,11 @@ fn pixel(op: u32, p: vec2<i32>) -> vec4<f32> {
let o = ld(p);
let a = o.a;
let na = 1.0 - a;
let k = select(0.0, na / a, a > 1e-6);
// (the CPU's threshold; `na / a` is only used when a > 1e-6)
var k = 0.0;
if a > 1e-6 {
k = na / a;
}
let blend = u.p0.x;
return vec4(o.rgb * k, na * (1.0 - blend) + a * blend);
}
@ -467,6 +647,69 @@ fn pixel(op: u32, p: vec2<i32>) -> vec4<f32> {
let o = ld(p);
return q + (o - q) * u.p0.z;
}
case OP_VIGNETTE: {
let o = ld(p);
let a = o.a;
if a <= 1e-6 {
return o;
}
let amt = u.p0.x;
if abs(amt) < 1e-5 {
return o;
}
let mid = u.p0.y;
let round = u.p0.z;
let feather = u.p0.w;
let tgt = u.p1.xyz;
let w = f32(s.x);
let h = f32(s.y);
let aspect = w / h;
var nx = pc.x / w * 2.0 - 1.0;
let ny = pc.y / h * 2.0 - 1.0;
if round > 0.0 {
nx *= 1.0 + (aspect - 1.0) * round;
}
let p_exp = select(2.0, 2.0 + (-round) * 6.0, round < 0.0);
let d = powf(powf(abs(nx), p_exp) + powf(abs(ny), p_exp), 1.0 / p_exp) / powf(2.0, 1.0 / p_exp);
let edge = smoothstep_fx(mid - feather * 0.5, mid + feather * 0.5, d);
let c = o.rgb / a;
let ec = enc(c);
let res = dec(ec + (tgt - ec) * (edge * abs(amt)));
return vec4(res * a, a);
}
case OP_VIDEO_LIMITER: {
let o = ld(p);
let a = o.a;
if a <= 1e-6 {
return o;
}
let c = o.rgb / a;
let v = vec3(linear_to_enc_limiter(c.x), linear_to_enc_limiter(c.y), linear_to_enc_limiter(c.z));
let max_val = u.p0.x;
let comp = u.p0.y;
let axis = u.i1.x;
let warn = u.i1.y != 0u;
let wc = u.p0.zw;
let wc_b = u.p1.x;
let warning_col = vec3(wc.x, wc.y, wc_b);
let out_enc = limit_video(v, max_val, comp, axis);
let diff = abs(v - out_enc);
if warn && (diff.x > 1e-4 || diff.y > 1e-4 || diff.z > 1e-4) {
return vec4(warning_col * a, a);
}
let res = vec3(enc_to_linear_limiter(out_enc.x), enc_to_linear_limiter(out_enc.y), enc_to_linear_limiter(out_enc.z));
return vec4(res * a, a);
}
case OP_LUMETRI: {
let o = ld(p);
let a = o.a;
if a <= 1e-6 {
return o;
}
let c = o.rgb / a;
let res = lumetri_op(c, p);
return vec4(res * a, a);
}
default: {
let o = ld(p);
let a = o.a;

183
crates/gpu/src/fx_tests.rs
View File

@ -41,6 +41,9 @@ fn fl(v: f64) -> ParamValue {
fn col(r: f32, g: f32, b: f32) -> ParamValue {
ParamValue::Color([r, g, b, 1.0])
}
fn ch(c: u32) -> ParamValue {
ParamValue::Choice(c)
}
fn pt(x: f64, y: f64) -> ParamValue {
ParamValue::Vec2(Vec2::new(x, y))
}
@ -187,6 +190,113 @@ fn cases() -> Vec<(&'static str, Vec<(&'static str, ParamValue)>, bool)> {
("alpha_adjust", vec![("opacity", fl(60.0))], false),
("alpha_adjust", vec![("ignore", ParamValue::Bool(true)), ("invert", ParamValue::Bool(true))], false),
("alpha_adjust", vec![("invert", ParamValue::Bool(true)), ("mask_only", ParamValue::Bool(true)), ("opacity", fl(150.0))], false),
("vignette", vec![], false),
("vignette", vec![("amount", fl(-100.0)), ("midpoint", fl(50.0)), ("roundness", fl(0.0)), ("feather", fl(50.0))], false),
(
"vignette",
vec![("amount", fl(-30.0)), ("midpoint", fl(20.0)), ("roundness", fl(-100.0)), ("feather", fl(0.0)), ("color", col(0.1, 0.4, 0.7))],
false,
),
("vignette", vec![("amount", fl(40.0)), ("midpoint", fl(90.0)), ("roundness", fl(100.0)), ("feather", fl(100.0))], false),
("video_limiter", vec![], false),
("video_limiter", vec![("axis", ch(0)), ("clip_level", ch(0)), ("compression", ch(0))], false),
("video_limiter", vec![("axis", ch(1)), ("clip_level", ch(9)), ("compression", ch(4))], false),
("video_limiter", vec![("axis", ch(2)), ("clip_level", ch(5)), ("compression", ch(2))], false),
("video_limiter", vec![("axis", ch(3)), ("clip_level", ch(0)), ("compression", ch(1))], false),
(
"video_limiter",
vec![
("axis", ch(2)),
("clip_level", ch(0)),
("compression", ch(0)),
("gamut_warning", ParamValue::Bool(true)),
("warning_color", col(0.0, 1.0, 0.0)),
],
false,
),
(
"lumetri",
vec![
("temperature", fl(20.0)),
("tint", fl(-10.0)),
("exposure", fl(0.5)),
("contrast", fl(15.0)),
("highlights", fl(-20.0)),
("shadows", fl(25.0)),
("whites", fl(10.0)),
("blacks", fl(-15.0)),
("saturation", fl(110.0)),
("creative_on", ParamValue::Bool(false)),
("vignette_on", ParamValue::Bool(false)),
],
false,
),
(
"lumetri",
vec![
("basic_on", ParamValue::Bool(false)),
("creative_sat", fl(120.0)),
("vibrance", fl(30.0)),
("faded_film", fl(25.0)),
("shadow_tint", col(0.4, 0.45, 0.6)),
("highlight_tint", col(0.6, 0.55, 0.4)),
("vignette_on", ParamValue::Bool(false)),
],
false,
),
(
"lumetri",
vec![
("basic_on", ParamValue::Bool(false)),
("creative_on", ParamValue::Bool(false)),
("vignette_amount", fl(-3.0)),
("vignette_midpoint", fl(45.0)),
("vignette_roundness", fl(-30.0)),
("vignette_feather", fl(60.0)),
],
false,
),
(
"lumetri",
vec![
("temperature", fl(-15.0)),
("tint", fl(10.0)),
("exposure", fl(0.3)),
("contrast", fl(20.0)),
("highlights", fl(-10.0)),
("shadows", fl(15.0)),
("whites", fl(-5.0)),
("blacks", fl(5.0)),
("saturation", fl(105.0)),
("creative_sat", fl(110.0)),
("vibrance", fl(20.0)),
("faded_film", fl(15.0)),
("shadow_tint", col(0.48, 0.5, 0.55)),
("highlight_tint", col(0.52, 0.5, 0.45)),
("vignette_amount", fl(2.0)),
("vignette_midpoint", fl(50.0)),
("vignette_roundness", fl(20.0)),
("vignette_feather", fl(50.0)),
],
false,
),
(
"lumetri",
vec![("exposure", fl(4.0)), ("contrast", fl(100.0)), ("temperature", fl(100.0)), ("tint", fl(100.0)), ("whites", fl(100.0)), ("blacks", fl(100.0))],
false,
),
(
"lumetri",
vec![
("exposure", fl(-4.0)),
("contrast", fl(-100.0)),
("temperature", fl(-100.0)),
("tint", fl(-100.0)),
("whites", fl(-100.0)),
("blacks", fl(-100.0)),
],
false,
),
]
}
@ -319,6 +429,18 @@ fn hostile_parameters_are_bounded() {
// minifying resamples (the CPU's mip path) stay on the CPU
effect("transform", &[("scale_height", fl(20.0))]),
effect("crop", &[("left", fl(-300.0)), ("zoom", ParamValue::Bool(true))]),
effect("vignette", &[("amount", fl(f64::NAN))]),
effect("vignette", &[("midpoint", fl(f64::INFINITY))]),
effect("vignette", &[("roundness", fl(f64::NAN))]),
effect("vignette", &[("feather", fl(f64::NAN))]),
effect("video_limiter", &[("clip_level", fl(f64::NAN))]),
effect("video_limiter", &[("compression", fl(f64::INFINITY))]),
effect("video_limiter", &[("axis", fl(f64::NAN))]),
effect("video_limiter", &[("warning_color", ParamValue::Color([f32::NAN, 0.0, 0.0, 1.0]))]),
effect("lumetri", &[("exposure", fl(f64::NAN))]),
effect("lumetri", &[("temperature", fl(f64::INFINITY))]),
effect("lumetri", &[("contrast", fl(f64::NAN))]),
effect("lumetri", &[("vignette_amount", fl(f64::NAN))]),
];
for e in &nan {
let op = FxOp::eval(e, &cx(Tick::ZERO, 1.0), w, h);
@ -357,6 +479,67 @@ fn hostile_parameters_are_bounded() {
assert!(c.effect_image(&frame, &LayerFx { size: (1 << 20, 4), decimation: 1, ops: vec![FxOp::BlackWhite] }).is_none());
}
#[test]
fn lumetri_unsupported_sections_fall_back() {
let (w, h) = (40usize, 24usize);
// sharpen > 0 and input LUT (explicitly required by brief)
let e = effect("lumetri", &[("sharpen", fl(10.0)), ("input_lut", ParamValue::Text("custom.cube".into()))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()), "sharpen > 0 and input_lut must fall back to CPU");
// sharpen alone
let e = effect("lumetri", &[("sharpen", fl(20.0))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// input_lut alone
let e = effect("lumetri", &[("input_lut", ParamValue::Text("test.cube".into()))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// look_lut
let e = effect("lumetri", &[("look_lut", ParamValue::Text("look.cube".into()))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// look > 0
let e = effect("lumetri", &[("look", ch(1))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// curve_luma modified
let e = effect("lumetri", &[("curve_luma", ParamValue::Curve(vec![[0.0, 0.1], [1.0, 0.9]]))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// hue_vs_sat non-empty
let e = effect("lumetri", &[("hue_vs_sat", ParamValue::Curve(vec![[0.5, 0.5]]))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// wheel offset
let e = effect("lumetri", &[("wheel_shadows", pt(0.5, 0.0))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// wheel lightness
let e = effect("lumetri", &[("wheel_midtones_l", fl(20.0))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// HSL secondary on
let e = effect("lumetri", &[("hsl_on", ParamValue::Bool(true))]);
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
// HDR working space
let e = effect("lumetri", &[]);
let mut cx_hdr = cx(Tick::ZERO, 1.0);
cx_hdr.working = filmcraft_color::WorkingSpace::Rec2100Pq;
let op = FxOp::eval(&e, &cx_hdr, w, h);
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()));
}
/// Layers with effect chains in a composited plan — YUV and RGBA sources, a decimated working
/// image, rotated / scaled placement, partial opacity and alpha, blend modes — match the CPU plan
/// executor.

131
crates/gpu/src/lib.rs
View File

@ -38,10 +38,12 @@ use filmcraft_frame::{Chroma, PixelData, VideoFrame};
use filmcraft_render::Blend;
use filmcraft_render::plan::{FramePlan, PlanLayer, PlanStep};
pub mod export_renderer;
pub mod fx;
pub mod lut;
pub mod mask;
mod transition;
pub use export_renderer::ExportRenderer;
pub use lut::GpuLut;
pub use mask::GpuMask;
@ -78,6 +80,7 @@ pub struct GpuCompositor {
/// Copy of the accumulator under a blend-mode layer (allocated on first use).
backdrop: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>,
output: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>,
accum_readback_buf: Option<(wgpu::Buffer, u64)>,
uploads: HashMap<(usize, u32, u32), Uploaded>,
clock: u64,
dummy: wgpu::TextureView,
@ -135,6 +138,46 @@ pub fn f32_to_f16(v: f32) -> u16 {
h as u16
}
/// IEEE half → f32 (the inverse of [`f32_to_f16`]), for reading `Rgba16Float` textures back.
pub fn f16_to_f32(h: u16) -> f32 {
let sign = ((h & 0x8000) as u32) << 16;
let exp = ((h >> 10) & 0x1f) as u32;
let mant = (h & 0x03ff) as u32;
let bits = match exp {
0 => {
if mant == 0 {
sign
} else {
// Half subnormal: value = mant × 2⁻²⁴ = (1 + frac) × 2^(b−24) with b the mantissa's
// highest set bit — renormalise into an f32.
let b = 31 - mant.leading_zeros();
let frac = (mant ^ (1 << b)) << (23 - b);
sign | ((103 + b) << 23) | frac
}
}
0x1f => sign | 0x7f80_0000 | (mant << 13),
_ => sign | ((exp + 112) << 23) | (mant << 13),
};
f32::from_bits(bits)
}
/// [`f16_to_f32`] with a branch-light path for normal numbers and zeros (the bulk of read-back
/// data); subnormals, infinities and NaNs take the general path. Same result for every input.
#[inline(always)]
pub fn f16_to_f32_fast(h: u16) -> f32 {
let u = u32::from(h);
let sign = (u & 0x8000) << 16;
let abs = u & 0x7fff;
if (0x0400..0x7c00).contains(&abs) {
// rebias the exponent (127 - 15 = 112) and widen the mantissa
f32::from_bits(sign | ((abs + (112 << 10)) << 13))
} else if abs == 0 {
f32::from_bits(sign)
} else {
f16_to_f32(h)
}
}
/// Half-float texel data for one frame (one byte vector per plane), converted off the UI thread.
/// 8-bit frames need no conversion and have none.
#[derive(Clone, Debug, Default)]
@ -357,6 +400,7 @@ impl GpuCompositor {
accum: None,
backdrop: None,
output: None,
accum_readback_buf: None,
uploads: HashMap::new(),
clock: 0,
dummy,
@ -869,6 +913,22 @@ impl GpuCompositor {
self.render_plan(&empty, None, false, true)
}
/// Export rendering: `plan` composited into the linear float accumulator and read back as
/// premultiplied RGBA f32 (the sRGB resolve is skipped). None for a plan the export path does
/// not run on the GPU (transitions: the caller renders those on the CPU) or a failed read-back.
pub fn render_export_prepared(&mut self, plan: &FramePlan, prep: Option<&PreparedPlan>) -> Option<(u32, u32, Vec<f32>)> {
if matches!(plan, FramePlan::Composite { .. }) {
return None;
}
let t = std::time::Instant::now();
self.render_plan(plan, prep, true, false);
let mut enc = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("filmcraft-composite-export") });
let size = self.copy_accumulator(&mut enc)?;
self.queue.submit([enc.finish()]);
export_renderer::timing::add_submit(t.elapsed());
self.map_accumulator(size)
}
/// Run the effect stage of a layer alone: `frame` decoded into its working image and `fx`
/// applied, read back as linear premultiplied RGBA f32 (what `LayerFx` ops produce on the
/// CPU from the same frame). None when the working image does not fit in a texture.
@ -930,6 +990,77 @@ impl GpuCompositor {
Some((w, h, out))
}
/// Read the **linear float accumulator** back as premultiplied RGBA f32 (export rendering):
/// the display texture this crate's [`read_output`](Self::read_output) resolves is sRGB 8-bit
/// and would clip the working-space data HDR exports need. The accumulator must have been
/// composited first ([`composite`](Self::composite) / [`composite_prepared`](Self::composite_prepared)).
pub fn read_accumulator(&mut self) -> Option<(u32, u32, Vec<f32>)> {
let mut enc = self.device.create_command_encoder(&Default::default());
let size = self.copy_accumulator(&mut enc)?;
self.queue.submit([enc.finish()]);
self.map_accumulator(size)
}
/// Record a copy of the accumulator into the staging buffer (grown when too small, reused
/// otherwise); returns its width, height and row pitch in bytes.
fn copy_accumulator(&mut self, enc: &mut wgpu::CommandEncoder) -> Option<(u32, u32, u32)> {
let (_, _, (w, h)) = self.accum.as_ref()?;
let (w, h) = (*w, *h);
let row = w.checked_mul(8)?.div_ceil(256).checked_mul(256)?;
let needed = u64::from(row) * u64::from(h);
if self.accum_readback_buf.as_ref().is_none_or(|(_, size)| *size < needed) {
let b = self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("accum-readback"),
size: needed,
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
mapped_at_creation: false,
});
self.accum_readback_buf = Some((b, needed));
}
let (tex, _, _) = self.accum.as_ref()?;
let (buf, _) = self.accum_readback_buf.as_ref()?;
enc.copy_texture_to_buffer(
wgpu::TexelCopyTextureInfo { texture: tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
wgpu::TexelCopyBufferInfo { buffer: buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(h) } },
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
Some((w, h, row))
}
/// Wait for the staging buffer filled by [`copy_accumulator`](Self::copy_accumulator) and
/// convert its half floats to f32.
fn map_accumulator(&self, (w, h, row): (u32, u32, u32)) -> Option<(u32, u32, Vec<f32>)> {
if w == 0 || h == 0 {
return None;
}
let t = std::time::Instant::now();
let (buf, _) = self.accum_readback_buf.as_ref()?;
let slice = buf.slice(..);
let (tx, rx) = std::sync::mpsc::channel();
slice.map_async(wgpu::MapMode::Read, move |r| {
let _ = tx.send(r);
});
let _ = self.device.poll(wgpu::PollType::wait_indefinitely());
rx.recv().ok()?.ok()?;
export_renderer::timing::add_map_wait(t.elapsed());
let t = std::time::Instant::now();
let (width, row) = (w as usize, row as usize);
let mut out = filmcraft_frame::pool::take_f32_overwritten(width.checked_mul(h as usize)?.checked_mul(4)?);
let data = slice.get_mapped_range().ok()?;
// every row must be there: the pooled output holds stale values until written
let converted = out.chunks_exact_mut(width * 4).enumerate().all(|(y, dst)| {
let Some(src) = data.get(y * row..y * row + width * 8) else { return false };
for (d, s) in dst.iter_mut().zip(src.as_chunks::<2>().0) {
*d = f16_to_f32_fast(u16::from_le_bytes(*s));
}
true
});
drop(data);
buf.unmap();
export_renderer::timing::add_convert(t.elapsed());
converted.then_some((w, h, out))
}
/// Read the output back as RGBA8 (tests / screenshots / thumbnails).
pub fn read_output(&self) -> Option<(u32, u32, Vec<u8>)> {
let (tex, _, (w, h)) = self.output.as_ref()?;

8
crates/gpu/src/tests.rs
View File

@ -492,3 +492,11 @@ fn gpu_draws_yuv_alpha_plane_like_the_cpu() {
assert!(r != at(&bg, w - 2), "{bits}-bit: right edge {r:?} should differ from the background");
}
}
#[test]
fn fast_half_float_conversion_matches_the_reference_for_every_value() {
for h in 0..=u16::MAX {
let (fast, slow) = (f16_to_f32_fast(h), f16_to_f32(h));
assert!(fast.to_bits() == slow.to_bits() || (fast.is_nan() && slow.is_nan()), "{h:#06x}: {fast} vs {slow}");
}
}

134
crates/gpu/tests/export_pipeline.rs Normal file
View File

@ -0,0 +1,134 @@
//! End-to-end GPU export parity: the registered `filmcraft-gpu` frame renderer drives the full
//! export pipeline (`filmcraft_export::export` → PNG sequence) and its frames must match the
//! `GpuRendering::Off` (CPU reference) frames within the GPU-vs-CPU parity tolerance — p99 of the
//! per-pixel max RGB difference over black ≤ 6 and mean < 1.5 (the golden-test criterion).
use filmcraft_export::{ExportSettings, Format, FrameRenderer, GpuRenderStats, GpuRendering, Progress, export, register_frame_renderer};
use filmcraft_media::generators::GeneratorSource;
use filmcraft_media::{Generator, MediaSource};
use filmcraft_project::{ItemKind, Label, MediaClip, MediaRef, SequenceSettings, TrackKind};
use filmcraft_render::SourceMap;
use filmcraft_time::{TICKS_PER_SECOND, TimeRange};
use std::sync::Arc;
/// The same wrapper the app and CLI register: `filmcraft-gpu`'s renderer behind
/// `filmcraft_export`'s trait.
struct GpuFrameRenderer(filmcraft_gpu::ExportRenderer);
impl FrameRenderer for GpuFrameRenderer {
fn render(
&mut self,
project: &filmcraft_project::Project,
seq: filmcraft_project::ItemId,
t: filmcraft_time::Tick,
opts: filmcraft_render::RenderOptions,
sources: &dyn filmcraft_render::SourceProvider,
) -> Option<filmcraft_render::Image> {
self.0.render(project, seq, t, opts, sources)
}
}
fn project(width: u32, height: u32) -> (Arc<filmcraft_project::Project>, filmcraft_project::ItemId, SourceMap) {
let mut p = filmcraft_project::Project::new("x");
let g = GeneratorSource::new(
Generator::ColorMatte { color: [0.2, 0.5, 0.9, 1.0] },
320,
180,
filmcraft_time::FrameRate::FPS_24,
filmcraft_time::Tick(2 * TICKS_PER_SECOND),
);
let info = g.info().clone();
let red = p.add_item(
&info.name.clone(),
Label::Iris,
ItemKind::Media(MediaClip {
media: MediaRef::Generator(g.generator.clone()),
info,
interpret: Default::default(),
mark_in: None,
mark_out: None,
markers: vec![],
offline: false,
proxy: None,
identity: None,
}),
None,
);
let seq = p.new_sequence("s", SequenceSettings { width, height, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 1, 1, None);
let v = p
.make_track_item(
red,
TrackKind::Video,
filmcraft_time::Tick::ZERO,
TimeRange::new(filmcraft_time::Tick::ZERO, filmcraft_time::FrameRate::FPS_24.tick_of(2)),
filmcraft_time::FrameRate::FPS_24,
)
.unwrap();
p.sequence_mut(seq).unwrap().video_tracks[0].items.push(v);
let mut m = SourceMap::default();
m.0.insert(red, Arc::new(g));
(Arc::new(p), seq, m)
}
fn png(dir: &str, name: &str) -> Vec<u8> {
// image_sequence_path flattens the path stem: <stem><index>.png in the temp dir itself.
let index: u64 = name.trim_start_matches("img").trim_end_matches(".png").parse().unwrap_or(0);
std::fs::read(std::env::temp_dir().join(format!("fc-gpu-export-{dir}{index:03}.png"))).unwrap()
}
fn export_png(dir: &str, gpu: GpuRendering, (w, h): (u32, u32)) {
let (p, seq, m) = project(w, h);
let mut s = ExportSettings {
format: Format::PngSequence,
path: std::env::temp_dir().join(format!("fc-gpu-export-{dir}")).to_string_lossy().into_owned(),
..Default::default()
};
s.range = Some(TimeRange::new(filmcraft_time::Tick::ZERO, filmcraft_time::FrameRate::FPS_24.tick_of(2)));
s.gpu_rendering = gpu;
export(&p, seq, &s, &m, &Progress::default()).unwrap();
}
fn assert_parity(cpu_png: &[u8], gpu_png: &[u8], what: &str) {
let a = image::load_from_memory(cpu_png).unwrap().to_rgba8();
let b = image::load_from_memory(gpu_png).unwrap().to_rgba8();
assert_eq!(a.dimensions(), b.dimensions(), "{what}: size mismatch");
let d: Vec<u32> = a
.as_raw()
.as_chunks::<4>()
.0
.iter()
.zip(b.as_raw().as_chunks::<4>().0)
.map(|(x, y)| (0..3).map(|k| (x[k] as u32).abs_diff(y[k] as u32)).max().unwrap_or(0))
.collect();
let mut d = d;
d.sort_unstable();
let p99 = d[d.len() * 99 / 100];
let mean = d.iter().sum::<u32>() as f64 / d.len() as f64;
eprintln!("{what}: GPU vs CPU p99 {p99}, mean {mean:.3}");
assert!(p99 <= 6 && mean < 1.5, "{what}: GPU parity p99 {p99}, mean {mean:.3}");
}
#[test]
fn gpu_export_matches_cpu_and_counts() {
let Some(_) = filmcraft_gpu::ExportRenderer::new() else {
eprintln!("SKIPPED (gpu export pipeline): no adapter");
return;
};
// Clean registry first: Auto with no factory must be byte-equal to Off.
export_png("off-cpu", GpuRendering::Off, (320, 180));
export_png("off-auto", GpuRendering::Auto, (320, 180));
assert_eq!(png("off-cpu", "img000.png"), png("off-auto", "img000.png"), "Auto with no registered renderer must be byte-equal to Off");
register_frame_renderer(|| Some(Box::new(GpuFrameRenderer(filmcraft_gpu::ExportRenderer::new().expect("adapter present")))));
let sizes = [(320u32, 180u32), (17, 13), (1, 1), (257, 129)];
for (i, size) in sizes.into_iter().enumerate() {
export_png("cpu", GpuRendering::Off, size);
export_png("gpu", GpuRendering::Auto, size);
for f in ["000", "001"] {
assert_parity(&png("cpu", &format!("img{f}.png")), &png("gpu", &format!("img{f}.png")), &format!("{:?} frame {f}", size));
}
let _ = i;
}
let GpuRenderStats { frames, .. } = filmcraft_export::gpu_render_stats();
assert!(frames >= 8, "gpu frames counted across the parity exports: {frames}");
}

57
crates/h264enc/src/lib.rs
View File

@ -297,7 +297,60 @@ pub struct Encoder {
const LOG2_MAX_FRAME_NUM: u32 = 8;
const LOG2_MAX_POC_LSB: u32 = 8;
/// The bitrate a level must allow for constant quality `crf`, which has no bitrate cap. It
/// assumes the worst footage: about 3 bits per pixel at CRF 0 (FilmCraft's CRF 0 exports of real
/// camera footage measured 1.7 on average and 2.4 in their busiest second), halving every 6 steps
/// (one doubling of the quantiser step). The default CRF 23 keeps 1080p30 at level 4.
fn crf_level_kbps(crf: f32, mbw: usize, mbh: usize, fps: f64) -> u32 {
let crf = if crf.is_finite() { f64::from(crf.clamp(0.0, 51.0)) } else { 23.0 };
let bpp = 3.0 * (-crf / 6.0).exp2();
let pixels = (mbw as f64) * (mbh as f64) * 256.0;
// a float-to-int `as` saturates, and NaN (a hostile fps) becomes 0
(pixels * fps * bpp / 1000.0).ceil() as u32
}
impl Encoder {
/// The settings that shaped the stream, x264-style (`key=value` separated by spaces), for the
/// user_data_unregistered SEI: a file says how it was made (`strings`, `ffprobe -show_data`).
fn options(&self) -> String {
let c = &self.cfg;
let rate = match c.rate {
RateControl::Qp(q) => format!("rc=cqp qp={q}"),
RateControl::Crf(f) => format!("rc=crf crf={:.1}", f.clamp(0.0, 51.0)),
RateControl::Cbr { kbps } => format!("rc=cbr bitrate={kbps}"),
RateControl::Vbr { target_kbps, max_kbps } => format!("rc=vbr bitrate={target_kbps} vbv_maxrate={max_kbps}"),
};
let pass = match c.pass {
Pass::Single => "",
Pass::First => " pass=1",
Pass::Second(_) => " pass=2",
};
let profile = match c.profile {
Profile::Baseline => "baseline",
Profile::Main => "main",
Profile::High => "high",
};
let preset = match c.preset {
Preset::Speed => "speed",
Preset::Balanced => "balanced",
Preset::Quality => "quality",
};
let level = self.sps.level_idc;
format!(
"{rate}{pass} profile={profile} level={}.{} preset={preset} keyint={} bframes={} refs={} cabac={} 8x8dct={} slices={} aq={:.2} scenecut={}",
level / 10,
level % 10,
c.keyint,
c.bframes,
self.max_refs,
u8::from(c.profile != Profile::Baseline),
u8::from(c.profile == Profile::High),
self.slices,
c.aq_strength,
u8::from(c.scenecut),
)
}
pub fn new(mut cfg: EncoderConfig) -> Result<Self, Error> {
if cfg.width == 0 || cfg.height == 0 || !cfg.width.is_multiple_of(2) || !cfg.height.is_multiple_of(2) || cfg.width > 16384 || cfg.height > 16384 {
return Err(Error::InvalidDimensions(cfg.width, cfg.height));
@ -376,7 +429,7 @@ impl Encoder {
let max_refs = if cfg.bframes > 0 { 2 } else { 1 };
let (kbps_for_level, rc_kind, vbv) = match (&cfg.rate, &cfg.pass) {
(RateControl::Qp(q), _) => (None, RcKind::Qp((*q).min(51)), (None, None)),
(RateControl::Crf(c), _) => (None, RcKind::Crf(c.clamp(0.0, 51.0)), (None, None)),
(RateControl::Crf(c), _) => (Some(crf_level_kbps(*c, mbw, mbh, fps)), RcKind::Crf(c.clamp(0.0, 51.0)), (None, None)),
(RateControl::Cbr { kbps }, Pass::Second(st)) => (Some(*kbps), ratecontrol::RateControl::plan_two_pass(st, *kbps), (Some(*kbps), Some(*kbps))),
(RateControl::Cbr { kbps }, Pass::First) => (Some(*kbps), RcKind::Abr { kbps: *kbps }, (None, None)),
(RateControl::Cbr { kbps }, Pass::Single) => (Some(*kbps), RcKind::Abr { kbps: *kbps }, (Some(*kbps), Some(*kbps))),
@ -832,7 +885,7 @@ impl Encoder {
}
if !self.sei_sent {
self.sei_sent = true;
let text = format!("filmcraft-h264enc {} - clean-room H.264 encoder", env!("CARGO_PKG_VERSION"));
let text = format!("filmcraft-h264enc {} - clean-room H.264 encoder - options: {}", env!("CARGO_PKG_VERSION"), self.options());
nals.push(nal::nal(0, NAL_SEI, &nal::sei_user_data_rbsp(&text)));
if self.cfg.mastering_display.is_some() || self.cfg.content_light.is_some() {
nals.push(nal::nal(0, NAL_SEI, &nal::sei_hdr_rbsp(self.cfg.mastering_display.as_ref(), self.cfg.content_light)));

45
crates/h264enc/tests/crf_level_and_note.rs Normal file
View File

@ -0,0 +1,45 @@
//! Constant quality has no bitrate cap, so the level must allow what low factors spend; and every
//! stream says which settings made it.
mod common;
use common::synth;
use filmcraft_h264enc::*;
fn crf(w: u32, h: u32, fps: u32, f: f32) -> Encoder {
let mut c = EncoderConfig::new(w, h, fps, 1);
c.rate = RateControl::Crf(f);
c.format = PacketFormat::LengthPrefixed;
Encoder::new(c).unwrap()
}
#[test]
fn crf_level_allows_the_bitrate_of_the_factor() {
let level = |w, h, fps, f| crf(w, h, fps, f).avcc()[3];
// 1080p30: the default and visually lossless factors stay at 4; CRF 0 (measured at 105 Mb/s
// average, 147 Mb/s peak on camera footage) needs more than 4.2's 62.5 Mb/s
assert_eq!(level(1920, 1080, 30, 23.0), 40);
assert_eq!(level(1920, 1080, 30, 18.0), 40);
assert_eq!(level(1920, 1080, 30, 0.0), 51);
// never below what the picture size and rate need
assert_eq!(level(3840, 2160, 30, 51.0), 51);
// a hostile factor is treated as in range
assert_eq!(level(1920, 1080, 30, f32::NAN), 40);
assert_eq!(level(1920, 1080, 30, -100.0), 51);
}
#[test]
fn first_access_unit_carries_the_settings() {
let (w, h) = (64, 48);
let mut enc = crf(w as u32, h as u32, 30, 18.0);
let f = synth(w, h, 0);
let mut packets = enc.encode(&f.frame(), 0).unwrap();
packets.extend(enc.flush());
let text = |p: &Packet| String::from_utf8_lossy(&p.data).into_owned();
let first = text(&packets[0]);
for want in ["filmcraft-h264enc", "rc=crf crf=18.0", "profile=high", "level=", "bframes=2", "cabac=1"] {
assert!(first.contains(want), "{want} missing from {first:?}");
}
// once per stream
assert!(packets.iter().skip(1).all(|p| !text(p).contains("filmcraft-h264enc")));
}

69
crates/interchange/src/aaf/hostile_tests.rs Normal file
View File

@ -0,0 +1,69 @@
//! Hostile object trees: a crafted file must give an error, never a crash, a hang or an
//! allocation sized by a count in the file (AGENTS.md §0).
use filmcraft_cfb::{Version, Writer};
use super::store::{self, SF_WEAK_SET, SF_WEAK_VECTOR, Value, utf16z};
const PID: u16 = 0x0007;
/// A root object whose one property `PID` is a weak reference vector or set (stored form `sf`)
/// named `Keys`, with `index` as its `Keys index` stream.
fn weak_collection_file(sf: u16, index: Vec<u8>) -> Vec<u8> {
let name = utf16z("Keys");
// byte order 'L', format version, 1 entry { pid, stored form, value length }, the value
let mut props = vec![0x4C, 0x20];
for x in [1, PID, sf, name.len() as u16] {
props.extend_from_slice(&x.to_le_bytes());
}
props.extend_from_slice(&name);
let mut w = Writer::new(Version::V3);
w.stream(Writer::ROOT, "properties", props).unwrap();
w.stream(Writer::ROOT, "Keys index", index).unwrap();
w.finish()
}
/// A weak collection index: u32 count, u16 tag, u16 key pid, u8 key size, then the keys.
fn index(count: u32, size: u8, keys: &[u8]) -> Vec<u8> {
let mut v = count.to_le_bytes().to_vec();
v.extend_from_slice(&0u16.to_le_bytes());
v.extend_from_slice(&0x1B01u16.to_le_bytes());
v.push(size);
v.extend_from_slice(keys);
v
}
fn read_keys(sf: u16, index: Vec<u8>) -> Result<Vec<Vec<u8>>, String> {
match store::read(&weak_collection_file(sf, index))?.get(PID) {
Some(Value::WeakVec(_, _, keys)) => Ok(keys.clone()),
other => panic!("not a weak collection: {other:?}"),
}
}
/// With a key size of 0 every key fits, so the count alone drove the loop: up to 4.29 billion
/// empty keys (about 100 GB) from a 9-byte index. A million is enough to show it, cheaply.
#[test]
fn zero_size_weak_collection_keys_are_an_error() {
for sf in [SF_WEAK_VECTOR, SF_WEAK_SET] {
match read_keys(sf, index(1_000_000, 0, &[])) {
Err(e) => assert!(e.contains("zero-size"), "{e}"),
Ok(keys) => panic!("read {} empty keys instead of an error", keys.len()),
}
}
}
#[test]
fn weak_collection_count_beyond_the_index_is_an_error() {
for sf in [SF_WEAK_VECTOR, SF_WEAK_SET] {
let e = read_keys(sf, index(u32::MAX, 4, &[1, 2, 3, 4, 5, 6, 7, 8])).unwrap_err();
assert!(e.contains("short weak collection index"), "{e}");
}
}
#[test]
fn weak_collections_within_the_index_are_read() {
for sf in [SF_WEAK_VECTOR, SF_WEAK_SET] {
assert_eq!(read_keys(sf, index(0, 0, &[])).unwrap(), Vec::<Vec<u8>>::new());
assert_eq!(read_keys(sf, index(2, 4, &[1, 2, 3, 4, 5, 6, 7, 8])).unwrap(), [vec![1, 2, 3, 4], vec![5, 6, 7, 8]]);
}
}

2
crates/interchange/src/aaf/mod.rs
View File

@ -86,5 +86,7 @@ pub fn sniff(bytes: &[u8]) -> bool {
}
}
#[cfg(test)]
mod hostile_tests;
#[cfg(test)]
mod tests;

7
crates/interchange/src/aaf/store.rs
View File

@ -480,6 +480,13 @@ impl R<'_> {
return Err(bad("short weak collection index"));
};
let size = size as usize;
if count > 0 && size == 0 {
return Err(bad("zero-size weak collection key"));
}
// every key must lie in the index, so its length (not the count) bounds the loop
if (count as usize).checked_mul(size).and_then(|n| n.checked_add(9)).is_none_or(|end| end > idx.len()) {
return Err(bad("short weak collection index"));
}
let mut keys = Vec::new();
for k in 0..count as usize {
keys.push(idx.get(9 + k * size..9 + (k + 1) * size).ok_or_else(|| bad("short weak collection index"))?.to_vec());

24
crates/interchange/src/edl.rs
View File

@ -467,7 +467,10 @@ fn import_event(ctx: &mut Ctx, ev: &Event, report: &mut Report) {
let t0 = ctx.rec(l2.tcs[2], df);
let t1 = ctx.rec(l2.tcs[3], df);
let d = ctx.rate.tick_of(l2.dur);
let effect = transition_for(l2.trans, ev.effect.as_deref(), report);
// Resolve separately for each track kind: named audio dissolves have
// different definitions from their video counterparts.
let mut video_effect = None;
let mut audio_effect = None;
let to_black = l2.reel.eq_ignore_ascii_case("BL");
let from_black = l1.reel.eq_ignore_ascii_case("BL");
// Incoming clip.
@ -505,10 +508,8 @@ fn import_event(ctx: &mut Ctx, ev: &Event, report: &mut Report) {
continue;
}
let audio = kind == TrackKind::Audio;
let mut eff = effect.clone();
if audio && eff.def().is_some_and(|d| d.kind != filmcraft_project::EffectKind::AudioTransition) {
eff = find_effect("constant_power").map(|d| d.instance()).unwrap_or(eff);
}
let cache = if audio { &mut audio_effect } else { &mut video_effect };
let eff = cache.get_or_insert_with(|| transition_for(l2.trans, ev.effect.as_deref(), audio, report)).clone();
let id = TransitionId(ctx.b.alloc());
let track = &mut ctx.seq.tracks_mut(kind)[idx];
track.transitions.push(Transition { id, effect: eff, start: t0, duration: d, from, to, align, reverse: false });
@ -564,9 +565,18 @@ fn lay_cut(ctx: &mut Ctx, ev: &Event, l: &Line, video_base: usize, df: bool, m2:
ctx.lay(l, item, &name, t0, t1 - t0, src, *m2, video_base);
}
fn transition_for(t: Trans, effect_name: Option<&str>, report: &mut Report) -> filmcraft_project::EffectInstance {
fn transition_for(t: Trans, effect_name: Option<&str>, audio: bool, report: &mut Report) -> filmcraft_project::EffectInstance {
if let Some(n) = effect_name {
return transition_effect(n, false, report);
let effect = transition_effect(n, audio, report);
if audio && effect.def().is_some_and(|d| d.kind != filmcraft_project::EffectKind::AudioTransition) {
report.warn(format!("transition \"{n}\" is not an audio transition; imported as Constant Power"));
return find_effect("constant_power").map(|d| d.instance()).unwrap_or(effect);
}
return effect;
}
if audio {
// Unnamed EDL audio dissolves default to constant-power fading.
return transition_effect("Constant Power", true, report);
}
match t {
Trans::Wipe(code) => {

45
crates/interchange/tests/edl.rs
View File

@ -99,6 +99,51 @@ fn golden_drop_frame_dissolves() {
assert!(gm.as_media().unwrap().offline);
}
#[test]
fn named_edl_audio_dissolves_keep_their_effects() {
let fixture = "TITLE: AudioDissolve
FCM: NON-DROP FRAME
001 AX A C 00:00:00:00 00:00:02:00 01:00:00:00 01:00:02:00
* FROM CLIP NAME: A
002 AX A C 00:00:02:00 00:00:02:00 01:00:02:00 01:00:02:00
002 AX A D 024 00:00:02:00 00:00:04:00 01:00:02:00 01:00:04:00
* TO CLIP NAME: B
";
let opts = ImportOptions { edl_frame_rate: Some(FrameRate::FPS_24), ..Default::default() };
for (name, expected, warning) in [
(Some("CONSTANT GAIN"), "constant_gain", false),
(Some("EXPONENTIAL FADE"), "exponential_fade", false),
(Some("CONSTANT POWER"), "constant_power", false),
(Some("UNRECOGNIZED TRANSITION"), "constant_power", true),
(None, "constant_power", false),
] {
let text = format!("{fixture}{}", name.map(|name| format!("* EFFECT NAME: {name}\n")).unwrap_or_default());
let (imp, rep) = import_with(text.as_bytes(), Format::Edl, &opts).unwrap();
let seq = imp.project.sequence(only_seq(&imp)).unwrap();
let track = &seq.audio_tracks[0];
assert_eq!(track.transitions.len(), 1, "{name:?}");
let tr = &track.transitions[0];
assert_eq!(tr.effect.effect, expected, "{name:?}: {rep}");
assert_eq!(tr.start, f(FrameRate::FPS_24, 48), "{name:?}");
assert_eq!(tr.duration, f(FrameRate::FPS_24, 24), "{name:?}");
assert_eq!(tr.align, TransitionAlign::StartAtCut, "{name:?}");
assert!(tr.from.is_some() && tr.to.is_some(), "{name:?}");
assert_eq!(rep.mentions("not supported"), warning, "{name:?}: {rep}");
if warning {
assert!(rep.mentions("Constant Power"), "{rep}");
}
}
// The same event with video channels retains the video-only Cross Dissolve.
let video = fixture.replace("AX A", "AX V") + "* EFFECT NAME: CROSS DISSOLVE\n";
let (imp, rep) = import_with(video.as_bytes(), Format::Edl, &opts).unwrap();
let seq = imp.project.sequence(only_seq(&imp)).unwrap();
assert_eq!(seq.video_tracks[0].transitions[0].effect.effect, "cross_dissolve");
assert!(!rep.mentions("not supported"), "{rep}");
}
#[test]
fn explicit_rate_and_wipes_and_keys() {
let edl = "TITLE: Keys

57
crates/matroska/src/demux.rs
View File

@ -301,6 +301,9 @@ struct BlockRef {
discardable: bool,
invisible: bool,
discard_padding: Option<i64>,
/// (offset, size) of the `BlockAdditional` with `BlockAddID` 1 (WebM: the VP8/VP9 alpha
/// layer), kept for unlaced blocks only.
addition: Option<(u64, u32)>,
}
#[derive(Clone, Copy, Debug)]
@ -318,6 +321,7 @@ struct FrameRef {
discard_padding: Option<i64>,
block_offset: u64,
lace: u16,
addition: Option<(u64, u32)>,
}
fn track_lookup(tracks: &[Track]) -> impl Fn(u64) -> Option<usize> + '_ {
@ -354,6 +358,36 @@ fn parse_block<S: ByteSource + ?Sized>(
Ok(Some((track, rel, flags, frames)))
}
/// The `BlockAdditional` with `BlockAddID` 1 (the default ID) in a `BlockAdditions` element:
/// (offset, size) of its data. Other IDs, empty additions and damaged children are skipped.
fn block_addition<S: ByteSource + ?Sized>(io: &mut Io<'_, S>, start: u64, end: u64) -> Result<Option<(u64, u32)>> {
let mut pos = start;
while pos < end {
let Some(more) = io.header(pos, end)? else { break };
let me = more.end(end);
if more.id == BLOCK_MORE {
let (mut id, mut data) = (1u64, None);
let mut p = more.data_start();
while p < me {
let Some(c) = io.header(p, me)? else { break };
let ce = c.end(me);
let len = ce.saturating_sub(c.data_start());
match c.id {
BLOCK_ADD_ID => id = ebml::uint(io.peek(c.data_start(), len.min(8) as usize)?),
BLOCK_ADDITIONAL => data = u32::try_from(len).ok().filter(|&n| n > 0).map(|n| (c.data_start(), n)),
_ => {}
}
p = ce;
}
if id == 1 && data.is_some() {
return Ok(data);
}
}
pos = me;
}
Ok(None)
}
/// Parse one SimpleBlock or BlockGroup element.
fn parse_block_elem<S: ByteSource + ?Sized>(
io: &mut Io<'_, S>,
@ -376,17 +410,19 @@ fn parse_block_elem<S: ByteSource + ?Sized>(
discardable: flags & 0x01 != 0,
invisible: flags & 0x08 != 0,
discard_padding: None,
addition: None,
}))
}
BLOCK_GROUP => {
let mut pos = ds;
let mut block = None;
let (mut dur, mut has_ref, mut pad) = (None, false, None);
let (mut dur, mut has_ref, mut pad, mut addition) = (None, false, None, None);
while pos < end {
let Some(c) = io.header(pos, end)? else { break };
let ce = c.end(end);
match c.id {
BLOCK => block = parse_block(io, c.data_start(), ce - c.data_start(), lookup)?,
BLOCK_ADDITIONS => addition = block_addition(io, c.data_start(), ce)?,
BLOCK_DURATION => dur = Some(ebml::uint(io.peek(c.data_start(), (ce - c.data_start()) as usize)?)),
REFERENCE_BLOCK => has_ref = true,
DISCARD_PADDING => pad = Some(ebml::int(io.peek(c.data_start(), (ce - c.data_start()) as usize)?)),
@ -395,6 +431,8 @@ fn parse_block_elem<S: ByteSource + ?Sized>(
pos = ce;
}
let Some((track, rel, flags, frames)) = block else { return Ok(None) };
// with lacing, an addition belongs to the block, not to one of its frames
let addition = addition.filter(|_| frames.len() == 1);
Ok(Some(BlockRef {
track,
block_offset: h.pos,
@ -405,6 +443,7 @@ fn parse_block_elem<S: ByteSource + ?Sized>(
discardable: false,
invisible: flags & 0x08 != 0,
discard_padding: pad,
addition,
}))
}
_ => Ok(None),
@ -450,6 +489,7 @@ fn expand(b: &BlockRef, t: &Track, scale: u64, skip: u16, out: &mut impl Extend<
discard_padding: b.discard_padding,
block_offset: b.block_offset,
lace: i as u16,
addition: b.addition,
}
}));
}
@ -780,6 +820,7 @@ fn add_blocks(file: &mut MkvFile, cluster: u32, blocks: &[BlockRef]) {
cluster,
block_offset: f.block_offset,
lace: f.lace,
addition: f.addition,
});
}
}
@ -838,6 +879,20 @@ impl MkvFile {
Ok(data)
}
/// The `BlockAdditional` (ID 1) of sample `index` of `track`, `None` when the block has none
/// (see [`Sample::addition`]). Stored as is: header stripping applies to frames only.
pub fn read_sample_addition<S: ByteSource + ?Sized>(&self, src: &S, track: usize, index: usize) -> Result<Option<Vec<u8>>> {
let t = self.tracks.get(track).ok_or(Error::NoSuchTrack(track))?;
let s = t.samples.get(index).ok_or(Error::NoSuchSample(index))?;
let Some((offset, size)) = s.addition else { return Ok(None) };
if offset.checked_add(size as u64).is_none_or(|end| end > src.len()) {
return Err(Error::Invalid("block addition past end of file".into()));
}
let mut data = vec![0; size as usize];
src.read_at(offset, &mut data)?;
Ok(Some(data))
}
/// Keyframe sample of `track` with the greatest pts ≤ `time_ns` (indexed files only).
pub fn keyframe_before(&self, track: usize, time_ns: i64) -> Option<usize> {
let t = self.tracks.get(track)?;

3
crates/matroska/src/ids.rs
View File

@ -55,6 +55,9 @@ pub const ENCRYPTED_BLOCK: u32 = 0xAF;
pub const SILENT_TRACKS: u32 = 0x5854;
pub const BLOCK: u32 = 0xA1;
pub const BLOCK_ADDITIONS: u32 = 0x75A1;
pub const BLOCK_MORE: u32 = 0xA6;
pub const BLOCK_ADD_ID: u32 = 0xEE;
pub const BLOCK_ADDITIONAL: u32 = 0xA5;
pub const BLOCK_DURATION: u32 = 0x9B;
pub const REFERENCE_PRIORITY: u32 = 0xFA;
pub const REFERENCE_BLOCK: u32 = 0xFB;

4
crates/matroska/src/track.rs
View File

@ -191,6 +191,10 @@ pub struct Sample {
pub block_offset: u64,
/// Index of this frame within a laced block.
pub lace: u16,
/// (offset, size) of the block's `BlockAdditional` with `BlockAddID` 1, when it has one and
/// is not laced. In WebM this is the alpha layer of a VP8/VP9 track with `AlphaMode` 1: a
/// second, independently coded stream whose luma plane is the frame's alpha.
pub addition: Option<(u64, u32)>,
}
/// A Matroska track (`TrackEntry`) plus, once indexed, its sample table.

230
crates/media/src/reader.rs
View File

@ -2,8 +2,9 @@
//!
//! A [`ByteReader`] serves reads at any offset (a native file, a web `Blob` read in chunks, an
//! in-memory buffer). Container openers that only need the index and then single samples (MP4/MOV,
//! Matroska) open from a reader ([`ReaderOpener`]); formats decoded in one go (stills, WAV,
//! standalone compressed audio) fall back to reading the whole file through it.
//! Matroska) open from a reader ([`ReaderOpener`]), as does WAV (header now, sample ranges on
//! demand); formats decoded in one go (stills, standalone compressed audio) fall back to reading
//! the whole file through it.
use std::io;
use std::sync::Arc;
@ -43,24 +44,131 @@ impl ByteReader for MemReader {
}
}
/// A file on disk, read in place: the file stays open and only the requested ranges are read, so
/// opening a clip costs its index rather than its size.
/// Files [`FileReader`]s keep open at most, together (#67). Each open media source used to hold
/// its own descriptor, so a project with hundreds of sources hit the soft descriptor limit (256 on
/// macOS, 1024 on Linux) and the rest showed as failed or offline. Past this many, the least
/// recently read file is closed and its reader reopens it on demand.
#[cfg(any(unix, windows))]
pub const MAX_OPEN_FILES: usize = 128;
/// The open files of a set of [`FileReader`]s, least recently used first.
#[cfg(any(unix, windows))]
#[derive(Debug)]
pub(crate) struct FilePool {
cap: usize,
state: std::sync::Mutex<PoolState>,
}
#[cfg(any(unix, windows))]
#[derive(Debug, Default)]
struct PoolState {
next_id: u64,
open: std::collections::VecDeque<(u64, Arc<std::fs::File>)>,
}
#[cfg(any(unix, windows))]
impl FilePool {
pub(crate) fn new(cap: usize) -> Self {
Self { cap: cap.max(1), state: Default::default() }
}
fn state(&self) -> std::sync::MutexGuard<'_, PoolState> {
self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn next_id(&self) -> u64 {
let mut s = self.state();
s.next_id = s.next_id.wrapping_add(1);
s.next_id
}
/// The open file of reader `id`, now the most recently used.
fn get(&self, id: u64) -> Option<Arc<std::fs::File>> {
let mut s = self.state();
let at = s.open.iter().position(|(i, _)| *i == id)?;
let entry = s.open.remove(at)?;
let file = entry.1.clone();
s.open.push_back(entry);
Some(file)
}
/// Keep `file` open for reader `id`, closing the least recently used files past the cap. A
/// file another thread is reading from stays open until that read ends.
fn insert(&self, id: u64, file: std::fs::File) -> Arc<std::fs::File> {
let mut s = self.state();
if let Some((_, open)) = s.open.iter().find(|(i, _)| *i == id) {
return open.clone(); // another thread reopened it first
}
let file = Arc::new(file);
s.open.push_back((id, file.clone()));
while s.open.len() > self.cap {
s.open.pop_front();
}
file
}
fn remove(&self, id: u64) {
self.state().open.retain(|(i, _)| *i != id);
}
#[cfg(test)]
fn open_count(&self) -> usize {
self.state().open.len()
}
}
#[cfg(any(unix, windows))]
static FILE_POOL: std::sync::LazyLock<Arc<FilePool>> = std::sync::LazyLock::new(|| Arc::new(FilePool::new(MAX_OPEN_FILES)));
/// A file on disk, read in place: only the requested ranges are read, so opening a clip costs its
/// index rather than its size. The file is kept open in a shared pool of at most
/// [`MAX_OPEN_FILES`] and reopened by path when the pool closed it.
#[cfg(any(unix, windows))]
#[derive(Debug)]
pub struct FileReader {
file: std::fs::File,
path: std::path::PathBuf,
len: u64,
id: u64,
pool: Arc<FilePool>,
}
#[cfg(any(unix, windows))]
impl FileReader {
pub fn open(path: &std::path::Path) -> io::Result<Self> {
Self::open_in(path, &FILE_POOL)
}
pub(crate) fn open_in(path: &std::path::Path, pool: &Arc<FilePool>) -> io::Result<Self> {
let file = std::fs::File::open(path)?;
let m = file.metadata()?;
if !m.is_file() {
return Err(io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display())));
}
Ok(Self { file, len: m.len() })
let id = pool.next_id();
pool.insert(id, file);
Ok(Self { path: path.to_path_buf(), len: m.len(), id, pool: pool.clone() })
}
/// The open file, reopened when the pool closed it. A file now shorter than when it was opened
/// (replaced or truncated) is refused rather than read as the old one; a longer one (still
/// being written) is read up to the opened length, as before.
fn file(&self) -> io::Result<Arc<std::fs::File>> {
if let Some(file) = self.pool.get(self.id) {
return Ok(file);
}
let file = std::fs::File::open(&self.path)?;
let m = file.metadata()?;
if !m.is_file() || m.len() < self.len {
return Err(io::Error::other(format!("{} changed since it was opened", self.path.display())));
}
Ok(self.pool.insert(self.id, file))
}
}
#[cfg(any(unix, windows))]
impl Drop for FileReader {
fn drop(&mut self) {
self.pool.remove(self.id);
}
}
@ -71,15 +179,16 @@ impl ByteReader for FileReader {
}
/// Positional reads: threads decoding different parts of the file do not share a cursor.
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
let file = self.file()?;
#[cfg(unix)]
{
std::os::unix::fs::FileExt::read_exact_at(&self.file, buf, offset)
std::os::unix::fs::FileExt::read_exact_at(&*file, buf, offset)
}
#[cfg(windows)]
{
let (mut done, mut at) = (0usize, offset);
while let Some(rest) = buf.get_mut(done..).filter(|r| !r.is_empty()) {
match std::os::windows::fs::FileExt::seek_read(&self.file, rest, at) {
match std::os::windows::fs::FileExt::seek_read(&*file, rest, at) {
Ok(0) => return Err(eof()),
Ok(n) => {
done = done.saturating_add(n);
@ -109,15 +218,15 @@ pub type ReaderOpener = fn(name: &str, head: &[u8], reader: &SharedReader) -> Op
/// Bytes sniffed by [`open_reader`].
pub const HEAD_LEN: usize = 64 * 1024;
/// Open media from a reader: reader openers first; otherwise read the whole file and use the
/// byte openers (`extra`, then stills / WAV).
/// Open media from a reader: reader openers first, then WAV (read in place); otherwise read the
/// whole file and use the byte openers (`extra`, then stills).
pub fn open_reader(name: &str, reader: SharedReader, reader_openers: &[ReaderOpener], extra: &[Opener]) -> Result<SharedSource> {
open_reader_within(name, reader, reader_openers, extra, u64::MAX)
}
/// Like [`open_reader`], but a file no reader opener takes is read whole only up to `max_whole`
/// bytes; a larger one is refused instead. For looking at files rather than importing them (the
/// Media Browser's properties and thumbnails, #157): reading a multi-gigabyte AVI or WAV into
/// Media Browser's properties and thumbnails, #157): reading a multi-gigabyte AVI into
/// memory to show its duration, or to find out it isn't supported, thrashes the disk.
pub fn open_reader_within(name: &str, reader: SharedReader, reader_openers: &[ReaderOpener], extra: &[Opener], max_whole: u64) -> Result<SharedSource> {
let head = read_range(&*reader, 0, HEAD_LEN).map_err(|e| MediaError::Io(format!("{name}: {e}")))?;
@ -126,6 +235,9 @@ pub fn open_reader_within(name: &str, reader: SharedReader, reader_openers: &[Re
return r;
}
}
if crate::wav::sniff(&head) {
return Ok(Arc::new(crate::wav::WavSource::open(name, reader)?));
}
if reader.len() > max_whole {
return Err(MediaError::Unsupported(format!("{name}: no streaming reader for this format, and it is too large to read whole here")));
}
@ -169,6 +281,37 @@ mod tests {
std::fs::remove_dir_all(&dir).unwrap();
}
/// #67: more readers than the pool keeps open stay readable; past the cap the least recently
/// read file is closed and reopened on demand, and a file truncated meanwhile is refused.
#[cfg(any(unix, windows))]
#[test]
fn more_file_readers_than_the_pool_holds_stay_readable() {
let dir = std::env::temp_dir().join(format!("filmcraft-file-pool-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let pool = Arc::new(FilePool::new(2));
let paths: Vec<_> = (0..5).map(|i| dir.join(format!("{i}.bin"))).collect();
for (i, p) in paths.iter().enumerate() {
std::fs::write(p, format!("file{i}-0123456789")).unwrap();
}
let readers: Vec<_> = paths.iter().map(|p| FileReader::open_in(p, &pool).unwrap()).collect();
assert_eq!(pool.open_count(), 2);
for _ in 0..2 {
for (i, r) in readers.iter().enumerate() {
let mut b = [0u8; 5];
r.read_at(0, &mut b).unwrap();
assert_eq!(b, *format!("file{i}").as_bytes());
assert!(pool.open_count() <= 2);
}
}
// reader 0 was closed by the later reads; its file is now shorter than when opened
std::fs::write(&paths[0], b"short").unwrap();
let mut b = [0u8; 5];
assert!(readers[0].read_at(0, &mut b).is_err());
drop(readers);
assert_eq!(pool.open_count(), 0, "dropped readers close their files");
std::fs::remove_dir_all(&dir).unwrap();
}
/// A reader that counts the bytes read from it.
struct Counting(MemReader, std::sync::atomic::AtomicU64);
impl ByteReader for Counting {
@ -181,35 +324,58 @@ mod tests {
}
}
/// #157: looking at a file no streaming reader takes (an AVI, a large WAV) read all of it.
/// Within a limit it reads the head only and refuses; a small file still opens.
/// #157: looking at a file no streaming reader takes (an AVI) read all of it. Within a limit
/// it reads the head only and refuses.
#[test]
fn open_within_reads_only_the_head_of_large_unstreamable_files() {
let mut wav = crate::wav::write_wav16(&vec![0.25; 48_000 * 2], 2, 48_000);
let small = wav.len() as u64;
let r = Arc::new(Counting(MemReader(Arc::from(wav.clone())), Default::default()));
let src = open_reader_within("a.wav", r.clone(), &[], &[], small).unwrap();
assert_eq!(src.info().audio().unwrap().channels, 2);
// the same file over the limit: refused after reading the sniffing head only
let r = Arc::new(Counting(MemReader(Arc::from(wav.clone())), Default::default()));
let e = open_reader_within("a.wav", r.clone(), &[], &[], small - 1).err().unwrap();
let mut avi = vec![0u8; 4 * HEAD_LEN];
avi[..4].copy_from_slice(b"RIFF");
avi[8..12].copy_from_slice(b"AVI ");
let r = Arc::new(Counting(MemReader(Arc::from(avi)), Default::default()));
let e = open_reader_within("a.avi", r.clone(), &[], &[], HEAD_LEN as u64).err().unwrap();
assert!(matches!(e, MediaError::Unsupported(_)), "{e:?}");
assert!(r.1.load(std::sync::atomic::Ordering::SeqCst) <= HEAD_LEN as u64);
// an unsupported format is refused without being read whole either
wav.resize(4 * HEAD_LEN, 0);
wav[..4].copy_from_slice(b"RIFF");
wav[8..12].copy_from_slice(b"AVI ");
}
/// #279: a WAV is read in place, never whole: opening it reads the sniffing head and its chunk
/// headers, and audio reads only the requested samples, however large the file (and over the
/// Media Browser's whole-file limit too).
#[test]
fn wav_opens_in_place_and_reads_only_requested_samples() {
let rate = 48_000usize;
let samples: Vec<f32> = (0..rate * 2 * 20).map(|i| ((i / 2) % 100) as f32 / 200.0).collect();
let wav = crate::wav::write_wav16(&samples, 2, rate as u32);
let size = wav.len() as u64;
assert!(size > 3 * HEAD_LEN as u64);
let r = Arc::new(Counting(MemReader(Arc::from(wav)), Default::default()));
assert!(open_reader_within("a.avi", r.clone(), &[], &[], HEAD_LEN as u64).is_err());
assert!(r.1.load(std::sync::atomic::Ordering::SeqCst) <= HEAD_LEN as u64);
let read = || r.1.load(std::sync::atomic::Ordering::SeqCst);
let src = open_reader_within("a.wav", r.clone(), &[], &[], HEAD_LEN as u64).unwrap();
assert_eq!(src.info().audio().unwrap().channels, 2);
assert_eq!(src.info().duration, filmcraft_time::Tick::from_units(20, 1));
assert_eq!(src.info().file_size, Some(size));
let after_open = read();
assert!(after_open <= HEAD_LEN as u64 + 64, "{after_open}");
// 10 ms from the middle: about that much is read, and the samples are right
let a = src.audio(rate as i64 * 10 + 7, 480, rate as u32).unwrap();
assert!(read() - after_open <= 481 * 4, "{}", read() - after_open);
let frame = rate * 10 + 7 + 3;
assert!((a.channels[0][3] - samples[frame * 2]).abs() < 1e-4);
assert!((a.channels[1][3] - samples[frame * 2 + 1]).abs() < 1e-4);
// past the end: silence, nothing read
let before = read();
let a = src.audio(rate as i64 * 30, 16, rate as u32).unwrap();
assert!(a.channels[0].iter().all(|&s| s == 0.0));
assert_eq!(read(), before);
}
#[test]
fn falls_back_to_byte_openers() {
// a WAV is opened through the whole-file path
let wav = crate::wav::write_wav16(&[0.0, 0.5, -0.5, 0.25], 2, 48_000);
let r: SharedReader = Arc::new(MemReader(wav.into()));
let s = open_reader("a.wav", r, &[], &[]).unwrap();
assert!(s.info().has_audio());
// a format without a reader opener is read whole and handed to the byte openers
fn whole(_: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource>> {
Some(Err(MediaError::Unsupported(format!("got {} bytes", bytes.len()))))
}
let r: SharedReader = Arc::new(MemReader(Arc::from(&b"bytes"[..])));
let e = open_reader("a.xyz", r, &[], &[whole]).err().unwrap();
assert!(matches!(&e, MediaError::Unsupported(m) if m == "got 5 bytes"), "{e:?}");
}
}

182
crates/media/src/wav.rs
View File

@ -7,17 +7,32 @@ use std::sync::Arc;
use filmcraft_frame::{AudioBuffer, VideoFrame};
use filmcraft_time::Tick;
use crate::reader::{MemReader, SharedReader, read_range};
use crate::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, Result};
pub fn sniff(b: &[u8]) -> bool {
b.len() >= 12 && (&b[0..4] == b"RIFF" || &b[0..4] == b"RF64") && &b[8..12] == b"WAVE"
}
/// Chunks walked looking for `fmt `/`data`/`bext`/`iXML` (real files have a handful).
const MAX_CHUNKS: usize = 1 << 16;
/// Bytes of an iXML chunk searched for the time reference.
const IXML_MAX: u64 = 1 << 20;
/// Output frames converted per read of the data chunk.
const BLOCK_FRAMES: usize = 4096;
/// Largest single read of the data chunk; a wider span (an extreme resampling ratio) reads the
/// two frames each output frame needs instead.
const MAX_SPAN_BYTES: u64 = 4 << 20;
/// A WAV file read in place: opening it reads its chunk headers (not its samples), and
/// [`MediaSource::audio`] reads only the sample range it is asked for, so a 90-minute recording
/// costs its header rather than its size (#279).
pub struct WavSource {
info: MediaInfo,
bytes: Arc<[u8]>,
data_off: usize,
data_len: usize,
reader: SharedReader,
data_off: u64,
/// Whole frames only.
frames: u64,
channels: usize,
bits: u16,
float: bool,
@ -26,63 +41,80 @@ pub struct WavSource {
}
fn le16(b: &[u8], o: usize) -> u16 {
u16::from_le_bytes([b[o], b[o + 1]])
b.get(o..o.saturating_add(2)).and_then(|s| s.try_into().ok()).map_or(0, u16::from_le_bytes)
}
fn le32(b: &[u8], o: usize) -> u32 {
u32::from_le_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]])
b.get(o..o.saturating_add(4)).and_then(|s| s.try_into().ok()).map_or(0, u32::from_le_bytes)
}
impl WavSource {
/// Parse a WAV file held in memory.
pub fn parse(name: &str, bytes: Arc<[u8]>) -> Result<Self> {
let b = &bytes[..];
let mut pos = 12;
Self::open(name, Arc::new(MemReader(bytes)))
}
/// Open a WAV file through a random-access reader.
pub fn open(name: &str, reader: SharedReader) -> Result<Self> {
let io = |e: std::io::Error| MediaError::Io(format!("{name}: {e}"));
let len = reader.len();
let mut pos: u64 = 12;
let mut fmt = None;
let mut data = None;
let mut bext_tr = None;
let mut ixml_tr = None;
while pos + 8 <= b.len() {
let id = &b[pos..pos + 4];
let mut len = le32(b, pos + 4) as usize;
let body = pos + 8;
if id == b"data" && (len == 0xFFFF_FFFF || body + len > b.len()) {
len = b.len() - body;
for _ in 0..MAX_CHUNKS {
let Some(body) = pos.checked_add(8).filter(|&b| b <= len) else { break };
let mut hdr = [0u8; 8];
reader.read_at(pos, &mut hdr).map_err(io)?;
let id = &hdr[0..4];
let mut clen = u64::from(le32(&hdr, 4));
let rest = len - body;
if id == b"data" && (clen == 0xFFFF_FFFF || clen > rest) {
clen = rest;
}
if body + len > b.len() {
if clen > rest {
break;
}
match id {
b"fmt " if len >= 16 => {
let mut tag = le16(b, body);
let ch = le16(b, body + 2);
let rate = le32(b, body + 4);
let bits = le16(b, body + 14);
if tag == 0xFFFE && len >= 40 {
tag = le16(b, body + 24);
b"fmt " if clen >= 16 => {
let b = read_range(&*reader, body, clen.min(40) as usize).map_err(io)?;
let mut tag = le16(&b, 0);
let ch = le16(&b, 2);
let rate = le32(&b, 4);
let bits = le16(&b, 14);
if tag == 0xFFFE && clen >= 40 {
tag = le16(&b, 24);
}
fmt = Some((tag, ch, rate, bits));
}
b"data" => data = Some((body, len)),
b"data" => data = Some((body, clen)),
// bext: Description 256, Originator 32, OriginatorReference 32, OriginationDate 10,
// OriginationTime 8, then TimeReference (low u32, high u32)
b"bext" if len >= 346 => bext_tr = Some(le32(b, body + 338) as u64 | (le32(b, body + 342) as u64) << 32),
b"iXML" => ixml_tr = ixml_time_reference(&b[body..body + len]),
b"bext" if clen >= 346 => {
let b = read_range(&*reader, body + 338, 8).map_err(io)?;
bext_tr = Some(u64::from(le32(&b, 0)) | u64::from(le32(&b, 4)) << 32);
}
b"iXML" => {
let b = read_range(&*reader, body, clen.min(IXML_MAX) as usize).map_err(io)?;
ixml_tr = ixml_time_reference(&b);
}
_ => {}
}
pos = body + len + (len & 1);
pos = body + clen + (clen & 1);
}
let (tag, ch, rate, bits) = fmt.ok_or_else(|| MediaError::Decode(format!("{name}: missing fmt chunk")))?;
let (off, len) = data.ok_or_else(|| MediaError::Decode(format!("{name}: missing data chunk")))?;
let (off, dlen) = data.ok_or_else(|| MediaError::Decode(format!("{name}: missing data chunk")))?;
let float = tag == 3;
if !(tag == 1 || float) || ch == 0 || !matches!(bits, 8 | 16 | 24 | 32 | 64) {
return Err(MediaError::Unsupported(format!("{name}: WAV format tag {tag}, {bits} bits")));
}
let frame_bytes = ch as usize * bits as usize / 8;
let frames = len / frame_bytes;
let frame_bytes = u64::from(ch) * u64::from(bits / 8);
let frames = dlen / frame_bytes;
let time_reference = bext_tr.or(ixml_tr);
let mut info = MediaInfo {
name: name.into(),
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
duration: Tick::from_units(i64::try_from(frames).unwrap_or(i64::MAX), rate as i64),
video: None,
audio_streams: vec![AudioStreamInfo {
sample_rate: rate,
@ -92,16 +124,16 @@ impl WavSource {
}],
container: if time_reference.is_some() { "Broadcast WAV".into() } else { "WAV".into() },
start_timecode: None,
file_size: Some(b.len() as u64),
file_size: Some(len),
};
// start timecode: frames at the media's frame rate (the default rate for audio-only media)
if let Some(tr) = time_reference.filter(|_| rate > 0) {
let r = info.frame_rate();
let n = tr as i128 * r.num as i128;
let d = rate as i128 * r.den as i128;
info.start_timecode = Some(((n + d / 2) / d) as i64);
info.start_timecode = (n + d / 2).checked_div(d).and_then(|t| i64::try_from(t).ok());
}
Ok(Self { info, data_off: off, data_len: frames * frame_bytes, channels: ch as usize, bits, float, rate, bytes, time_reference })
Ok(Self { info, reader, data_off: off, frames, channels: ch as usize, bits, float, rate, time_reference })
}
/// Broadcast Wave start time: samples since midnight.
@ -109,17 +141,32 @@ impl WavSource {
self.time_reference
}
fn sample(&self, frame: usize, ch: usize) -> f32 {
fn frame_bytes(&self) -> usize {
self.channels * (self.bits as usize / 8)
}
/// Read whole frames `first..=last` of the data chunk.
fn read_frames(&self, first: u64, last: u64) -> Result<Vec<u8>> {
let fb = self.frame_bytes() as u64;
let n = last.saturating_sub(first).saturating_add(1).saturating_mul(fb);
let off = self.data_off.saturating_add(first.saturating_mul(fb));
let mut buf = vec![0u8; usize::try_from(n).map_err(|_| MediaError::Decode(format!("{}: audio read too large", self.info.name)))?];
self.reader.read_at(off, &mut buf).map_err(|e| MediaError::Io(format!("{}: {e}", self.info.name)))?;
Ok(buf)
}
/// Sample `ch` of frame `frame` of `block`, a run of whole frames.
fn sample(&self, block: &[u8], frame: usize, ch: usize) -> f32 {
let bps = self.bits as usize / 8;
let o = self.data_off + (frame * self.channels + ch) * bps;
let b = &self.bytes;
match (self.bits, self.float) {
(8, _) => (b[o] as f32 - 128.0) / 128.0,
(16, _) => i16::from_le_bytes([b[o], b[o + 1]]) as f32 / 32768.0,
(24, _) => ((i32::from_le_bytes([0, b[o], b[o + 1], b[o + 2]]) >> 8) as f32) / 8_388_608.0,
(32, false) => i32::from_le_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]]) as f32 / 2_147_483_648.0,
(32, true) => f32::from_le_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]]),
(64, true) => f64::from_le_bytes(b[o..o + 8].try_into().unwrap_or([0; 8])) as f32,
let o = (frame * self.channels + ch) * bps;
let Some(b) = block.get(o..o.saturating_add(bps)) else { return 0.0 };
match (self.bits, self.float, b) {
(8, _, &[a]) => (a as f32 - 128.0) / 128.0,
(16, _, &[a, b]) => i16::from_le_bytes([a, b]) as f32 / 32768.0,
(24, _, &[a, b, c]) => ((i32::from_le_bytes([0, a, b, c]) >> 8) as f32) / 8_388_608.0,
(32, false, &[a, b, c, d]) => i32::from_le_bytes([a, b, c, d]) as f32 / 2_147_483_648.0,
(32, true, &[a, b, c, d]) => f32::from_le_bytes([a, b, c, d]),
(64, true, b) => f64::from_le_bytes(b.try_into().unwrap_or([0; 8])) as f32,
_ => 0.0,
}
}
@ -133,25 +180,54 @@ impl MediaSource for WavSource {
Err(MediaError::NoStream("video"))
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer> {
let total = self.data_len / (self.channels * self.bits as usize / 8);
let total = self.frames;
let mut out = AudioBuffer::silence(sample_rate, self.channels, frames);
if total == 0 || sample_rate == 0 {
return Ok(out);
}
// Linear-interpolating resample when rates differ (the audio crate provides sinc resampling).
let ratio = self.rate as f64 / sample_rate as f64;
for i in 0..frames {
let src = (start + i as i64) as f64 * ratio;
if src < 0.0 {
let src_of = |i: usize| start.saturating_add(i as i64) as f64 * ratio;
let mut i = 0;
while i < frames {
let end = frames.min(i + BLOCK_FRAMES);
let (lo, hi) = (src_of(i), src_of(end - 1));
if hi < 0.0 {
i = end;
continue;
}
let i0 = src.floor() as usize;
if i0 >= total {
let first = lo.max(0.0).floor() as u64;
if first >= total {
break;
}
let fr = (src - i0 as f64) as f32;
for c in 0..self.channels {
let a = self.sample(i0, c);
let b = if i0 + 1 < total { self.sample(i0 + 1, c) } else { a };
out.channels[c][i] = a + (b - a) * fr;
let last = (hi.floor() as u64).saturating_add(1).min(total - 1);
let span = last.saturating_sub(first).saturating_add(1).saturating_mul(self.frame_bytes() as u64);
let block = if span <= MAX_SPAN_BYTES { Some(self.read_frames(first, last)?) } else { None };
for slot in i..end {
let src = src_of(slot);
if src < 0.0 {
continue;
}
let i0 = src.floor() as u64;
if i0 >= total {
break;
}
let i1 = (i0 + 1).min(total - 1);
let fr = (src - i0 as f64) as f32;
let (buf, base) = match &block {
Some(b) => (std::borrow::Cow::Borrowed(b.as_slice()), first),
None => (std::borrow::Cow::Owned(self.read_frames(i0, i1)?), i0),
};
let (k0, k1) = ((i0 - base) as usize, (i1 - base) as usize);
for c in 0..self.channels {
let a = self.sample(&buf, k0, c);
let b = self.sample(&buf, k1, c);
if let Some(s) = out.channels.get_mut(c).and_then(|ch| ch.get_mut(slot)) {
*s = a + (b - a) * fr;
}
}
}
i = end;
}
Ok(out)
}

3
crates/platform/src/hardware_encode.rs
View File

@ -33,6 +33,9 @@ pub fn config_for(w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Result
if s.bitrate_mode == BitrateMode::Vbr2Pass || !matches!(s.h264_pass, H264Pass::Single) {
return Err("two-pass encoding".into());
}
if s.bitrate_mode == BitrateMode::Crf {
return Err("CRF (constant quality)".into());
}
if s.signal.is_hdr() {
return Err("HDR".into());
}

4
crates/platform/src/nvenc/export.rs
View File

@ -162,6 +162,9 @@ fn config(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -
if !matches!(s.h264_pass, H264Pass::Single) || (hevc && s.bitrate_mode == BitrateMode::Vbr2Pass) {
return Err("two-pass VBR".into());
}
if s.bitrate_mode == BitrateMode::Crf {
return Err("CRF (constant quality)".into());
}
// the hardware path writes the aspect ratio of H.264 only
if hevc && s.pixel_aspect.is_some_and(|(n, d)| n != d) {
return Err("non-square pixels".into());
@ -357,6 +360,7 @@ mod tests {
("an unwritable colour signal", ExportSettings { signal: ColorSignal { primaries: 9, transfer: 1, matrix: 1 }, ..ok.clone() }),
("HDR with BT.709 primaries", ExportSettings { signal: ColorSignal { primaries: 1, transfer: 16, matrix: 9 }, ..ok.clone() }),
("two-pass", ExportSettings { bitrate_mode: BitrateMode::Vbr2Pass, ..ok.clone() }),
("CRF", ExportSettings { bitrate_mode: BitrateMode::Crf, ..ok.clone() }),
("analysis pass", ExportSettings { h264_pass: H264Pass::First, ..ok.clone() }),
("interlaced", ExportSettings { field_order: FieldOrder::UpperFirst, ..ok.clone() }),
("non-square pixels", ExportSettings { pixel_aspect: Some((4, 3)), ..ok.clone() }),

38
crates/platform/src/videotoolbox_encode.rs
View File

@ -19,7 +19,7 @@
//! for H.264, VPS / SPS / PPS and VideoToolbox's own `hvcC` record for HEVC) come with the first
//! compressed frame, and the muxer asks for them after the first group of frames.
use std::ffi::{c_int, c_void};
use std::ffi::{CStr, c_char, c_int, c_void};
use std::panic::AssertUnwindSafe;
use std::ptr::NonNull;
use std::sync::{Mutex, PoisonError};
@ -39,10 +39,10 @@ use objc2_core_video::{
use objc2_video_toolbox::{
VTCompressionSession, VTEncodeInfoFlags, VTSessionCopyProperty, VTSessionSetProperty, kVTCompressionPropertyKey_AllowFrameReordering,
kVTCompressionPropertyKey_AllowOpenGOP, kVTCompressionPropertyKey_AverageBitRate, kVTCompressionPropertyKey_ColorPrimaries,
kVTCompressionPropertyKey_ConstantBitRate, kVTCompressionPropertyKey_DataRateLimits, kVTCompressionPropertyKey_ExpectedFrameRate,
kVTCompressionPropertyKey_MaxKeyFrameInterval, kVTCompressionPropertyKey_ProfileLevel, kVTCompressionPropertyKey_RealTime,
kVTCompressionPropertyKey_TransferFunction, kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, kVTCompressionPropertyKey_YCbCrMatrix,
kVTProfileLevel_H264_Baseline_AutoLevel, kVTProfileLevel_H264_High_AutoLevel, kVTProfileLevel_H264_Main_AutoLevel, kVTProfileLevel_HEVC_Main_AutoLevel,
kVTCompressionPropertyKey_DataRateLimits, kVTCompressionPropertyKey_ExpectedFrameRate, kVTCompressionPropertyKey_MaxKeyFrameInterval,
kVTCompressionPropertyKey_ProfileLevel, kVTCompressionPropertyKey_RealTime, kVTCompressionPropertyKey_TransferFunction,
kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, kVTCompressionPropertyKey_YCbCrMatrix, kVTProfileLevel_H264_Baseline_AutoLevel,
kVTProfileLevel_H264_High_AutoLevel, kVTProfileLevel_H264_Main_AutoLevel, kVTProfileLevel_HEVC_Main_AutoLevel,
kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
};
@ -421,6 +421,28 @@ fn set_bool(session: &VTCompressionSession, key: &CFString, on: bool) -> i32 {
set_value(session, key, CFBoolean::new(on).as_ref())
}
unsafe extern "C" {
fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;
}
/// `kVTCompressionPropertyKey_ConstantBitRate`, or `None` on a macOS that does not have it.
///
/// The key exists from macOS 13. Naming it as a linked symbol makes dyld refuse to start the whole
/// app on macOS 11 and 12 ("Symbol not found"), although the encoder already falls back to an
/// average bitrate when the key is unavailable. So it is looked up at run time instead.
fn constant_bitrate_key() -> Option<&'static CFString> {
// `RTLD_DEFAULT` on macOS: search every image loaded into the process, VideoToolbox included.
const RTLD_DEFAULT: *mut c_void = -2isize as *mut c_void;
const NAME: &CStr = c"kVTCompressionPropertyKey_ConstantBitRate";
// SAFETY: `NAME` is a valid NUL-terminated string; `dlsym` only reads it.
let symbol = unsafe { dlsym(RTLD_DEFAULT, NAME.as_ptr()) };
let slot = NonNull::new(symbol)?.cast::<*const CFString>();
// SAFETY: the symbol is a `CFStringRef` constant exported by VideoToolbox, so `slot` points at a
// pointer to an immutable CFString that lives as long as the framework stays loaded (the whole
// process); it is read once and never written.
unsafe { slot.read().as_ref() }
}
fn set_value(session: &VTCompressionSession, key: &CFString, value: &CFType) -> i32 {
// SAFETY: the session is valid, `key` is a framework constant and `value` a live CF object.
unsafe { VTSessionSetProperty(session.as_ref(), key, Some(value)) }
@ -527,14 +549,14 @@ impl VtEncoder {
let _ = set_value(session, fps_key, CFNumber::new_f64(fps).as_ref());
// SAFETY: reading immutable framework constants.
let (avg_key, limits_key, cbr_key) =
unsafe { (kVTCompressionPropertyKey_AverageBitRate, kVTCompressionPropertyKey_DataRateLimits, kVTCompressionPropertyKey_ConstantBitRate) };
let (avg_key, limits_key) = unsafe { (kVTCompressionPropertyKey_AverageBitRate, kVTCompressionPropertyKey_DataRateLimits) };
let cbr_key = constant_bitrate_key();
let bps = |kbps: u32| i32::try_from(u64::from(kbps).saturating_mul(1000)).unwrap_or(i32::MAX);
match c.rate {
VtRate::Cbr { kbps } => {
// true constant bitrate where the OS offers it (macOS 13+); otherwise an average
// with a one-second ceiling at the same rate
if set_value(session, cbr_key, CFNumber::new_i32(bps(kbps)).as_ref()) != 0 {
if !cbr_key.is_some_and(|key| set_value(session, key, CFNumber::new_i32(bps(kbps)).as_ref()) == 0) {
require(set_value(session, avg_key, CFNumber::new_i32(bps(kbps)).as_ref()), "the bitrate")?;
let limits = CFArray::<CFType>::from_objects(&[CFNumber::new_i32(bps(kbps) / 8).as_ref(), CFNumber::new_f64(1.0).as_ref()]);
let _ = set_value(session, limits_key, limits.as_ref());

Some files were not shown because too many files have changed in this diff Show More