From d867178137b1ff2e7fbba0e2a8a9456590c8bc58 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 14:28:14 -0400 Subject: [PATCH] M6.8: media import via FilmCraft codecs (probe, frame cache, stills, sequences) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 722 +++++++++++++++++++++++++++++++++- Cargo.toml | 45 +++ crates/media/Cargo.toml | 25 ++ crates/media/src/convert.rs | 378 ++++++++++++++++++ crates/media/src/lib.rs | 65 +++ crates/media/src/pool.rs | 497 +++++++++++++++++++++++ crates/media/src/probe.rs | 168 ++++++++ crates/media/tests/footage.rs | 451 +++++++++++++++++++++ 8 files changed, 2347 insertions(+), 4 deletions(-) create mode 100644 crates/media/Cargo.toml create mode 100644 crates/media/src/convert.rs create mode 100644 crates/media/src/lib.rs create mode 100644 crates/media/src/pool.rs create mode 100644 crates/media/src/probe.rs create mode 100644 crates/media/tests/footage.rs diff --git a/Cargo.lock b/Cargo.lock index 1c73e8d9..3c19d7ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "arrayref" version = "0.3.9" @@ -35,6 +41,18 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.2" @@ -61,12 +79,39 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "cfg-if" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -92,6 +137,12 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "effectcraft-color" version = "0.1.0" @@ -132,6 +183,25 @@ dependencies = [ "serde_json", ] +[[package]] +name = "effectcraft-media" +version = "0.1.0" +dependencies = [ + "effectcraft-project", + "effectcraft-raster", + "effectcraft-render", + "effectcraft-time", + "filmcraft-codecs", + "filmcraft-color", + "filmcraft-frame", + "filmcraft-media", + "filmcraft-time", + "image", + "log", + "rayon", + "thiserror", +] + [[package]] name = "effectcraft-path" version = "0.1.0" @@ -209,6 +279,20 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "errno" version = "0.3.14" @@ -228,12 +312,220 @@ dependencies = [ "num-traits", ] +[[package]] +name = "exr" +version = "1.74.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711fe42c9964295e01ee3fba3f9fe0e1d24b98886950d68efe81b1c76e21adf3" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide 0.8.9", + "num-complex", + "pulp", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "filmcraft-aac" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "thiserror", +] + +[[package]] +name = "filmcraft-av1" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror", +] + +[[package]] +name = "filmcraft-bitstream" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" + +[[package]] +name = "filmcraft-codecs" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-aac", + "filmcraft-av1", + "filmcraft-bitstream", + "filmcraft-color", + "filmcraft-dnx", + "filmcraft-frame", + "filmcraft-h264", + "filmcraft-hevc", + "filmcraft-isobmff", + "filmcraft-matroska", + "filmcraft-media", + "filmcraft-opus", + "filmcraft-prores", + "filmcraft-time", + "filmcraft-vp9", + "image", + "log", + "symphonia", + "thiserror", +] + +[[package]] +name = "filmcraft-color" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "serde", +] + +[[package]] +name = "filmcraft-dnx" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror", +] + +[[package]] +name = "filmcraft-frame" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-color", + "filmcraft-time", + "rayon", + "serde", +] + +[[package]] +name = "filmcraft-h264" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror", +] + +[[package]] +name = "filmcraft-hevc" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror", +] + +[[package]] +name = "filmcraft-isobmff" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "thiserror", +] + +[[package]] +name = "filmcraft-matroska" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" + +[[package]] +name = "filmcraft-media" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-color", + "filmcraft-frame", + "filmcraft-time", + "image", + "rayon", + "serde", + "thiserror", +] + +[[package]] +name = "filmcraft-opus" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" + +[[package]] +name = "filmcraft-prores" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror", +] + +[[package]] +name = "filmcraft-time" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "serde", +] + +[[package]] +name = "filmcraft-vp9" +version = "0.1.0" +source = "git+https://github.com/storytold/filmcraft?rev=89cfbf24f698b0f0e538eb7795929f19bbfb1dbb#89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -272,18 +564,69 @@ dependencies = [ "r-efi 6.0.0", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "harfrust" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c03d949a14aa089bbb282f7dd76a498a7f684428e4257202efc119ec010376f9" dependencies = [ - "bitflags", + "bitflags 2.13.2", "bytemuck", "read-fonts", "smallvec", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error 2.0.1", +] + [[package]] name = "itoa" version = "1.0.18" @@ -303,12 +646,30 @@ dependencies = [ "smallvec", ] +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -327,6 +688,52 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -342,6 +749,25 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "polycool" version = "0.4.0" @@ -377,7 +803,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", - "bitflags", + "bitflags 2.13.2", "num-traits", "rand", "rand_chacha", @@ -388,12 +814,47 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quick-error" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.47" @@ -453,6 +914,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.2", +] + [[package]] name = "rayon" version = "1.12.0" @@ -484,6 +954,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -496,13 +972,19 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "rusty-fork" version = "0.3.1" @@ -510,11 +992,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" dependencies = [ "fnv", - "quick-error", + "quick-error 1.2.3", "tempfile", "wait-timeout", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.229" @@ -558,6 +1046,18 @@ dependencies = [ "zmij", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "skrifa" version = "0.44.0" @@ -583,6 +1083,164 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8413fa754942ac16a73634c9dfd1500ed5c61430956b33728567f667fdd393ab" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "2.0.119" @@ -638,6 +1296,20 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error 2.0.1", + "weezl", + "zune-jpeg", +] + [[package]] name = "tiny-skia" version = "0.12.0" @@ -687,6 +1359,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -705,6 +1383,12 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "windows-link" version = "0.2.1" @@ -746,8 +1430,38 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 3ac047d4..5743aa8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ effectcraft-project = { path = "crates/project" } effectcraft-text = { path = "crates/text" } effectcraft-effects = { path = "crates/effects" } effectcraft-render = { path = "crates/render" } +effectcraft-media = { path = "crates/media" } effectcraft-engine = { path = "crates/engine" } effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } @@ -41,12 +42,28 @@ skrifa = { version = "0.44", default-features = false, features = ["std"] } harfrust = { version = "0.12", default-features = false, features = ["std"] } image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +# media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). +# Only `crates/media` (and later `crates/export`) may depend on these. +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } + # ui egui = "0.36" eframe = { version = "0.36", default-features = false } egui_kittest = "0.36" rfd = "0.17" +# Local co-development against a FilmCraft checkout next to this repo: uncomment. +# [patch."https://github.com/storytold/filmcraft"] +# filmcraft-codecs = { path = "../filmcraft/crates/codecs" } +# filmcraft-media = { path = "../filmcraft/crates/media" } +# filmcraft-frame = { path = "../filmcraft/crates/frame" } +# filmcraft-color = { path = "../filmcraft/crates/color" } +# filmcraft-time = { path = "../filmcraft/crates/time" } + [workspace.lints.rust] unsafe_code = "deny" @@ -76,6 +93,34 @@ opt-level = 3 [profile.dev.package.effectcraft-path] opt-level = 3 +[profile.dev.package.effectcraft-media] +opt-level = 3 + +# FilmCraft codec hot loops (same settings as FilmCraft's own workspace). +[profile.dev.package.filmcraft-h264] +opt-level = 3 + +[profile.dev.package.filmcraft-hevc] +opt-level = 3 + +[profile.dev.package.filmcraft-vp9] +opt-level = 3 + +[profile.dev.package.filmcraft-prores] +opt-level = 3 + +[profile.dev.package.filmcraft-dnx] +opt-level = 3 + +[profile.dev.package.filmcraft-av1] +opt-level = 3 + +[profile.dev.package.filmcraft-aac] +opt-level = 3 + +[profile.dev.package.filmcraft-opus] +opt-level = 3 + [profile.release] lto = "thin" codegen-units = 1 diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml new file mode 100644 index 00000000..89992569 --- /dev/null +++ b/crates/media/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "effectcraft-media" +description = "Footage import: probing, stills, image sequences, video and audio via FilmCraft's pure-Rust codecs, memory-budgeted frame cache" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-time = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-render = { workspace = true } +filmcraft-codecs = { workspace = true } +filmcraft-media = { workspace = true } +filmcraft-frame = { workspace = true } +filmcraft-color = { workspace = true } +filmcraft-time = { workspace = true } +image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr"] } +rayon = { workspace = true } +thiserror = { workspace = true } +log = { workspace = true } diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs new file mode 100644 index 00000000..fa5df238 --- /dev/null +++ b/crates/media/src/convert.rs @@ -0,0 +1,378 @@ +//! Decoded pictures → compositor [`Image`]s (premultiplied f32 RGBA, sRGB/Rec.709-encoded values). +//! +//! Y'CbCr is converted with the matrix and range the stream signals (BT.601 / BT.709 / BT.2020, +//! limited or full), upsampling chroma bilinearly with centred siting (like ffmpeg's swscale, +//! which is also how most tools subsample when encoding). +//! Values stay gamma-encoded: like After Effects, footage pixels enter the working space as they +//! are encoded in the file. Alpha follows the footage's [`AlphaMode`] interpretation. + +use effectcraft_project::AlphaMode; +use effectcraft_raster::{Image, Px}; +use filmcraft_color::{Matrix, Range}; +use filmcraft_frame::{Chroma, PixelData, VideoFrame}; +use rayon::prelude::*; + +/// How to turn a (colour, alpha) sample into a premultiplied pixel. +#[derive(Clone, Copy)] +pub(crate) struct AlphaOp { + mode: AlphaMode, + matte: [f32; 3], +} + +impl AlphaOp { + pub(crate) fn new(mode: AlphaMode, matte: [f32; 3]) -> Self { + Self { mode, matte } + } + /// `c` is the file's colour (straight or premultiplied as the mode says), `a` its alpha. + #[inline(always)] + fn apply(self, c: [f32; 3], a: f32) -> Px { + match self.mode { + AlphaMode::Ignore => [c[0], c[1], c[2], 1.0], + AlphaMode::Straight => [c[0] * a, c[1] * a, c[2] * a, a], + AlphaMode::Premultiplied => { + // premultiplied against a matte colour m: c = s·a + m·(1−a) → s·a = c − m·(1−a) + let k = 1.0 - a; + let m = self.matte; + [(c[0] - m[0] * k).max(0.0), (c[1] - m[1] * k).max(0.0), (c[2] - m[2] * k).max(0.0), a] + } + } + } +} + +/// Convert a FilmCraft frame to an [`Image`]. +#[cfg(test)] +pub(crate) fn frame_to_image(f: &VideoFrame, op: AlphaOp) -> Image { + frame_to_image_in(f, op, Vec::new()) +} + +/// Convert a FilmCraft frame to an [`Image`], reusing `buf`'s allocation when it is large enough +/// (every pixel is overwritten, so a recycled frame's contents do not matter). +pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) -> Image { + let (w, h) = (f.width as usize, f.height as usize); + if buf.len() != w * h { + buf = vec![[0.0; 4]; w * h]; + } + let mut img = Image { width: f.width, height: f.height, data: buf }; + if w == 0 || h == 0 { + return img; + } + match &f.data { + PixelData::Rgba8(d) => { + img.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let src = &d[y * w * 4..(y + 1) * w * 4]; + for (o, s) in row.iter_mut().zip(src.chunks_exact(4)) { + *o = op.apply([U8[s[0] as usize], U8[s[1] as usize], U8[s[2] as usize]], U8[s[3] as usize]); + } + }); + } + PixelData::RgbaF32(d) => { + // linear premultiplied → encoded premultiplied + img.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let src = &d[y * w * 4..(y + 1) * w * 4]; + for (o, s) in row.iter_mut().zip(src.chunks_exact(4)) { + let a = s[3]; + let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; + let c = [encode(s[0] * inv) * a, encode(s[1] * inv) * a, encode(s[2] * inv) * a]; + *o = if op.mode == AlphaMode::Ignore { [c[0], c[1], c[2], 1.0] } else { [c[0], c[1], c[2], a] }; + } + }); + } + PixelData::Yuv8 { planes, chroma, alpha } => { + let yuv = Yuv::new(f.color.matrix, f.color.range, 8); + let amax = 255.0; + convert_yuv(&mut img, w, h, *chroma, [&planes[0][..], &planes[1][..], &planes[2][..]], alpha.as_deref().map(|a| &a[..]), amax, &yuv, op); + } + PixelData::Yuv16 { planes, chroma, bits, alpha } => { + let yuv = Yuv::new(f.color.matrix, f.color.range, *bits); + let amax = ((1u32 << *bits) - 1) as f32; + convert_yuv(&mut img, w, h, *chroma, [&planes[0][..], &planes[1][..], &planes[2][..]], alpha.as_deref().map(|a| &a[..]), amax, &yuv, op); + } + } + img +} + +/// 0..255 → 0..1. +static U8: [f32; 256] = { + let mut t = [0f32; 256]; + let mut i = 0; + while i < 256 { + t[i] = i as f32 / 255.0; + i += 1; + } + t +}; + +/// Linear → sRGB-encoded (sign-preserving, unclamped above 1 for float footage). +#[inline] +fn encode(v: f32) -> f32 { + if v < 0.0 { + -encode(-v) + } else if v <= 0.003_130_8 { + v * 12.92 + } else { + 1.055 * v.powf(1.0 / 2.4) - 0.055 + } +} + +/// Integer code values → normalised Y'CbCr and the matrix coefficients. +struct Yuv { + y_scale: f32, + y_off: f32, + c_scale: f32, + c_off: f32, + cr_r: f32, + cr_g: f32, + cb_g: f32, + cb_b: f32, +} + +impl Yuv { + fn new(m: Matrix, range: Range, bits: u32) -> Self { + let s = (1u32 << (bits - 8)) as f32; + let (y_scale, y_off, c_scale, c_off) = match range { + Range::Limited => (1.0 / (219.0 * s), 16.0 * s, 1.0 / (224.0 * s), 128.0 * s), + Range::Full => { + let max = ((1u32 << bits) - 1) as f32; + (1.0 / max, 0.0, 1.0 / max, (1u32 << (bits - 1)) as f32) + } + }; + let (kr, kb) = m.kr_kb(); + let kg = 1.0 - kr - kb; + let cr_r = 2.0 * (1.0 - kr); + let cb_b = 2.0 * (1.0 - kb); + Self { y_scale, y_off, c_scale, c_off, cr_r, cr_g: cr_r * kr / kg, cb_g: cb_b * kb / kg, cb_b } + } +} + +trait Sample: Copy + Send + Sync { + fn f(self) -> f32; +} +impl Sample for u8 { + #[inline(always)] + fn f(self) -> f32 { + self as f32 + } +} +impl Sample for u16 { + #[inline(always)] + fn f(self) -> f32 { + self as f32 + } +} + +/// Centred chroma siting (as ffmpeg's swscale and JPEG): for a subsampled axis, full-resolution +/// sample `i` lies ¼ of a chroma sample from chroma sample `i/2` towards its neighbour on that +/// side. Returns (nearest chroma index, neighbour index); the weights are ¾ and ¼. +#[inline(always)] +fn centred(i: usize, n: usize) -> (usize, usize) { + let c = (i >> 1).min(n - 1); + let o = if i & 1 == 0 { c.saturating_sub(1) } else { (c + 1).min(n - 1) }; + (c, o) +} + +#[allow(clippy::too_many_arguments)] +fn convert_yuv(img: &mut Image, w: usize, h: usize, chroma: Chroma, planes: [&[T]; 3], alpha: Option<&[T]>, amax: f32, k: &Yuv, op: AlphaOp) { + let (sx, sy) = chroma.shifts(); + let cw = w.div_ceil(1 << sx); + let ch = h.div_ceil(1 << sy); + let inv_amax = 1.0 / amax; + img.data.par_chunks_mut(w).enumerate().for_each_init( + || (vec![0f32; cw], vec![0f32; cw], vec![0f32; w], vec![0f32; w]), + |(uc, vc, uf, vf), (y, row)| { + // vertical chroma interpolation into normalised rows (Cb, Cr in −0.5..0.5) + let (r0, r1, t) = if sy == 1 { + let (c, o) = centred(y, ch); + (c, o, 0.25) + } else { + (y, y, 0.0) + }; + for (dst, plane) in [(&mut *uc, planes[1]), (&mut *vc, planes[2])] { + let a = &plane[r0 * cw..r0 * cw + cw]; + let b = &plane[r1 * cw..r1 * cw + cw]; + for ((d, a), b) in dst.iter_mut().zip(a).zip(b) { + let v = a.f() * (1.0 - t) + b.f() * t; + *d = (v - k.c_off) * k.c_scale; + } + } + // horizontal + if sx == 1 { + for x in 0..w { + let (c, o) = centred(x, cw); + uf[x] = uc[c] * 0.75 + uc[o] * 0.25; + vf[x] = vc[c] * 0.75 + vc[o] * 0.25; + } + } else { + uf.copy_from_slice(&uc[..w]); + vf.copy_from_slice(&vc[..w]); + } + let yrow = &planes[0][y * w..y * w + w]; + let rgb = |x: usize| { + let (u, v) = (uf[x], vf[x]); + let yy = (yrow[x].f() - k.y_off) * k.y_scale; + [(yy + k.cr_r * v).clamp(0.0, 1.0), (yy - k.cr_g * v - k.cb_g * u).clamp(0.0, 1.0), (yy + k.cb_b * u).clamp(0.0, 1.0)] + }; + match alpha { + // opaque: every alpha interpretation gives the colour itself + None => { + for (x, o) in row.iter_mut().enumerate() { + let c = rgb(x); + *o = [c[0], c[1], c[2], 1.0]; + } + } + Some(al) => { + let arow = &al[y * w..y * w + w]; + for (x, o) in row.iter_mut().enumerate() { + *o = op.apply(rgb(x), (arow[x].f() * inv_amax).clamp(0.0, 1.0)); + } + } + } + }, + ); +} + +/// Convert a still decoded by the `image` crate. Float images (OpenEXR, float TIFF) hold linear +/// light and are sRGB-encoded here (values above 1 are kept). +pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image { + use image::DynamicImage as D; + let (w, h) = (img.width(), img.height()); + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + match img { + D::ImageRgb32F(_) | D::ImageRgba32F(_) => { + let px = img.to_rgba32f(); + let raw = px.as_raw(); + let premul = op.mode == AlphaMode::Premultiplied; + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; + let a = s[3]; + let c = if premul { + let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; + [encode(s[0] * inv) * a, encode(s[1] * inv) * a, encode(s[2] * inv) * a] + } else { + [encode(s[0]), encode(s[1]), encode(s[2])] + }; + *o = op.apply(c, a.clamp(0.0, 1.0)); + } + }); + } + D::ImageLuma16(_) | D::ImageLumaA16(_) | D::ImageRgb16(_) | D::ImageRgba16(_) => { + let px = img.to_rgba16(); + let raw = px.as_raw(); + let k = 1.0 / 65535.0; + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; + *o = op.apply([s[0] as f32 * k, s[1] as f32 * k, s[2] as f32 * k], s[3] as f32 * k); + } + }); + } + _ => { + let px = img.to_rgba8(); + let raw = px.as_raw(); + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; + *o = op.apply([U8[s[0] as usize], U8[s[1] as usize], U8[s[2] as usize]], U8[s[3] as usize]); + } + }); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn yuv_frame(y: u8, u: u8, v: u8, color: filmcraft_color::ColorInfo) -> VideoFrame { + VideoFrame { + width: 4, + height: 2, + data: PixelData::Yuv8 { planes: [Arc::new(vec![y; 8]), Arc::new(vec![u; 2]), Arc::new(vec![v; 2])], chroma: Chroma::C420, alpha: None }, + color, + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + } + } + + #[test] + fn limited_range_grey_and_white() { + let op = AlphaOp::new(AlphaMode::Straight, [0.0; 3]); + let img = frame_to_image(&yuv_frame(235, 128, 128, filmcraft_color::ColorInfo::REC709), op); + assert!(img.data.iter().all(|p| (p[0] - 1.0).abs() < 1e-5 && (p[1] - 1.0).abs() < 1e-5 && p[3] == 1.0)); + let img = frame_to_image(&yuv_frame(126, 128, 128, filmcraft_color::ColorInfo::REC709), op); + assert!((img.data[0][0] - 110.0 / 219.0).abs() < 1e-5); + } + + #[test] + fn bt709_red() { + // 8-bit limited BT.709 red: Y 63, Cb 102, Cr 240 + let op = AlphaOp::new(AlphaMode::Straight, [0.0; 3]); + let p = frame_to_image(&yuv_frame(63, 102, 240, filmcraft_color::ColorInfo::REC709), op).data[0]; + assert!(p[0] > 0.98 && p[1] < 0.02 && p[2] < 0.02, "{p:?}"); + } + + #[test] + fn alpha_modes() { + let c = [0.8, 0.4, 0.2]; + let s = AlphaOp::new(AlphaMode::Straight, [0.0; 3]).apply(c, 0.5); + assert_eq!(s, [0.4, 0.2, 0.1, 0.5]); + let p = AlphaOp::new(AlphaMode::Premultiplied, [0.0; 3]).apply([0.4, 0.2, 0.1], 0.5); + assert_eq!(p, [0.4, 0.2, 0.1, 0.5]); + // premultiplied against white: c = s·a + 1·(1−a) + let p = AlphaOp::new(AlphaMode::Premultiplied, [1.0; 3]).apply([0.9, 0.7, 0.6], 0.5); + assert!((p[0] - 0.4).abs() < 1e-6 && (p[2] - 0.1).abs() < 1e-6); + let i = AlphaOp::new(AlphaMode::Ignore, [0.0; 3]).apply(c, 0.0); + assert_eq!(i, [0.8, 0.4, 0.2, 1.0]); + } + + #[test] + fn float_stills_are_encoded() { + let img = image::DynamicImage::ImageRgba32F(image::ImageBuffer::from_raw(1, 1, vec![0.2140, 1.0, 4.0, 1.0]).expect("buf")); + let p = dynamic_to_image(&img, AlphaOp::new(AlphaMode::Premultiplied, [0.0; 3])).data[0]; + assert!((p[0] - 0.5).abs() < 1e-3 && (p[1] - 1.0).abs() < 1e-5 && p[2] > 1.5, "{p:?}"); + } + + /// `cargo test -p effectcraft-media --lib bench -- --ignored --nocapture` + #[test] + #[ignore] + fn bench_convert_1080p() { + let (w, h) = (1920usize, 1080usize); + let f = VideoFrame { + width: w as u32, + height: h as u32, + data: PixelData::Yuv8 { + planes: [Arc::new((0..w * h).map(|i| (i % 251) as u8).collect()), Arc::new(vec![100; w * h / 4]), Arc::new(vec![150; w * h / 4])], + chroma: Chroma::C420, + alpha: None, + }, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + }; + let op = AlphaOp::new(AlphaMode::Straight, [0.0; 3]); + for _ in 0..3 { + let t = std::time::Instant::now(); + for _ in 0..20 { + std::hint::black_box(Image::new(1920, 1080)); + } + eprintln!("alloc {:?}", t.elapsed() / 20); + let t = std::time::Instant::now(); + for _ in 0..20 { + std::hint::black_box(frame_to_image(&f, op)); + } + eprintln!("convert {:?}", t.elapsed() / 20); + let mut keep = Vec::new(); + let t = std::time::Instant::now(); + for _ in 0..20 { + keep.push(frame_to_image(&f, op)); + } + eprintln!("convert into fresh memory {:?}", t.elapsed() / 20); + } + } +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs new file mode 100644 index 00000000..29546f7e --- /dev/null +++ b/crates/media/src/lib.rs @@ -0,0 +1,65 @@ +//! Footage import for EffectCraft (layer **L3**: it implements `effectcraft_render::FootageSource`). +//! +//! - [`probe`] / [`probe_bytes`]: inspect a file and describe it as an [`effectcraft_project::Footage`] +//! (video, still, image sequence or audio; size, rate, duration, streams, codec, alpha). +//! - [`MediaPool`]: decodes footage frames on demand into [`effectcraft_raster::Image`]s +//! (premultiplied f32, sRGB/Rec.709-encoded values, as the compositor expects) with a +//! memory-budgeted LRU frame cache, one decoder per source, and the decoder kept positioned for +//! sequential playback. Also audio ([`MediaPool::audio_samples`]) and thumbnails. +//! +//! Video containers (MP4/MOV, Matroska/WebM) and codecs (H.264, HEVC, VP9, AV1, ProRes, DNxHD, +//! MJPEG; AAC, Opus, PCM, MP3/FLAC/Vorbis) come from FilmCraft's pure-Rust crates (git dependency, +//! pinned; `plan/adr/0001`). FilmCraft types never leave this crate. Stills use the `image` crate +//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). + +mod convert; +mod pool; +mod probe; + +pub use pool::{DEFAULT_BUDGET, MediaPool, PoolStats}; +pub use probe::{DEFAULT_SEQUENCE_RATE, probe, probe_bytes, probe_single, sequence_files}; + +/// Errors from probing or decoding footage. +#[derive(Debug, thiserror::Error)] +pub enum MediaError { + #[error("I/O error: {0}")] + Io(String), + #[error("unsupported media: {0}")] + Unsupported(String), + #[error("decode error: {0}")] + Decode(String), +} + +pub type Result = std::result::Result; + +impl From for MediaError { + fn from(e: filmcraft_media::MediaError) -> Self { + use filmcraft_media::MediaError as F; + match e { + F::Unsupported(s) => MediaError::Unsupported(s), + F::Io(s) | F::Offline(s) => MediaError::Io(s), + other => MediaError::Decode(other.to_string()), + } + } +} + +/// File extensions recognised as stills. +pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"]; +/// File extensions recognised as audio-only files. +pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; +/// File extensions recognised as movies. +pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "m4v", "mov", "mkv", "webm"]; + +/// Whether `path` has an extension we can import. +pub fn is_importable(path: &std::path::Path) -> bool { + let ext = ext_of(path); + STILL_EXTENSIONS.contains(&ext.as_str()) || AUDIO_EXTENSIONS.contains(&ext.as_str()) || VIDEO_EXTENSIONS.contains(&ext.as_str()) +} + +pub(crate) fn ext_of(path: &std::path::Path) -> String { + path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase() +} + +pub(crate) fn rate_from_fc(r: filmcraft_time::FrameRate) -> effectcraft_time::FrameRate { + if r.num > 0 && r.den > 0 { effectcraft_time::FrameRate::new(r.num, r.den) } else { effectcraft_time::FrameRate::FPS_30 } +} diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs new file mode 100644 index 00000000..37787442 --- /dev/null +++ b/crates/media/src/pool.rs @@ -0,0 +1,497 @@ +//! [`MediaPool`]: decoded footage frames for the compositor, with a memory-budgeted LRU cache. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemId}; +use effectcraft_raster::{Image, Px}; +use effectcraft_render::FootageSource; +use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; +use filmcraft_media::{FrameRequest, SharedSource}; + +use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in}; +use crate::{MediaError, Result}; + +/// Default frame-cache budget: 1 GiB of decoded frames. +pub const DEFAULT_BUDGET: usize = 1 << 30; + +#[derive(Clone, PartialEq, Eq, Hash)] +struct Key { + path: Arc, + frame: i64, + alpha: u8, + matte: [u32; 3], +} + +struct Entry { + img: Arc, + stamp: u64, + bytes: usize, +} + +/// At most this many recycled pixel buffers are kept (outside the budget). +const MAX_SPARE: usize = 4; + +/// Least-recently-used frames, bounded by bytes. +#[derive(Default)] +struct Lru { + map: HashMap, + order: BTreeMap, + clock: u64, + bytes: usize, + /// Pixel buffers of evicted frames nobody else holds, reused for new frames: playback then + /// writes into memory that is already mapped instead of faulting in fresh pages every frame. + spare: Vec>, +} + +impl Lru { + fn get(&mut self, k: &Key) -> Option> { + let e = self.map.get_mut(k)?; + self.order.remove(&e.stamp); + self.clock += 1; + e.stamp = self.clock; + self.order.insert(e.stamp, k.clone()); + Some(e.img.clone()) + } + fn insert(&mut self, k: Key, img: Arc, budget: usize) { + let bytes = img.data.len() * std::mem::size_of::() + 64; + self.clock += 1; + if let Some(old) = self.map.insert(k.clone(), Entry { img, stamp: self.clock, bytes }) { + self.order.remove(&old.stamp); + self.bytes -= old.bytes; + } + self.order.insert(self.clock, k); + self.bytes += bytes; + self.evict(budget); + } + fn remove(&mut self, k: &Key) { + if let Some(e) = self.map.remove(k) { + self.order.remove(&e.stamp); + self.bytes -= e.bytes; + } + } + /// Evict least-recently-used frames until within `budget` (the newest frame always stays). + fn evict(&mut self, budget: usize) { + while self.bytes > budget && self.map.len() > 1 { + let Some((_, k)) = self.order.pop_first() else { break }; + if let Some(e) = self.map.remove(&k) { + self.bytes -= e.bytes; + if self.spare.len() < MAX_SPARE + && let Ok(img) = Arc::try_unwrap(e.img) + { + self.spare.push(img.data); + } + } + } + } + /// A recycled buffer of exactly `len` pixels, or an empty Vec. + fn take_spare(&mut self, len: usize) -> Vec { + match self.spare.iter().position(|b| b.len() == len) { + Some(i) => self.spare.swap_remove(i), + None => Vec::new(), + } + } +} + +/// Cache counters (see [`MediaPool::stats`]). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PoolStats { + /// Requests answered from the frame cache. + pub hits: u64, + /// Requests that had to decode (or wait for a read-ahead decode). + pub misses: u64, + /// Frames decoded ahead of sequential playback. + pub prefetched: u64, + /// Frames currently cached and their total size in bytes. + pub frames: usize, + pub bytes: usize, +} + +#[derive(Default)] +struct CacheState { + lru: Lru, + /// Frames being decoded right now (requests for them wait instead of decoding again). + inflight: HashSet, + /// Last frame index requested per movie (sequential-playback detection). + last: HashMap, i64>, +} + +struct Inner { + /// Opened movie/audio sources by path (`None`: failed to open; not retried until `forget`). + sources: Mutex, Option>>, + /// In-memory files registered with `add_bytes` (web builds, tests). + files: Mutex>>, + cache: Mutex, + done: Condvar, + budget: AtomicU64, + read_ahead: AtomicBool, + hits: AtomicU64, + misses: AtomicU64, + prefetched: AtomicU64, +} + +/// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus +/// audio. Thread-safe and cheap to clone (clones share decoders and cache). +/// +/// - One decoder per movie: FilmCraft's GOP cache keeps it positioned, so sequential frames +/// decode without re-seeking, and a seek decodes forward from the preceding sync sample. +/// - Converted frames live in an LRU cache bounded by a byte budget ([`DEFAULT_BUDGET`]). +/// - Sequential playback reads one frame ahead on a worker thread (native builds), so decoding +/// and conversion of the next frame overlap with the caller's use of the current one. +/// - Concurrent requests for the same frame decode it once. +#[derive(Clone)] +pub struct MediaPool { + inner: Arc, +} + +impl Default for MediaPool { + fn default() -> Self { + Self::new() + } +} + +/// Where a frame comes from. +struct Loc { + key: Key, + /// Media time for movies (`None`: an image file, `key.path`). + media_t: Option, +} + +/// Removes an in-flight key and wakes waiters, also when decoding panics. +struct Inflight<'a> { + inner: &'a Inner, + key: Key, +} + +impl Drop for Inflight<'_> { + fn drop(&mut self) { + lock(&self.inner.cache).inflight.remove(&self.key); + self.inner.done.notify_all(); + } +} + +impl MediaPool { + /// A pool with the [`DEFAULT_BUDGET`] frame cache. + pub fn new() -> Self { + Self::with_budget(DEFAULT_BUDGET) + } + + /// A pool whose frame cache holds at most `bytes` of decoded frames. + pub fn with_budget(bytes: usize) -> Self { + Self { + inner: Arc::new(Inner { + sources: Mutex::default(), + files: Mutex::default(), + cache: Mutex::default(), + done: Condvar::new(), + budget: AtomicU64::new(bytes as u64), + read_ahead: AtomicBool::new(cfg!(not(target_arch = "wasm32"))), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + prefetched: AtomicU64::new(0), + }), + } + } + + pub fn budget(&self) -> usize { + self.inner.budget() + } + + /// Change the frame-cache budget (evicts immediately when shrinking). + pub fn set_budget(&self, bytes: usize) { + self.inner.budget.store(bytes as u64, Ordering::Relaxed); + lock(&self.inner.cache).lru.evict(bytes); + } + + /// Enable or disable reading one frame ahead during sequential playback (on by default on + /// native builds; ignored on wasm32). + pub fn set_read_ahead(&self, on: bool) { + self.inner.read_ahead.store(on && cfg!(not(target_arch = "wasm32")), Ordering::Relaxed); + } + + pub fn stats(&self) -> PoolStats { + let i = &self.inner; + let c = lock(&i.cache); + PoolStats { + hits: i.hits.load(Ordering::Relaxed), + misses: i.misses.load(Ordering::Relaxed), + prefetched: i.prefetched.load(Ordering::Relaxed), + frames: c.lru.map.len(), + bytes: c.lru.bytes, + } + } + + /// Drop every cached frame and decoder. + pub fn clear(&self) { + let mut c = lock(&self.inner.cache); + c.lru = Lru::default(); + c.last.clear(); + drop(c); + lock(&self.inner.sources).clear(); + } + + /// Forget a file (after it changed on disk, or to retry a file that failed to open). + pub fn forget(&self, path: &str) { + lock(&self.inner.sources).remove(path); + let mut c = lock(&self.inner.cache); + let keys: Vec = c.lru.map.keys().filter(|k| &*k.path == path).cloned().collect(); + for k in keys { + c.lru.remove(&k); + } + c.last.remove(path); + } + + /// Provide the contents of `path` from memory (used instead of the file system; web builds). + pub fn add_bytes(&self, path: &str, bytes: Arc<[u8]>) { + lock(&self.inner.files).insert(path.to_string(), bytes); + self.forget(path); + } + + /// Decode (or fetch from cache) the frame of `footage` at source time `t`. + pub fn frame_at(&self, footage: &Footage, t: Tick) -> Result> { + Inner::frame_at(&self.inner, footage, t, true) + } + + /// `frames` stereo sample frames of `footage`'s audio starting at source time `start`, at + /// `rate` Hz, interleaved (L R L R …). Mono is duplicated to both channels; channels beyond + /// the first two are dropped. Silence where there is no audio (or before the media starts). + pub fn audio_samples(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> Vec { + let mut out = vec![0.0; frames * 2]; + if !footage.has_audio || footage.missing || rate == 0 || frames == 0 { + return out; + } + let path: Arc = footage.path.as_str().into(); + let Some(src) = self.inner.source(&path) else { return out }; + let s0 = start.to_units_floor(rate as i64); + let skip = (-s0).max(0) as usize; + if skip >= frames { + return out; + } + let buf = match src.audio(s0.max(0), frames - skip, rate) { + Ok(b) => b, + Err(e) => { + log::warn!("media: audio of {path}: {e}"); + return out; + } + }; + let chans = buf.channels.len(); + if chans == 0 { + return out; + } + let (l, r) = (&buf.channels[0], &buf.channels[if chans > 1 { 1 } else { 0 }]); + for i in 0..(frames - skip).min(l.len()).min(r.len()) { + out[(skip + i) * 2] = l[i]; + out[(skip + i) * 2 + 1] = r[i]; + } + out + } + + /// A thumbnail of `footage` (its first frame) fitting in `max_side` × `max_side`, aspect kept. + pub fn thumbnail(&self, footage: &Footage, max_side: u32) -> Result { + let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false)?; + let (w, h) = (img.width.max(1), img.height.max(1)); + let s = (max_side.max(1) as f64 / w.max(h) as f64).min(1.0); + if s >= 1.0 { + return Ok((*img).clone()); + } + let (tw, th) = (((w as f64 * s).round() as u32).max(1), ((h as f64 * s).round() as u32).max(1)); + Ok(effectcraft_raster::resample(&img, tw, th)) + } +} + +impl Inner { + fn budget(&self) -> usize { + self.budget.load(Ordering::Relaxed) as usize + } + + fn read(&self, path: &str) -> Result> { + if let Some(b) = lock(&self.files).get(path) { + return Ok(b.clone()); + } + std::fs::read(path).map(Into::into).map_err(|e| MediaError::Io(format!("{path}: {e}"))) + } + + /// The opened movie/audio source for `path`. + fn source(&self, path: &Arc) -> Option { + if let Some(s) = lock(&self.sources).get(path) { + return s.clone(); + } + // open outside the lock (reading and parsing a large file takes a moment) + let opened = self.read(path).and_then(|bytes| { + let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from) + }); + let opened = match opened { + Ok(s) => Some(s), + Err(e) => { + log::warn!("media: cannot open {path}: {e}"); + None + } + }; + lock(&self.sources).entry(path.clone()).or_insert(opened).clone() + } + + /// The frame index (in the footage's interpreted rate) shown at source time `t`, after + /// looping (`loop_count`) and clamping to the footage's frames; `n` = frames per loop. + fn frame_index(rate: FrameRate, t: Tick, n: i64, loops: u32) -> i64 { + let n = n.max(1); + let total = n.saturating_mul(loops.max(1) as i64); + rate.frame_at(t).clamp(0, total - 1) % n + } + + /// Number of frames in `d` at `rate` (rounded). + fn frame_count(rate: FrameRate, d: Tick) -> i64 { + let den = TICKS_PER_SECOND as i128 * rate.den as i128; + ((d.0 as i128 * rate.num as i128 + den / 2) / den) as i64 + } + + fn locate(footage: &Footage, t: Tick) -> Loc { + let alpha = match footage.alpha { + AlphaMode::Straight => 0, + AlphaMode::Premultiplied => 1, + AlphaMode::Ignore => 2, + }; + let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] }; + let key = |path: &str, frame| Key { path: path.into(), frame, alpha, matte }; + match footage.kind { + FootageKind::Sequence if !footage.sequence.is_empty() => { + let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); + Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } + } + FootageKind::Still | FootageKind::Sequence => Loc { key: key(&footage.path, 0), media_t: None }, + FootageKind::Video | FootageKind::Audio => { + let rate = footage.frame_rate; + let n = Self::frame_count(rate, footage.duration); + let i = Self::frame_index(rate, t, n, footage.loop_count); + // Middle of the frame in the file's own timing (robust to timestamp rounding). + let native = footage.native_rate.unwrap_or(rate); + let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 2); + Loc { key: key(&footage.path, i), media_t: Some(mt) } + } + } + } + + fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool) -> Result> { + if !footage.has_video { + return Err(MediaError::Unsupported(format!("{}: no video", footage.path))); + } + let loc = Self::locate(footage, t); + let movie = loc.media_t.is_some(); + let mut c = lock(&self.cache); + // sequential playback of a movie: read the next frame ahead + if playback && movie { + let prev = c.last.insert(loc.key.path.clone(), loc.key.frame); + if prev == Some(loc.key.frame - 1) && self.read_ahead.load(Ordering::Relaxed) { + let next_t = t + footage.frame_rate.frame_duration(); + let next = Self::locate(footage, next_t); + if next.key != loc.key && !c.lru.map.contains_key(&next.key) && !c.inflight.contains(&next.key) { + self.prefetch(footage, next_t); + } + } + } + loop { + if let Some(img) = c.lru.get(&loc.key) { + self.hits.fetch_add(1, Ordering::Relaxed); + return Ok(img); + } + if !c.inflight.contains(&loc.key) { + break; + } + // another thread (or the read-ahead) is decoding this frame + c = self.done.wait(c).unwrap_or_else(|e| e.into_inner()); + } + self.misses.fetch_add(1, Ordering::Relaxed); + c.inflight.insert(loc.key.clone()); + drop(c); + let _guard = Inflight { inner: self, key: loc.key.clone() }; + let img = Arc::new(self.decode(&loc, footage)?); + lock(&self.cache).lru.insert(loc.key, img.clone(), self.budget()); + Ok(img) + } + + #[cfg(not(target_arch = "wasm32"))] + fn prefetch(self: &Arc, footage: &Footage, t: Tick) { + let (me, f) = (self.clone(), footage.clone()); + self.prefetched.fetch_add(1, Ordering::Relaxed); + rayon::spawn(move || { + let _ = me.frame_at(&f, t, false); + }); + } + + #[cfg(target_arch = "wasm32")] + fn prefetch(self: &Arc, _footage: &Footage, _t: Tick) {} + + fn decode(&self, loc: &Loc, footage: &Footage) -> Result { + let op = AlphaOp::new(footage.alpha, footage.premul_color); + let path = &loc.key.path; + match loc.media_t { + None => { + let bytes = self.read(path)?; + let img = image::load_from_memory(&bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + Ok(dynamic_to_image(&img, op)) + } + Some(mt) => { + let src = self.source(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; + let vf = src.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))).map_err(MediaError::from)?; + let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize); + Ok(frame_to_image_in(&vf, op, buf)) + } + } + } +} + +impl FootageSource for MediaPool { + fn frame(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { + if footage.missing { + return None; + } + match self.frame_at(footage, t) { + Ok(img) => Some(img), + Err(e) => { + log::warn!("media: frame of {} at {:?}: {e}", footage.path, t); + None + } + } + } +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn frame_index_loops_and_clamps() { + let r = FrameRate::FPS_30; + assert_eq!(Inner::frame_index(r, r.tick_of(-3), 10, 1), 0); + assert_eq!(Inner::frame_index(r, r.tick_of(4), 10, 1), 4); + assert_eq!(Inner::frame_index(r, r.tick_of(25), 10, 1), 9); + assert_eq!(Inner::frame_index(r, r.tick_of(25), 10, 3), 5); + assert_eq!(Inner::frame_index(r, r.tick_of(45), 10, 3), 9); + assert_eq!(Inner::frame_count(r, Tick(2 * TICKS_PER_SECOND)), 60); + } + + #[test] + fn lru_respects_budget_and_recycles() { + let mut c = Lru::default(); + let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead + let k = |i| Key { path: "a".into(), frame: i, alpha: 0, matte: [0; 3] }; + let budget = 3 * (4096 + 64); + for i in 0..10 { + c.insert(k(i), img(), budget); + } + assert_eq!(c.map.len(), 3); + assert!(c.get(&k(9)).is_some() && c.get(&k(6)).is_none()); + // touching 7 makes 8 the oldest + c.get(&k(7)); + c.insert(k(10), img(), budget); + assert!(c.get(&k(7)).is_some() && c.get(&k(8)).is_none()); + assert_eq!(c.spare.len(), MAX_SPARE); + assert_eq!(c.take_spare(256).len(), 256); + assert!(c.take_spare(17).is_empty()); + } +} diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs new file mode 100644 index 00000000..a248cfc6 --- /dev/null +++ b/crates/media/src/probe.rs @@ -0,0 +1,168 @@ +//! Probing: describe a file as an [`effectcraft_project::Footage`]. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use effectcraft_project::{AlphaMode, Footage, FootageKind}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::{MediaError, Result, STILL_EXTENSIONS, ext_of, rate_from_fc}; + +/// Frame rate given to image sequences on import (After Effects' default preference). +pub const DEFAULT_SEQUENCE_RATE: FrameRate = FrameRate::FPS_30; + +/// Probe a file. Stills whose name ends in a frame number (`shot_0001.png`) with at least one +/// numbered sibling are imported as an image sequence; use [`probe_single`] to import one frame. +pub fn probe(path: impl AsRef) -> Result { + let path = path.as_ref(); + if STILL_EXTENSIONS.contains(&ext_of(path).as_str()) { + let files = sequence_files(path); + if files.len() >= 2 { + return probe_sequence(&files); + } + } + probe_single(path) +} + +/// Probe a file without image-sequence detection. +pub fn probe_single(path: impl AsRef) -> Result { + let path = path.as_ref(); + let bytes = std::fs::read(path).map_err(|e| MediaError::Io(format!("{}: {e}", path.display())))?; + probe_bytes(&path.to_string_lossy(), bytes.into()) +} + +/// Probe in-memory file contents; `path` names the footage (and is stored as `Footage::path`). +pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { + if let Ok(fmt) = image::guess_format(&bytes) { + return probe_still_bytes(path, &bytes, fmt); + } + let name = Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); + let src = filmcraft_codecs::open_bytes(&name, bytes)?; + Ok(footage_from_info(path, src.info())) +} + +pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) -> Footage { + let v = info.video.as_ref(); + let kind = match (info.kind, v) { + (filmcraft_media::MediaKind::Still, _) => FootageKind::Still, + (_, Some(_)) => FootageKind::Video, + (_, None) => FootageKind::Audio, + }; + let rate = v.map_or(FrameRate::FPS_30, |v| rate_from_fc(v.frame_rate)); + let codec = match (v, info.audio.as_ref()) { + (Some(v), _) => v.codec.clone(), + (None, Some(a)) => a.codec.clone(), + _ => String::new(), + }; + Footage { + path: path.to_string(), + kind, + width: v.map_or(0, |v| v.width), + height: v.map_or(0, |v| v.height), + pixel_aspect: v.map_or(1.0, |v| if v.par.1 > 0 { v.par.0 as f64 / v.par.1 as f64 } else { 1.0 }), + frame_rate: rate, + native_rate: None, + duration: if kind == FootageKind::Still { Tick::ZERO } else { Tick(info.duration.0) }, + has_video: v.is_some(), + has_audio: info.audio.is_some(), + alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, + premul_color: [0.0; 3], + loop_count: 1, + codec, + missing: false, + sequence: Vec::new(), + } +} + +fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: bool) -> Footage { + let codec = fmt.extensions_str().first().map_or("image", |s| s).to_ascii_uppercase(); + // OpenEXR stores premultiplied colour by convention; other formats straight alpha. + let alpha = match (has_alpha, fmt) { + (false, _) => AlphaMode::Ignore, + (true, image::ImageFormat::OpenExr) => AlphaMode::Premultiplied, + (true, _) => AlphaMode::Straight, + }; + Footage { + path: path.to_string(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: DEFAULT_SEQUENCE_RATE, + native_rate: None, + duration: Tick::ZERO, + has_video: true, + has_audio: false, + alpha, + premul_color: [0.0; 3], + loop_count: 1, + codec, + missing: false, + sequence: Vec::new(), + } +} + +fn probe_still_bytes(path: &str, bytes: &[u8], fmt: image::ImageFormat) -> Result { + use image::ImageDecoder; + let reader = image::ImageReader::with_format(std::io::Cursor::new(bytes), fmt); + let dec = reader.into_decoder().map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = dec.dimensions(); + let has_alpha = dec.color_type().has_alpha(); + Ok(still_footage(path, w, h, fmt, has_alpha)) +} + +fn probe_sequence(files: &[PathBuf]) -> Result { + let first = &files[0]; + let mut f = probe_single(first)?; + f.kind = FootageKind::Sequence; + f.frame_rate = DEFAULT_SEQUENCE_RATE; + f.duration = DEFAULT_SEQUENCE_RATE.tick_of(files.len() as i64); + f.sequence = files.iter().map(|p| p.to_string_lossy().to_string()).collect(); + Ok(f) +} + +/// Split a file stem into (prefix, frame number digits). `None` when it does not end in digits. +fn split_number(stem: &str) -> Option<(&str, &str)> { + let digits = stem.bytes().rev().take_while(u8::is_ascii_digit).count(); + (digits > 0).then(|| stem.split_at(stem.len() - digits)) +} + +/// The numbered files of the image sequence `path` belongs to (sorted by frame number; empty when +/// the name has no frame number). Siblings must share the prefix and extension. +pub fn sequence_files(path: impl AsRef) -> Vec { + let path = path.as_ref(); + let ext = ext_of(path); + let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else { return Vec::new() }; + let Some((prefix, _)) = split_number(stem) else { return Vec::new() }; + let dir = path.parent().filter(|d| !d.as_os_str().is_empty()).unwrap_or(Path::new(".")); + let Ok(rd) = std::fs::read_dir(dir) else { return Vec::new() }; + let mut out: Vec<(u64, PathBuf)> = rd + .filter_map(|e| e.ok()) + .filter_map(|e| { + let p = e.path(); + if ext_of(&p) != ext || !p.is_file() { + return None; + } + let s = p.file_stem()?.to_str()?; + let (pre, num) = split_number(s)?; + if pre != prefix { + return None; + } + Some((num.parse().ok()?, p)) + }) + .collect(); + out.sort(); + out.into_iter().map(|(_, p)| p).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_numbers() { + assert_eq!(split_number("shot_0012"), Some(("shot_", "0012"))); + assert_eq!(split_number("shot"), None); + assert_eq!(split_number("v2_10"), Some(("v2_", "10"))); + } +} diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rs new file mode 100644 index 00000000..38fa7cc7 --- /dev/null +++ b/crates/media/tests/footage.rs @@ -0,0 +1,451 @@ +//! Footage import against ffmpeg as an external oracle. +//! +//! Fixtures are generated with ffmpeg into `target/fixtures/media` (never committed); every test +//! skips (passes with a note) when ffmpeg is not installed. Decoded frames are compared with +//! frames ffmpeg extracts to PNG (PSNR, premultiplied RGBA). + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use std::time::Instant; + +use effectcraft_media::{MediaPool, probe, probe_single}; +use effectcraft_project::{AlphaMode, FootageKind, ItemId}; +use effectcraft_raster::Image; +use effectcraft_render::FootageSource; +use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; + +fn fixtures() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/media") +} + +fn have_ffmpeg() -> bool { + static HAVE: std::sync::OnceLock = std::sync::OnceLock::new(); + *HAVE.get_or_init(|| Command::new("ffmpeg").arg("-version").output().is_ok_and(|o| o.status.success())) +} + +/// Run ffmpeg to produce `name` (once; written to a temp name and renamed so concurrent test +/// processes never see a partial file). `None` when ffmpeg is unavailable. +fn ffmpeg_fixture(name: &str, args: &[&str]) -> Option { + if !have_ffmpeg() { + eprintln!("ffmpeg not found: skipping"); + return None; + } + let out = fixtures().join(name); + if out.exists() { + return Some(out); + } + std::fs::create_dir_all(out.parent().expect("parent")).expect("mkdir"); + let ext = out.extension().map(|e| e.to_string_lossy().to_string()).unwrap_or_default(); + let tmp = out.with_file_name(format!("tmp-{}-{:?}.{ext}", std::process::id(), std::thread::current().id()).replace(['(', ')'], "")); + let st = Command::new("ffmpeg").args(["-hide_banner", "-loglevel", "error", "-y"]).args(args).arg(&tmp).status().expect("run ffmpeg"); + assert!(st.success(), "ffmpeg failed for {name}"); + std::fs::rename(&tmp, &out).expect("rename"); + Some(out) +} + +fn h264_mp4() -> Option { + ffmpeg_fixture( + "testsrc2_1080p.mp4", + &[ + "-f", + "lavfi", + "-i", + "testsrc2=size=1920x1080:rate=30:duration=2", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=2", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-preset", + "medium", + "-crf", + "18", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-c:a", + "aac", + "-ac", + "2", + "-shortest", + ], + ) +} + +fn prores_mov() -> Option { + ffmpeg_fixture( + "prores4444_alpha.mov", + &[ + "-f", + "lavfi", + "-i", + "testsrc2=size=640x360:rate=24:duration=1,format=rgba,geq=r='r(X,Y)':g='g(X,Y)':b='b(X,Y)':a='255*X/W'", + "-c:v", + "prores_ks", + "-profile:v", + "4444", + "-pix_fmt", + "yuva444p10le", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + ], + ) +} + +fn vp9_webm() -> Option { + ffmpeg_fixture( + "testsrc2_vp9.webm", + &[ + "-f", + "lavfi", + "-i", + "testsrc2=size=640x360:rate=25:duration=1", + "-c:v", + "libvpx-vp9", + "-pix_fmt", + "yuv420p", + "-crf", + "20", + "-b:v", + "0", + "-deadline", + "good", + "-cpu-used", + "4", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + ], + ) +} + +fn png_sequence() -> Option { + // ffmpeg's image2 muxer writes seq_0001.png … seq_0010.png; generate into a temp dir and rename it. + if !have_ffmpeg() { + return None; + } + let dir = fixtures().join("seq"); + if !dir.join("seq_0010.png").exists() { + let tmp = fixtures().join(format!("seq-tmp-{}-{:?}", std::process::id(), std::thread::current().id()).replace(['(', ')'], "")); + std::fs::create_dir_all(&tmp).expect("mkdir"); + let st = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=30:duration=1", "-frames:v", "10"]) + .arg(tmp.join("seq_%04d.png")) + .status() + .expect("ffmpeg"); + assert!(st.success()); + if std::fs::rename(&tmp, &dir).is_err() { + let _ = std::fs::remove_dir_all(&tmp); // another process won the race + } + } + Some(dir.join("seq_0001.png")) +} + +fn alpha_png() -> Option { + ffmpeg_fixture( + "alpha.png", + &["-f", "lavfi", "-i", "testsrc2=size=256x128:rate=1:duration=1,format=rgba,geq=r='r(X,Y)':g='g(X,Y)':b='b(X,Y)':a='255*Y/H'", "-frames:v", "1"], + ) +} + +fn tone_wav() -> Option { + ffmpeg_fixture("tone.wav", &["-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=48000:duration=1", "-ac", "2", "-c:a", "pcm_s16le"]) +} + +/// Frame `n` of `src` as RGBA PNG extracted by ffmpeg (Y'CbCr → RGB with the BT.709 matrix). +fn reference_frame(src: &Path, n: usize) -> Image { + let stem = src.file_stem().expect("stem").to_string_lossy().to_string(); + let name = format!("ref_{stem}_{n}.png"); + let vf = format!("select='eq(n\\,{n})',scale=in_color_matrix=bt709:in_range=auto:out_range=pc:flags=bilinear+accurate_rnd+full_chroma_int,format=rgba"); + let p = ffmpeg_fixture(&name, &["-i", src.to_str().expect("utf8"), "-vf", &vf, "-fps_mode", "passthrough", "-frames:v", "1"]).expect("ffmpeg"); + load_png(&p) +} + +/// A PNG as a premultiplied image (straight alpha in the file). +fn load_png(p: &Path) -> Image { + let img = image::open(p).expect("png").to_rgba8(); + let (w, h) = img.dimensions(); + let data = img + .pixels() + .map(|p| { + let a = p[3] as f32 / 255.0; + [p[0] as f32 / 255.0 * a, p[1] as f32 / 255.0 * a, p[2] as f32 / 255.0 * a, a] + }) + .collect(); + Image { width: w, height: h, data } +} + +fn psnr(a: &Image, b: &Image) -> f64 { + assert_eq!((a.width, a.height), (b.width, b.height), "size"); + let mut se = 0f64; + for (p, q) in a.data.iter().zip(&b.data) { + for k in 0..4 { + let d = (p[k].clamp(0.0, 1.0) - q[k]) as f64; + se += d * d; + } + } + let mse = se / (a.data.len() * 4) as f64; + if mse == 0.0 { f64::INFINITY } else { 10.0 * (1.0 / mse).log10() } +} + +fn t_of(rate: FrameRate, n: i64) -> Tick { + rate.tick_of(n) +} + +#[test] +fn h264_mp4_probe_and_frames() { + let Some(p) = h264_mp4() else { return }; + let f = probe(&p).expect("probe"); + assert_eq!(f.kind, FootageKind::Video); + assert_eq!((f.width, f.height), (1920, 1080)); + assert_eq!(f.frame_rate, FrameRate::FPS_30); + assert!((f.duration.seconds() - 2.0).abs() < 0.1, "duration {:?}", f.duration.seconds()); + assert!(f.has_video && f.has_audio); + assert_eq!(f.alpha, AlphaMode::Ignore); + assert!(!f.codec.is_empty()); + eprintln!("h264: codec {:?}", f.codec); + let pool = MediaPool::new(); + // sequential, then a backwards seek and a forward jump into the GOP + for n in [0usize, 1, 2, 3, 45, 10, 59] { + let img = pool.frame(ItemId(1), &f, t_of(f.frame_rate, n as i64)).expect("frame"); + let db = psnr(&img, &reference_frame(&p, n)); + eprintln!("h264 frame {n}: {db:.2} dB"); + assert!(db > 35.0, "frame {n}: {db:.2} dB"); + } + // clamped past the end: last frame + let last = pool.frame_at(&f, Tick(10 * TICKS_PER_SECOND)).expect("frame"); + let db = psnr(&last, &reference_frame(&p, 59)); + assert!(db > 35.0, "clamp: {db:.2} dB"); + // cached + let a = pool.frame_at(&f, t_of(f.frame_rate, 45)).expect("frame"); + let b = pool.frame_at(&f, t_of(f.frame_rate, 45)).expect("frame"); + assert!(Arc::ptr_eq(&a, &b)); +} + +#[test] +fn h264_audio_samples() { + let Some(p) = h264_mp4() else { return }; + let f = probe(&p).expect("probe"); + let pool = MediaPool::new(); + let s = pool.audio_samples(&f, Tick(TICKS_PER_SECOND / 2), 4800, 48_000); + assert_eq!(s.len(), 9600); + let rms = (s.iter().map(|v| v * v).sum::() / s.len() as f32).sqrt(); + // ffmpeg's sine source: −24 dBFS RMS + assert!((rms - 0.0625).abs() < 0.01, "rms {rms}"); + // before the start: silence + let pre = pool.audio_samples(&f, Tick(-TICKS_PER_SECOND), 100, 48_000); + assert!(pre.iter().all(|v| *v == 0.0)); +} + +#[test] +fn prores_4444_with_alpha() { + let Some(p) = prores_mov() else { return }; + let f = probe(&p).expect("probe"); + assert_eq!(f.kind, FootageKind::Video); + assert_eq!((f.width, f.height), (640, 360)); + assert_eq!(f.frame_rate, FrameRate::FPS_24); + assert!(!f.has_audio); + assert_eq!(f.alpha, AlphaMode::Straight, "ProRes 4444 alpha detected"); + eprintln!("prores: codec {:?}", f.codec); + let pool = MediaPool::new(); + for n in [0usize, 7, 23] { + let img = pool.frame_at(&f, t_of(f.frame_rate, n as i64)).expect("frame"); + let db = psnr(&img, &reference_frame(&p, n)); + eprintln!("prores frame {n}: {db:.2} dB"); + assert!(db > 35.0, "frame {n}: {db:.2} dB"); + } + // alpha ramps left to right + let img = pool.frame_at(&f, Tick::ZERO).expect("frame"); + assert!(img.data[0][3] < 0.02 && img.data[639][3] > 0.97); + // Ignore → opaque + let mut g = f.clone(); + g.alpha = AlphaMode::Ignore; + assert!(pool.frame_at(&g, Tick::ZERO).expect("frame").data.iter().all(|p| p[3] == 1.0)); +} + +#[test] +fn vp9_webm_frames() { + let Some(p) = vp9_webm() else { return }; + let f = probe(&p).expect("probe"); + assert_eq!(f.kind, FootageKind::Video); + assert_eq!((f.width, f.height), (640, 360)); + assert_eq!(f.frame_rate, FrameRate::FPS_25); + eprintln!("vp9: codec {:?}", f.codec); + let pool = MediaPool::new(); + for n in [0usize, 1, 12, 24, 5] { + let img = pool.frame_at(&f, t_of(f.frame_rate, n as i64)).expect("frame"); + let db = psnr(&img, &reference_frame(&p, n)); + eprintln!("vp9 frame {n}: {db:.2} dB"); + assert!(db > 35.0, "frame {n}: {db:.2} dB"); + } +} + +#[test] +fn png_sequence_import() { + let Some(first) = png_sequence() else { return }; + let f = probe(&first).expect("probe"); + assert_eq!(f.kind, FootageKind::Sequence); + assert_eq!(f.sequence.len(), 10); + assert_eq!((f.width, f.height), (320, 240)); + assert_eq!(f.duration, FrameRate::FPS_30.tick_of(10)); + let pool = MediaPool::new(); + let img = pool.frame_at(&f, t_of(f.frame_rate, 3)).expect("frame"); + assert_eq!(*img, load_png(&first.with_file_name("seq_0004.png"))); + // past the end clamps to the last file; with 2 loops it wraps + let img = pool.frame_at(&f, t_of(f.frame_rate, 13)).expect("frame"); + assert_eq!(*img, load_png(&first.with_file_name("seq_0010.png"))); + let mut looped = f.clone(); + looped.loop_count = 2; + let img = pool.frame_at(&looped, t_of(f.frame_rate, 13)).expect("frame"); + assert_eq!(*img, load_png(&first.with_file_name("seq_0004.png"))); + // a single frame of it + let one = probe_single(&first).expect("probe"); + assert_eq!(one.kind, FootageKind::Still); +} + +#[test] +fn png_with_alpha() { + let Some(p) = alpha_png() else { return }; + let f = probe(&p).expect("probe"); + assert_eq!(f.kind, FootageKind::Still); + assert_eq!((f.width, f.height), (256, 128)); + assert_eq!(f.alpha, AlphaMode::Straight); + assert_eq!(f.codec, "PNG"); + let pool = MediaPool::new(); + let a = pool.frame_at(&f, Tick::ZERO).expect("frame"); + let b = pool.frame_at(&f, Tick(5 * TICKS_PER_SECOND)).expect("frame"); + assert!(Arc::ptr_eq(&a, &b), "stills are timeless"); + assert_eq!(*a, load_png(&p)); + assert!(a.data[0][3] == 0.0 && a.data[a.data.len() - 1][3] > 0.98); + let th = pool.thumbnail(&f, 64).expect("thumb"); + assert_eq!((th.width, th.height), (64, 32)); +} + +#[test] +fn wav_audio() { + let Some(p) = tone_wav() else { return }; + let f = probe(&p).expect("probe"); + assert_eq!(f.kind, FootageKind::Audio); + assert!(f.has_audio && !f.has_video); + assert!((f.duration.seconds() - 1.0).abs() < 0.01); + let pool = MediaPool::new(); + assert!(pool.frame(ItemId(1), &f, Tick::ZERO).is_none()); + let s = pool.audio_samples(&f, Tick::ZERO, 48_000, 48_000); + let rms = (s.iter().map(|v| v * v).sum::() / s.len() as f32).sqrt(); + assert!((rms - 0.0625).abs() < 0.005, "rms {rms}"); + // resampled to 44.1 kHz: same loudness + let s = pool.audio_samples(&f, Tick::ZERO, 44_100, 44_100); + let rms = (s.iter().map(|v| v * v).sum::() / s.len() as f32).sqrt(); + assert!((rms - 0.0625).abs() < 0.01, "rms {rms}"); +} + +#[test] +fn cache_budget_evicts() { + let Some(p) = vp9_webm() else { return }; + let f = probe(&p).expect("probe"); + let frame_bytes = 640 * 360 * 16; + let pool = MediaPool::with_budget(3 * frame_bytes + 1024); + for n in 0..10 { + pool.frame_at(&f, t_of(f.frame_rate, n)).expect("frame"); + } + let st = pool.stats(); + assert!(st.frames <= 3 && st.bytes <= 3 * frame_bytes + 1024, "{st:?}"); + pool.set_budget(0); + assert_eq!(pool.stats().frames, 1); +} + +#[test] +fn concurrent_requests() { + let Some(p) = h264_mp4() else { return }; + let f = probe(&p).expect("probe"); + let pool = MediaPool::new(); + std::thread::scope(|s| { + for k in 0..4i64 { + let (pool, f) = (&pool, &f); + s.spawn(move || { + for n in 0..15 { + let img = pool.frame_at(f, t_of(f.frame_rate, k * 15 + n)).expect("frame"); + assert_eq!((img.width, img.height), (1920, 1080)); + } + }); + } + }); +} + +#[test] +fn perf_sequential_1080p_h264() { + let Some(p) = h264_mp4() else { return }; + let f = probe(&p).expect("probe"); + let n = 60; + // FilmCraft alone (decode only, YUV frames) + let bytes: Arc<[u8]> = std::fs::read(&p).expect("read").into(); + let src = filmcraft_codecs::open_bytes("a.mp4", bytes).expect("open"); + let t0 = Instant::now(); + for i in 0..n { + let t = filmcraft_time::Tick(f.frame_rate.tick_of(i).0 + f.frame_rate.frame_duration().0 / 2); + src.video_frame(filmcraft_media::FrameRequest::full(t)).expect("frame"); + } + let raw = n as f64 / t0.elapsed().as_secs_f64(); + // Through the pool (decode + conversion to f32 RGBA), cold cache, without and with read-ahead + let pool = MediaPool::new(); + pool.set_read_ahead(false); + let t0 = Instant::now(); + for i in 0..n { + pool.frame_at(&f, t_of(f.frame_rate, i)).expect("frame"); + } + let plain = n as f64 / t0.elapsed().as_secs_f64(); + let pool = MediaPool::new(); + let t0 = Instant::now(); + for i in 0..n { + pool.frame_at(&f, t_of(f.frame_rate, i)).expect("frame"); + } + let ours = n as f64 / t0.elapsed().as_secs_f64(); + assert!(pool.stats().prefetched > 0); + // Cached (the last 20 frames fit the default budget) + let t0 = Instant::now(); + for i in n - 20..n { + pool.frame_at(&f, t_of(f.frame_rate, i)).expect("frame"); + } + let cached = 20.0 / t0.elapsed().as_secs_f64(); + assert!(pool.stats().hits >= 20); + eprintln!( + "1080p H.264 sequential: filmcraft decode {raw:.1} fps; MediaPool (decode + RGBA f32) {plain:.1} fps, with read-ahead {ours:.1} fps; cached {cached:.0} fps" + ); +} + +#[test] +fn exr_still_is_linear_float() { + // written with the image crate's OpenEXR encoder (no ffmpeg needed) + let dir = fixtures(); + std::fs::create_dir_all(&dir).expect("mkdir"); + let p = dir.join(format!("float-{}.exr", std::process::id())); + // linear 0.214 ≈ sRGB 0.5 (opaque); premultiplied 2.0 at alpha 0.5 + let px = vec![0.214f32, 0.214, 0.214, 1.0, 2.0, 1.0, 0.5, 0.5]; + image::DynamicImage::ImageRgba32F(image::ImageBuffer::from_raw(2, 1, px).expect("buf")).save(&p).expect("write exr"); + let f = probe(&p).expect("probe"); + assert_eq!(f.kind, FootageKind::Still); + assert_eq!((f.width, f.height), (2, 1)); + assert_eq!(f.codec, "EXR"); + assert_eq!(f.alpha, AlphaMode::Premultiplied); + let img = MediaPool::new().frame_at(&f, Tick::ZERO).expect("frame"); + std::fs::remove_file(&p).ok(); + let a = img.data[0]; + assert!((a[0] - 0.5).abs() < 2e-3 && a[3] == 1.0, "{a:?}"); + // straight 4.0 → encoded ≈ 1.83 (super-white kept) → re-premultiplied ≈ 0.92 + let b = img.data[1]; + assert!(b[3] == 0.5 && b[0] > 0.85 && b[0] < 1.0, "{b:?}"); +}