shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge main into #604: dialog hook in the new app creator; Install… keeps showing why a template was refused (#611) 

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas 2026-10-11 21:47:02 -04:00
parent 82242c3817 7c5367740d
commit f8701860d8
372 changed files with 48458 additions and 3032 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
.github/workflows/packaging-lint.yml vendored
View File

@ -19,57 +19,5 @@ concurrency:
jobs:
lint:
runs-on: ubuntu-latest
env:
ACTIONLINT_VERSION: 1.7.12
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install tools
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends shellcheck desktop-file-utils appstream libxml2-utils python3-yaml
curl -fsSL "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" \
| tar -xz -C "$RUNNER_TEMP" actionlint
- name: actionlint
run: |
"$RUNNER_TEMP/actionlint" -color
- name: shellcheck
run: shellcheck packaging/*.sh packaging/*/*.sh
- name: Linux tarball installer
run: python3 packaging/linux/test_install.py
- name: PowerShell
shell: pwsh
run: |
foreach ($f in Get-ChildItem packaging/windows/*.ps1) {
$errors = $null
[System.Management.Automation.Language.Parser]::ParseFile($f.FullName, [ref]$null, [ref]$errors) | Out-Null
if ($errors) { $errors | ForEach-Object { Write-Output "::error file=$($f.Name)::$_" }; exit 1 }
Write-Output "ok $($f.Name)"
}
- name: XML, desktop entry, AppStream, Flatpak manifest
run: |
xmllint --noout packaging/windows/filmcraft.wxs packaging/macos/*.plist* packaging/linux/*.xml*
# Windows Installer needs Icon ids ending in .ico or .exe, or shortcuts show a blank icon.
if grep -oE '<Icon Id="[^"]*"' packaging/windows/filmcraft.wxs | grep -vE '\.(ico|exe)"$'; then
echo "::error::Icon Id must end in .ico or .exe"; exit 1
fi
desktop-file-validate packaging/linux/ai.storyteller.filmcraft.desktop
sed -e 's/@VERSION@/0.0.0/' -e 's/@DATE@/2026-01-01/' packaging/linux/ai.storyteller.filmcraft.metainfo.xml.in >"$RUNNER_TEMP/ai.storyteller.filmcraft.metainfo.xml"
appstreamcli validate --no-net --explain "$RUNNER_TEMP/ai.storyteller.filmcraft.metainfo.xml"
python3 - <<'EOF'
import yaml
src = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.filmcraft.yml"))
bundle = yaml.safe_load(open("packaging/linux/flatpak/ai.storyteller.filmcraft.bundle.yml"))
for m in (src, bundle):
assert m["id"] == "ai.storyteller.filmcraft" and m["command"] == "filmcraft", m["id"]
for key in ("runtime", "runtime-version", "sdk", "finish-args"):
assert src[key] == bundle[key], f"Flatpak manifests disagree on {key}: {src[key]} vs {bundle[key]}"
# Both install the app and the CLI/MCP server (#346).
import re
def programs(m):
return {p for c in m["modules"][0]["build-commands"] for p in re.findall(r"/app/bin/([\w.-]+)", c)}
assert programs(src) == programs(bundle) == {"filmcraft", "filmcraft-cli"}, f"Flatpak manifests install {programs(src)} vs {programs(bundle)}"
print("Flatpak manifests ok")
EOF
- name: FreeBSD package layout (stub binaries)
run: bash packaging/freebsd/package.sh --dry-run
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
uses: storytold/craft-common-actions/.github/workflows/packaging-lint.yml@main

411
.github/workflows/release.yml vendored
View File

@ -1,12 +1,3 @@
# Release pipeline: every push to `release` builds signed installers for every platform and
# creates or updates the draft GitHub Release "FilmCraft v<version>". See docs/releasing.md.
#
# The jobs that sign (macOS, Windows) and the one that publishes the draft run in the `release`
# environment (restricted to the `release` branch), which holds the signing secrets. Every secret
# is optional: missing ones produce unsigned artifacts and a warning, never a failed build. The
# jobs that sign nothing (version, Linux, Flatpak, FreeBSD, web) get no environment and no
# secrets, so a dispatch on any other branch dry-runs them; there the signing jobs are refused,
# so the draft-release job (which needs them) is skipped and nothing is published.
name: Release
on:
@ -20,395 +11,27 @@ on:
default: ""
permissions:
contents: read
contents: write
concurrency:
group: release-${{ github.ref }}
# Never cancel a run mid-notarization or mid-upload; a newer push waits its turn.
cancel-in-progress: false
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
RUST_BACKTRACE: 1
FILMCRAFT_BUILD_SHA: ${{ github.sha }}
INPUT_VERSION: ${{ inputs.version }}
jobs:
version:
name: version
runs-on: ubuntu-latest
outputs:
version: ${{ steps.v.outputs.version }}
date: ${{ steps.v.outputs.date }}
prerelease: ${{ steps.v.outputs.prerelease }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- id: v
name: Resolve version
run: |
. packaging/env.sh
v="${INPUT_VERSION#v}"
v="${v:-$VERSION}"
if ! printf '%s' "$v" | grep -Eq '^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'; then
echo "::error::'$v' is not a version like 1.2.3 or 1.2.3-rc.1"
exit 1
fi
case "$v" in *-*) pre=true ;; *) pre=false ;; esac
{
echo "version=$v"
echo "date=$(date -u +%Y-%m-%d)"
echo "prerelease=$pre"
} >>"$GITHUB_OUTPUT"
echo "### FilmCraft v$v (\`${GITHUB_SHA::9}\`)" >>"$GITHUB_STEP_SUMMARY"
macos:
name: macOS universal
needs: version
runs-on: macos-15-xlarge # M2 Pro larger runner; use only in release.yml (actions_macos_xl budget caps spend)
environment: release
env:
FILMCRAFT_VERSION: ${{ needs.version.outputs.version }}
FILMCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
path: craft-fonts
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: aarch64-apple-darwin, x86_64-apple-darwin
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-macos-universal
- name: Apply version override
if: inputs.version != ''
run: cargo xtask version set "$FILMCRAFT_VERSION"
- name: Import signing certificate
env:
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
run: packaging/macos/import-cert.sh
- name: Build, sign, notarize
env:
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
run: packaging/macos/package.sh --arch universal
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-macos
path: dist/release/*
if-no-files-found: error
- name: Remove temporary keychain
if: always()
run: |
if [ -n "${MACOS_KEYCHAIN:-}" ]; then security delete-keychain "$MACOS_KEYCHAIN" || true; fi
windows:
name: Windows ${{ matrix.arch }}
needs: version
runs-on: windows-latest
environment: release
strategy:
fail-fast: false
matrix:
include:
- { arch: x64, target: x86_64-pc-windows-msvc }
- { arch: x86, target: i686-pc-windows-msvc }
# Windows on ARM: cross-compiled on the x64 runner, so signing and WiX run as above.
# .github/workflows/windows-arm64.yml installs and runs it on ARM64 hardware.
- { arch: arm64, target: aarch64-pc-windows-msvc }
env:
FILMCRAFT_VERSION: ${{ needs.version.outputs.version }}
FILMCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
path: craft-fonts
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: ${{ matrix.target }}
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-windows-${{ matrix.arch }}
- name: Apply version override
if: inputs.version != ''
shell: bash
run: cargo xtask version set "$FILMCRAFT_VERSION"
- name: Install WiX
shell: pwsh
run: dotnet tool install --global wix --version 5.0.2
- name: Build, sign, package
shell: pwsh
env:
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_SIGNING_ENDPOINT: ${{ secrets.AZURE_SIGNING_ENDPOINT }}
AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }}
AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }}
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
run: ./packaging/windows/package.ps1 -Arch ${{ matrix.arch }}
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-windows-${{ matrix.arch }}
path: dist/release/*
if-no-files-found: error
linux:
name: Linux ${{ matrix.arch }}
needs: version
# The oldest GitHub-hosted image, so the binaries only need glibc >= 2.35 (Ubuntu 22.04+,
# Debian 12+, Fedora 36+, RHEL 10). See docs/releasing.md.
runs-on: ${{ matrix.runner }}
# No `environment: release`: this job signs nothing, so it gets no release secrets.
strategy:
fail-fast: false
matrix:
include:
- { arch: x86_64, runner: ubuntu-22.04, deb: amd64 }
- { arch: aarch64, runner: ubuntu-22.04-arm, deb: arm64 }
env:
FILMCRAFT_VERSION: ${{ needs.version.outputs.version }}
FILMCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
NFPM_VERSION: 2.47.0
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
path: craft-fonts
- name: Install build and packaging tools
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev libasound2-dev \
desktop-file-utils appstream python3-yaml zsync
curl -fsSL -o "$RUNNER_TEMP/nfpm.deb" \
"https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_${{ matrix.deb }}.deb"
sudo dpkg -i "$RUNNER_TEMP/nfpm.deb"
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-linux-${{ matrix.arch }}
- name: Apply version override
if: inputs.version != ''
run: cargo xtask version set "$FILMCRAFT_VERSION"
- name: Build and package (AppImage, deb, rpm, tar.gz)
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
run: packaging/linux/package.sh
- name: Check the packages
run: |
dpkg-deb --info dist/release/*.deb
dpkg-deb --contents dist/release/*.deb | grep -E 'usr/bin/filmcraft$|metainfo|applications'
ldd target/release/filmcraft
python3 -c "import yaml; m = yaml.safe_load(open('packaging/linux/flatpak/ai.storyteller.filmcraft.yml')); assert m['id'] == 'ai.storyteller.filmcraft'"
for appimage in dist/release/*.AppImage; do
chmod +x "$appimage"
APPIMAGE_EXTRACT_AND_RUN=1 "$appimage" --version
# Update information is embedded, and the .zsync for delta updates sits beside it.
"$appimage" --appimage-updateinformation | grep -F 'gh-releases-zsync|'
test -s "$appimage.zsync"
done
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-linux-${{ matrix.arch }}
path: dist/release/*
if-no-files-found: error
flatpak:
name: Flatpak ${{ matrix.arch }}
needs: [version, linux]
# Repackages the Linux job's tarball (the same binaries as the AppImage/deb/rpm); no Rust
# build here. 24.04 for a newer flatpak-builder (1.4) than 22.04 ships.
runs-on: ${{ matrix.runner }}
# No `environment: release`: this job signs nothing, so it gets no release secrets.
strategy:
fail-fast: false
matrix:
include:
- { arch: x86_64, runner: ubuntu-24.04 }
- { arch: aarch64, runner: ubuntu-24.04-arm }
env:
FILMCRAFT_VERSION: ${{ needs.version.outputs.version }}
FILMCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: release-linux-${{ matrix.arch }}
path: dist/linux
- name: Install flatpak-builder
run: |
sudo apt-get update
# librsvg2-common: the SVG pixbuf loader. flatpak-builder runs the host's
# `appstreamcli compose`, which renders the scalable icon and fails without it.
sudo apt-get install -y --no-install-recommends flatpak flatpak-builder librsvg2-common
# Ubuntu 24.04's AppArmor blocks the unprivileged user namespaces bubblewrap needs.
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
- name: Build the Flatpak bundle
run: packaging/linux/flatpak-bundle.sh "dist/linux/filmcraft-$FILMCRAFT_VERSION-linux-${{ matrix.arch }}.tar.gz"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-flatpak-${{ matrix.arch }}
path: dist/release/*.flatpak
if-no-files-found: error
freebsd:
name: FreeBSD x86_64
needs: version
# GitHub has no FreeBSD runners: build in a FreeBSD 14.3 VM, the same image and packages as
# .github/workflows/freebsd.yml.
runs-on: ubuntu-latest
timeout-minutes: 180
# No `environment: release`: this job signs nothing, so it gets no release secrets.
env:
FILMCRAFT_VERSION: ${{ needs.version.outputs.version }}
FILMCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
path: craft-fonts
- name: Build and package in a FreeBSD VM (tar.gz)
# vmactions/freebsd-vm v1.5.9, pinned by commit (it runs our build).
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
with:
release: "14.3"
usesh: true
cpu: 4
mem: 12288
# Passed into the VM by name; never templated into the script below.
envs: "FILMCRAFT_VERSION FILMCRAFT_BUILD_DATE FILMCRAFT_BUILD_SHA INPUT_VERSION"
# The FreeBSD CI packages (.github/workflows/freebsd.yml), plus bash for the script.
prepare: |
pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
run: |
set -e
. "$HOME/.cargo/env"
git config --global --add safe.directory "$PWD"
# Build outside the checkout so only dist/ is copied back out of the VM (copying the
# target dir back fails on large builds). 4 jobs: more runs the VM out of memory.
export CARGO_TARGET_DIR=/var/tmp/filmcraft-target CARGO_BUILD_JOBS=4
if [ -n "$INPUT_VERSION" ]; then cargo xtask version set "$FILMCRAFT_VERSION"; fi
# craft-fonts was checked out next to the sources and synced into the VM with them.
export CRAFT_FONTS_DIR="$PWD/craft-fonts" CRAFT_FONTS_REQUIRED=1
bash packaging/freebsd/package.sh
tar -tzf dist/release/filmcraft-*-freebsd-x86_64.tar.gz \
| grep -E '/bin/filmcraft$|/bin/filmcraft-cli$|/share/applications/ai\.storyteller\.filmcraft\.desktop$|/share/doc/filmcraft/LICENSE-MIT$'
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-freebsd-x86_64
path: dist/release/*
if-no-files-found: error
web:
name: Web (wasm)
needs: version
runs-on: ubuntu-latest
# No `environment: release`: this job signs nothing, so it gets no release secrets.
env:
FILMCRAFT_VERSION: ${{ needs.version.outputs.version }}
FILMCRAFT_BUILD_DATE: ${{ needs.version.outputs.date }}
WASM_BINDGEN_VERSION: 0.2.129
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: storytold/craft-fonts
ref: abb83316d96aa59c1cf64784289e378fe9fa5695 # bump deliberately
path: craft-fonts
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: release-web
- name: Install wasm-bindgen CLI
run: |
# Must match the wasm-bindgen crate version exactly (WASM_BINDGEN in xtask/src/main.rs).
if ! { command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q "$WASM_BINDGEN_VERSION"; }; then
cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_VERSION" --locked
fi
wasm-bindgen --version
- name: Apply version override
if: inputs.version != ''
run: cargo xtask version set "$FILMCRAFT_VERSION"
- name: Build and zip
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
CRAFT_FONTS_REQUIRED: "1"
run: packaging/web/package.sh
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-web
path: dist/release/*
if-no-files-found: error
release:
name: Draft GitHub Release
needs: [version, macos, windows, linux, flatpak, freebsd, web]
runs-on: ubuntu-latest
environment: release
permissions:
contents: write
env:
VERSION: ${{ needs.version.outputs.version }}
PRERELEASE: ${{ needs.version.outputs.prerelease }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
steps:
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: release-*
path: artifacts
merge-multiple: true
- name: Checksums
working-directory: artifacts
run: |
sha256sum -- * | tee SHA256SUMS.txt
- name: Create or update the draft release
working-directory: artifacts
run: |
tag="v$VERSION"
title="FilmCraft v$VERSION"
pre=()
if [ "$PRERELEASE" = true ]; then pre=(--prerelease); fi
if draft="$(gh release view "$tag" --json isDraft --jq .isDraft 2>/dev/null)"; then
if [ "$draft" != "true" ]; then
echo "::error::Release $tag is already published. Bump the version (cargo xtask version set) before pushing to release."
exit 1
fi
echo "Updating draft $tag: replacing its assets"
gh release view "$tag" --json assets --jq '.assets[].name' | while read -r asset; do
gh release delete-asset "$tag" "$asset" --yes
done
gh release edit "$tag" --draft --title "$title" --target "$GITHUB_SHA" --prerelease="$PRERELEASE"
gh release upload "$tag" --clobber -- *
else
echo "Creating draft $tag"
gh release create "$tag" --draft --title "$title" --target "$GITHUB_SHA" --generate-notes "${pre[@]}" -- *
fi
{
echo "Draft release **$title** at \`${GITHUB_SHA::9}\`:"
echo
sed 's/^/ /' SHA256SUMS.txt
} >>"$GITHUB_STEP_SUMMARY"
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
uses: storytold/craft-common-actions/.github/workflows/release.yml@main
with:
app-name: filmcraft
app-title: FilmCraft
craft-fonts-ref: abb83316d96aa59c1cf64784289e378fe9fa5695
version: ${{ inputs.version }}
secrets:
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}

72
.github/workflows/windows-arm64.yml vendored
View File

@ -26,70 +26,10 @@ concurrency:
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
jobs:
package:
name: package (cross-compiled on x64)
runs-on: windows-latest
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
with:
targets: aarch64-pc-windows-msvc
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
key: windows-arm64-package
- name: Install WiX
shell: pwsh
run: dotnet tool install --global wix --version 5.0.2
- name: Build and package (unsigned)
shell: pwsh
env:
DIST: ${{ github.workspace }}\dist\arm64
run: ./packaging/windows/package.ps1 -Arch arm64
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: filmcraft-windows-arm64
path: dist/arm64/*
if-no-files-found: error
retention-days: 7
install:
name: install and run on ARM64
needs: package
runs-on: windows-11-arm
timeout-minutes: 30
steps:
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: filmcraft-windows-arm64
path: dist
- name: Install the MSI, check it, run it, uninstall it
shell: pwsh
run: |
$ErrorActionPreference = 'Stop'
$msi = (Get-ChildItem dist/*.msi | Select-Object -First 1).FullName
$p = Start-Process msiexec.exe -ArgumentList '/i', "`"$msi`"", '/qn', '/l*v', 'install.log' -Wait -PassThru
if ($p.ExitCode -ne 0) { Get-Content install.log -Tail 60; throw "msiexec /i exited $($p.ExitCode)" }
$dir = Join-Path $env:ProgramFiles 'FilmCraft'
foreach ($exe in 'filmcraft.exe', 'filmcraft-cli.exe') {
$path = Join-Path $dir $exe
if (-not (Test-Path $path)) { throw "$exe not installed in $dir" }
$bytes = [System.IO.File]::ReadAllBytes($path)
$machine = [BitConverter]::ToUInt16($bytes, [BitConverter]::ToInt32($bytes, 0x3C) + 4)
if ($machine -ne 0xAA64) { throw "$exe is for machine 0x$('{0:X}' -f $machine), not ARM64" }
Write-Output "ok $path (ARM64)"
}
# The installed CLI runs natively on ARM64.
$cli = Join-Path $dir 'filmcraft-cli.exe'
$out = & $cli --version 2>&1 | Out-String
if ($LASTEXITCODE -ne 0) { Write-Output $out; throw "filmcraft-cli --version exited $LASTEXITCODE" }
Write-Output "ok filmcraft-cli --version: $($out.Trim())"
$p = Start-Process msiexec.exe -ArgumentList '/x', "`"$msi`"", '/qn' -Wait -PassThru
if ($p.ExitCode -ne 0) { throw "msiexec /x exited $($p.ExitCode)" }
if (Test-Path (Join-Path $dir 'filmcraft.exe')) { throw 'uninstall left filmcraft.exe behind' }
Write-Output 'ok uninstalled'
windows-arm64:
# Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos.
uses: storytold/craft-common-actions/.github/workflows/windows-arm64.yml@main
with:
app-name: filmcraft
app-dir: FilmCraft

4
ATTRIBUTION.md
View File

@ -9,6 +9,10 @@ iconography, images or other Adobe assets; open licences only; every asset attri
| Asset | Author | Source | Licence |
|---|---|---|---|
| `crates/ui-egui/src/i18n/id.tsv` | FilmCraft contributors | Original work: clean-room Indonesian interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/fr.tsv` | FilmCraft contributors | Original work: clean-room French interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/pt-br.tsv` | FilmCraft contributors | Original work: clean-room Brazilian Portuguese interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/ru.tsv` | FilmCraft contributors | Original work: clean-room Russian interface translations | MIT OR Apache-2.0 |
| `crates/ui-egui/src/i18n/uk.tsv` | FilmCraft contributors | Original work: clean-room Ukrainian interface translations | MIT OR Apache-2.0 |
| `assets/fonts/Inter-Regular.ttf` | The Inter Project Authors | https://github.com/rsms/inter | OFL-1.1 (`assets/fonts/OFL-Inter.txt`) |
| `assets/fonts/Inter-Medium.ttf` | The Inter Project Authors | https://github.com/rsms/inter | OFL-1.1 (`assets/fonts/OFL-Inter.txt`) |

149
Cargo.lock generated
View File

@ -1011,6 +1011,15 @@ dependencies = [
"libc",
]
[[package]]
name = "craft-ui"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=5c874d51eeb3de70e735c8ce87a59a7dedd47675#5c874d51eeb3de70e735c8ce87a59a7dedd47675"
dependencies = [
"egui",
"serde",
]
[[package]]
name = "crc32fast"
version = "1.5.2"
@ -1665,7 +1674,7 @@ dependencies = [
[[package]]
name = "filmcraft"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"arboard",
"chrono",
@ -1687,6 +1696,7 @@ dependencies = [
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2",
"open",
"pollster 0.4.0",
"rfd",
"serde_json",
"sys-locale",
@ -1697,7 +1707,7 @@ dependencies = [
[[package]]
name = "filmcraft-aac"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -1707,7 +1717,7 @@ dependencies = [
[[package]]
name = "filmcraft-ac3"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
"thiserror 2.0.21",
@ -1715,7 +1725,7 @@ dependencies = [
[[package]]
name = "filmcraft-apv"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -1725,14 +1735,14 @@ dependencies = [
[[package]]
name = "filmcraft-audio-dsp"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
]
[[package]]
name = "filmcraft-automation"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"base64 0.22.1",
"filmcraft-engine",
@ -1748,7 +1758,7 @@ dependencies = [
[[package]]
name = "filmcraft-av1"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -1757,16 +1767,23 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-avi"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
]
[[package]]
name = "filmcraft-bitstream"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"proptest",
]
[[package]]
name = "filmcraft-captions"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"filmcraft-project",
@ -1779,11 +1796,11 @@ dependencies = [
[[package]]
name = "filmcraft-cfb"
version = "0.5.0"
version = "0.6.0"
[[package]]
name = "filmcraft-cli"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-automation",
"filmcraft-codecs",
@ -1797,6 +1814,7 @@ dependencies = [
"filmcraft-platform",
"filmcraft-project",
"filmcraft-render",
"filmcraft-testkit",
"filmcraft-time",
"serde_json",
"tokio",
@ -1804,12 +1822,13 @@ dependencies = [
[[package]]
name = "filmcraft-codecs"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-aac",
"filmcraft-ac3",
"filmcraft-apv",
"filmcraft-av1",
"filmcraft-avi",
"filmcraft-bitstream",
"filmcraft-color",
"filmcraft-dnx",
@ -1819,6 +1838,7 @@ dependencies = [
"filmcraft-isobmff",
"filmcraft-matroska",
"filmcraft-media",
"filmcraft-mpa",
"filmcraft-mpeg2v",
"filmcraft-mpegts",
"filmcraft-mxf",
@ -1827,9 +1847,11 @@ dependencies = [
"filmcraft-prores",
"filmcraft-testkit",
"filmcraft-time",
"filmcraft-vp8",
"filmcraft-vp9",
"image",
"log",
"rayon",
"serde_json",
"symphonia",
"thiserror 2.0.21",
@ -1838,7 +1860,7 @@ dependencies = [
[[package]]
name = "filmcraft-color"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"proptest",
"serde",
@ -1846,7 +1868,7 @@ dependencies = [
[[package]]
name = "filmcraft-dnx"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -1856,7 +1878,7 @@ dependencies = [
[[package]]
name = "filmcraft-edit"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-geom",
"filmcraft-media",
@ -1869,7 +1891,7 @@ dependencies = [
[[package]]
name = "filmcraft-engine"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-audio-dsp",
"filmcraft-captions",
@ -1881,6 +1903,7 @@ dependencies = [
"filmcraft-frame",
"filmcraft-geom",
"filmcraft-interchange",
"filmcraft-isobmff",
"filmcraft-media",
"filmcraft-mxf",
"filmcraft-project",
@ -1901,7 +1924,7 @@ dependencies = [
[[package]]
name = "filmcraft-export"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-aac",
"filmcraft-apv",
@ -1909,6 +1932,7 @@ dependencies = [
"filmcraft-codecs",
"filmcraft-color",
"filmcraft-dnx",
"filmcraft-flac",
"filmcraft-frame",
"filmcraft-geom",
"filmcraft-h264enc",
@ -1929,9 +1953,18 @@ dependencies = [
"web-time",
]
[[package]]
name = "filmcraft-flac"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-format"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-media",
"filmcraft-project",
@ -1944,7 +1977,7 @@ dependencies = [
[[package]]
name = "filmcraft-frame"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"filmcraft-time",
@ -1954,7 +1987,7 @@ dependencies = [
[[package]]
name = "filmcraft-geom"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"proptest",
"serde",
@ -1962,7 +1995,7 @@ dependencies = [
[[package]]
name = "filmcraft-golden"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"filmcraft-frame",
@ -1979,7 +2012,7 @@ dependencies = [
[[package]]
name = "filmcraft-gpu"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"filmcraft-export",
@ -1998,7 +2031,7 @@ dependencies = [
[[package]]
name = "filmcraft-h264"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -2008,7 +2041,7 @@ dependencies = [
[[package]]
name = "filmcraft-h264enc"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -2018,7 +2051,7 @@ dependencies = [
[[package]]
name = "filmcraft-hevc"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -2028,7 +2061,7 @@ dependencies = [
[[package]]
name = "filmcraft-interchange"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-cfb",
"filmcraft-color",
@ -2045,7 +2078,7 @@ dependencies = [
[[package]]
name = "filmcraft-isobmff"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -2056,7 +2089,7 @@ dependencies = [
[[package]]
name = "filmcraft-matroska"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
"serde_json",
@ -2064,7 +2097,7 @@ dependencies = [
[[package]]
name = "filmcraft-media"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"filmcraft-frame",
@ -2075,9 +2108,18 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-mpa"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-mpeg2v"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
"rayon",
@ -2086,7 +2128,7 @@ dependencies = [
[[package]]
name = "filmcraft-mpegts"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
"thiserror 2.0.21",
@ -2094,28 +2136,28 @@ dependencies = [
[[package]]
name = "filmcraft-mxf"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
]
[[package]]
name = "filmcraft-ogg"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
]
[[package]]
name = "filmcraft-opus"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
]
[[package]]
name = "filmcraft-platform"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-codecs",
@ -2141,7 +2183,7 @@ dependencies = [
[[package]]
name = "filmcraft-project"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"filmcraft-geom",
@ -2155,7 +2197,7 @@ dependencies = [
[[package]]
name = "filmcraft-prores"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"filmcraft-testkit",
@ -2165,7 +2207,7 @@ dependencies = [
[[package]]
name = "filmcraft-render"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-audio-dsp",
"filmcraft-captions",
@ -2185,7 +2227,7 @@ dependencies = [
[[package]]
name = "filmcraft-scopes"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"serde",
@ -2194,7 +2236,7 @@ dependencies = [
[[package]]
name = "filmcraft-speech"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"candle-core",
"candle-nn",
@ -2212,14 +2254,14 @@ dependencies = [
[[package]]
name = "filmcraft-testkit"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"png",
]
[[package]]
name = "filmcraft-text"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-color",
"harfrust",
@ -2231,7 +2273,7 @@ dependencies = [
[[package]]
name = "filmcraft-time"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"proptest",
"serde",
@ -2239,7 +2281,7 @@ dependencies = [
[[package]]
name = "filmcraft-tts"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"candle-core",
"candle-nn",
@ -2253,7 +2295,7 @@ dependencies = [
[[package]]
name = "filmcraft-tts-text"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"serde",
"thiserror 2.0.21",
@ -2261,9 +2303,10 @@ dependencies = [
[[package]]
name = "filmcraft-ui-egui"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"cpu-time",
"craft-ui",
"eframe",
"egui",
"egui_kittest",
@ -2293,12 +2336,20 @@ dependencies = [
"serde",
"serde_json",
"toml 1.1.6+spec-1.1.0",
"unicode-bidi",
"web-time",
]
[[package]]
name = "filmcraft-vp8"
version = "0.6.0"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-vp9"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-bitstream",
"rayon",
@ -2307,7 +2358,7 @@ dependencies = [
[[package]]
name = "filmcraft-web"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"eframe",
"egui",
@ -7332,7 +7383,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7"
[[package]]
name = "xtask"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"filmcraft-testkit",
"serde_json",

14
Cargo.toml
View File

@ -4,7 +4,7 @@ members = ["crates/*", "apps/*", "xtask"]
default-members = ["crates/*", "apps/*", "xtask"]
[workspace.package]
version = "0.5.0"
version = "0.6.0"
edition = "2024"
license = "MIT OR Apache-2.0"
rust-version = "1.95"
@ -17,21 +17,25 @@ filmcraft-geom = { path = "crates/geom" }
filmcraft-color = { path = "crates/color" }
filmcraft-bitstream = { path = "crates/bitstream" }
filmcraft-isobmff = { path = "crates/isobmff" }
filmcraft-avi = { path = "crates/avi" }
filmcraft-matroska = { path = "crates/matroska" }
filmcraft-mxf = { path = "crates/mxf" }
filmcraft-cfb = { path = "crates/cfb" }
filmcraft-mpegts = { path = "crates/mpegts" }
filmcraft-mpeg2v = { path = "crates/mpeg2v" }
filmcraft-ac3 = { path = "crates/ac3" }
filmcraft-mpa = { path = "crates/mpa" }
filmcraft-ogg = { path = "crates/ogg" }
filmcraft-h264 = { path = "crates/h264" }
filmcraft-hevc = { path = "crates/hevc" }
filmcraft-vp8 = { path = "crates/vp8" }
filmcraft-vp9 = { path = "crates/vp9" }
filmcraft-prores = { path = "crates/prores" }
filmcraft-dnx = { path = "crates/dnx" }
filmcraft-apv = { path = "crates/apv" }
filmcraft-av1 = { path = "crates/av1" }
filmcraft-aac = { path = "crates/aac" }
filmcraft-flac = { path = "crates/flac" }
filmcraft-opus = { path = "crates/opus" }
filmcraft-h264enc = { path = "crates/h264enc" }
filmcraft-audio-dsp = { path = "crates/audio-dsp" }
@ -67,6 +71,8 @@ proptest = "1"
roxmltree = "0.21"
# ui
# Shared frontend components; keep this dependency above the engine boundary.
craft-ui = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "5c874d51eeb3de70e735c8ce87a59a7dedd47675", features = ["serde"] }
egui = "0.36"
eframe = { version = "0.36", default-features = false }
egui_kittest = "0.36"
@ -113,6 +119,9 @@ opt-level = 3
[profile.dev.package.filmcraft-aac]
opt-level = 3
[profile.dev.package.filmcraft-flac]
opt-level = 3
# Audio effects: realtime-factor tests and long convolution / spectral runs.
[profile.dev.package.filmcraft-audio-dsp]
opt-level = 3
@ -123,6 +132,9 @@ opt-level = 3
[profile.dev.package.filmcraft-opus]
opt-level = 3
[profile.dev.package.filmcraft-vp8]
opt-level = 3
[profile.dev.package.filmcraft-vp9]
opt-level = 3

7
README.md
View File

@ -175,6 +175,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus
| **HEVC / H.265** | ✓ | | Main and Main 10, bit-exact on 41 streams (tiles, WPP, PCM, long-term references), about 225 fps at 1080p |
| **Apple ProRes** | ✓ | ✓ | Decodes 422 Proxy to 4444 XQ; export writes 422 HQ |
| **AAC-LC** | ✓ | ✓ | |
| **VP8** | ✓ | | Every coding tool, bitstream versions 0–3, hidden altref frames, alpha layer of transparent WebM; bit-exact with libvpx on its published VP8 test vectors; in WebM/Matroska and MP4 |
| **VP9** | ✓ | | Profiles 0–3, 8/10/12-bit, 4:2:0 to 4:4:4, tiles and superframes; bit-exact on 50+ streams; in WebM/Matroska and MP4 |
| **Opus** | ✓ | | SILK, CELT, hybrid and multistream surround; range-exact on every RFC 8251 conformance vector; in WebM/Matroska and MP4 |
| **AV1** | ✓ | | Main profile, 8/10-bit 4:2:0 and monochrome, every coding tool incl. film grain, superres and spatial layers; bit-exact with libdav1d on the libaom vectors tested; in MP4 and WebM/Matroska |
@ -183,6 +184,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus
| **MP3, FLAC, ALAC, Vorbis** | ✓ | | Via the [symphonia](https://github.com/pdeljanov/Symphonia) crate (MPL-2.0) for now, to be replaced by our own |
| **MP4 / MOV** | ✓ | ✓ | Fragmented MP4, edit lists, timecode tracks |
| **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata |
| **AVI** | ✓ | | AVI 1.0 and OpenDML (files past 1 GB); Motion JPEG, H.264, HEVC and uncompressed RGB / YUV with PCM, MP3, MP2 or AC-3. DivX / Xvid and DV not yet |
| **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF |
Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows and Linux (opt-in), and H.265 (HEVC Main, 8-bit) export works through it (NVENC on Windows, VideoToolbox on macOS) ([#30](https://github.com/storytold/filmcraft/issues/30)).
@ -304,7 +306,7 @@ A layered Cargo workspace:
| Layer | Crates |
|---|---|
| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `bitstream` |
| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `avi`, `bitstream` |
| Foundations | `time`, `geom`, `color`, `frame`, `media`, `text`, `project`, `audio-dsp` |
| Editing and interchange | `edit`, `codecs`, `captions`, `format` (project files), `interchange` |
| Rendering and output | `render`, `gpu`, `export` |
@ -362,6 +364,9 @@ and web backend defaults are unchanged.
| Fedora/RHEL/openSUSE | `filmcraft-<ver>-linux-x86_64.rpm` | `filmcraft-<ver>-linux-aarch64.rpm` | |
| Tarball | `filmcraft-<ver>-linux-x86_64.tar.gz` | `filmcraft-<ver>-linux-aarch64.tar.gz` | Unpack and run `./install.sh` |
RISC-V (riscv64): `filmcraft-<ver>-linux-riscv64.tar.gz` (tarball only; needs glibc 2.39+, e.g. Ubuntu 24.04
or Debian 13).
The tarball installer places FilmCraft, its command-line tool and desktop integration in
`~/.local`, without administrator permissions. For all users, run
`sudo ./install.sh --prefix /usr/local` instead. Running the installer again updates the installation.

15
ROADMAP.md
View File

@ -1,8 +1,11 @@
# FilmCraft Roadmap
- UI.6: Compact Source and Program monitors separate timecodes from view settings and retain Source drag choices. Transport controls use the shared toolbar sizing helper, with overflow actions in an operable More menu and a scrolling fallback at tiny widths.
- UI.3: Panel tabs, edge splits, tab reordering and floating groups use the shared docking model and drop resolver. Floating geometry is saved with named workspaces while legacy layout JSON remains readable. Dock dividers use bounded shared geometry and keyboard-accessible resize handles. Tools and dock tabs expose actual widget focus, paint focus outlines, and support keyboard traversal/activation. Control input reaches focused widgets in background windows; optional input isolation supports unattended validation. Existing workspace trees and saved layout formats are retained.
**Stage: alpha** · next: beta, ~19 points of ready-for-real-work and ~400–700 h away, plus opening Premiere's `.prproj` ([why](docs/target-app-parity.md#stage); passes the [core-workflow gate](docs/roadmap.md#alpha-gate))
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (full re-measure against Premiere Pro 2026 26.5.2; scorecard, gaps, milestones and localization moved into docs/ per the craftrules progress-docs standard) · **Target:** Adobe Premiere Pro 2026 (26.5.2)
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (language table: German ships the menus, #630) · **Target:** Adobe Premiere Pro 2026 (26.5.2)
Progress toward parity with Adobe Premiere Pro. This page is the summary; the detail is in
[docs/target-app-parity.md](docs/target-app-parity.md) (assessment, methodology, calibration),
@ -78,11 +81,11 @@ Premiere ships 10 interface languages. Measured from our catalogs (3,498 strings
| Spanish | partial (whole interface) | 99.9% |
| Hindi | none (no Devanagari shaping in the interface) | 0% |
| Arabic | none (no RTL interface) | 0% |
| French | none | 0% |
| Portuguese (Brazil) | menus only | 9.9% |
| Indonesian | none | 0% |
| French | partial (whole interface; native review welcome) | ~100% |
| Portuguese (Brazil) | partial (whole interface; native review welcome) | ~100% |
| Indonesian | partial (whole interface; native review welcome) | ~100% |
| Japanese | partial (whole interface) | 99.7% |
| German | none | 0% |
| German | menus only (native-reviewed) | 10.6% |
| Korean | none | 0% |
| Vietnamese | none | 0% |
@ -103,6 +106,7 @@ Ranked; detail and "done when" in [docs/roadmap.md](docs/roadmap.md) and [docs/g
## Progress log
- **2026-10-10 (changed media):** Settings ▸ Media ▸ Automatically refresh growing files works: media files that change on disk while the project is open (a recording still being written, a render written over the file a clip uses) are read again every *Refresh growing files every* seconds and as soon as the window comes back to the front: the new length, picture and identity, without an undo step (`media.refreshChanged`, `media_watch`).
- **2026-10-10 (re-measure):** full parity re-measure against Premiere Pro 26.5.2: menu label diff 338 / 366 (92%), installed bundle listings (10 UI languages, exporter and codec frameworks), 188 open issues. Breadth ~86% (file formats now counted), ready ~55%, stage **alpha**; progress docs reorganized per the craftrules standard ([target-app-parity](docs/target-app-parity.md), [gaps](docs/gaps.md), [roadmap](docs/roadmap.md), [codec](docs/codec-parity.md), [file-format](docs/file-format-parity.md), [hardware](docs/hardware-parity.md), [UI](docs/ui-parity.md), [localization](docs/localization-parity.md)).
- **2026-10-10:** ~75 community PRs merged: GPU export rendering (Auto) with Vignette, Video Limiter and Lumetri basic / creative / vignette on the GPU (#421); Linux NVDEC H.264 / HEVC decode (#549); text-to-speech narrations with Kokoro-82M (#278); Japanese for the whole interface (#449) and Ukrainian menus (#407); ALSA default-device fallback (#254, fixes #23); drop import into the timeline (#517); system appearance (#505); timecode scrub editing (#576); NVENC RGBA input (#486); marker CSV export (#454); many fixes (#526–#575).
- **2026-10-09:** release 0.5.0. NVENC H.265 export on Windows, Main and Main 10 HDR (#328, #360); NVENC H.264 export (#386); VA-API H.264 / HEVC decode (#358); Source monitor playback (#423); multi-stream container audio, schema v13 (#361); New Sequence and Sequence Settings dialogs (#269, #355); Spanish for the panels (#238); background decoders and memory stress tests (#385, #381).
@ -160,6 +164,7 @@ Ranked; detail and "done when" in [docs/roadmap.md](docs/roadmap.md) and [docs/g
| Date | Change | Summary |
|---|---|---|
| 2026-10-11 | minor | Language table: German ships the menus (#630, 10.6%) |
| 2026-10-10 | minor | Readiness-by-audience table with hours to ~95% per audience |
| 2026-10-10 | minor | Headline adds mainstream practitioner (~55%) and essentials user (~60%); ready for real work ~55% -> ~56% (formula written out, same inputs); beta distance ~19 points |
| 2026-10-10 | minor | Stage banner links the core-workflow gate (all six workflows pass on macOS; stays alpha) |

3
apps/filmcraft-cli/Cargo.toml
View File

@ -33,3 +33,6 @@ filmcraft-ogg = { workspace = true }
filmcraft-automation = { workspace = true }
serde_json = { workspace = true }
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
[dev-dependencies]
filmcraft-testkit = { workspace = true }

117
apps/filmcraft-cli/src/main.rs
View File

@ -76,18 +76,21 @@ SUBCOMMANDS
import <file>... import media into the project
export <out> [--preset name] [--format f] [--range r] [--start s --end s] [--settings json]
[--gpu-rendering auto|off]
[--scale f] [--quality 0-100] [--no-audio] [--queue]
[--scale f] [--quality 0-100] [--no-audio] [--queue] [--allow-offline]
export the active sequence and wait for it to finish: with an
export preset (`export --list-presets`; built-in or the user's), or
a format (h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff, guessed
a format (h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|gif|wav|aiff|aac, guessed
from the extension); --range entire|inOut|workArea, or a custom
range in seconds; --settings is ExportSettings JSON merged over the
preset; --scale renders at a fraction of the frame size (0.5 =
half); --quality 0-100 for the formats that take one; --no-audio
leaves the sound out; --queue adds to the export queue and runs it
instead
instead. Offline media (missing, made offline, unreadable) renders
as placeholder frames: the export then lists it in `offlineMedia`,
warns on stderr and exits 1, unless --allow-offline accepts it
export --list-presets [query] list export presets (name, category, format) as JSON
render --seconds S --out f.png [--scale 0.5] render one Program frame to PNG
render --seconds S --out f.png [--scale 0.5] [--allow-offline]
render one Program frame to PNG (offline media: as for export)
probe <media> [--image-sequence]
media info as JSON, with MXF / Ogg / BWF details; with
--image-sequence <media> is the first numbered still of a sequence
@ -113,7 +116,8 @@ VALUES
Times: 254016000000 ticks per second; most commands also take seconds=, frame= or timecode=.
EXIT STATUS
0 success · 1 a command failed · 2 usage error
0 success · 1 a command failed, or export/render drew offline media without --allow-offline
· 2 usage error
";
/// Writes one diagnostic line to `w`, dropping any write error. `eprintln!` panics (exit 101) when stderr's
@ -209,6 +213,7 @@ fn format_for(path: &str) -> Option<&'static str> {
"tif" | "tiff" => "tiff",
"bmp" => "bmp",
"aif" | "aiff" => "aiff",
"m4a" | "aac" => "aac",
_ => return None,
})
}
@ -243,6 +248,13 @@ fn register_hardware_decoders() -> filmcraft_platform::Availability {
filmcraft_platform::register()
}
/// `path` made absolute against the working directory, as the project stores it. Unlike
/// `canonicalize`, `std::path::absolute` keeps symlinks and adds no Windows verbatim `\\?\`
/// prefix (#850). A path it cannot resolve is kept as given.
fn absolute(path: &str) -> String {
std::path::absolute(path).ok().and_then(|p| p.into_os_string().into_string().ok()).unwrap_or_else(|| path.to_string())
}
#[tokio::main]
async fn main() {
cli().await;
@ -252,7 +264,8 @@ async fn main() {
}
async fn cli() {
let a = Args::parse(std::env::args().skip(1));
// `std::env::args` panics on an argument that is not Unicode (a Linux file name in Latin-1)
let a = Args::parse(std::env::args_os().skip(1).map(|a| a.to_string_lossy().into_owned()));
// `--help` anywhere (`filmcraft-cli --help`, `filmcraft-cli export --help`) prints the reference:
// the parser takes it as a flag option, so it never reaches the subcommand match.
if a.flag("--help") {
@ -372,8 +385,7 @@ async fn cli() {
print(&a, &v);
}
"import" => {
let paths: Vec<String> =
a.positionals[1..].iter().map(|p| std::fs::canonicalize(p).map(|c| c.to_string_lossy().into_owned()).unwrap_or_else(|_| p.clone())).collect();
let paths: Vec<String> = a.positionals.get(1..).unwrap_or_default().iter().map(|p| absolute(p)).collect();
if paths.is_empty() {
usage("import <file>...");
}
@ -441,7 +453,9 @@ async fn cli() {
match r {
Ok(v) => {
print(&a, &v);
diag(format_args!("exported {} in {:.1}s", written_paths(&v, out).join(", "), t0.elapsed().as_secs_f64()));
let written = written_paths(&v, out).join(", ");
check_offline(&a, &offline_media(&v), &format!("export: {written}"));
diag(format_args!("exported {written} in {:.1}s", t0.elapsed().as_secs_f64()));
}
Err(e) => fail(format!("export: {e}")),
}
@ -456,10 +470,12 @@ async fn cli() {
let out = a.opt("--out").unwrap_or("frame.png");
s.set_playhead(filmcraft_time::Tick::from_seconds_f64(secs));
let t0 = std::time::Instant::now();
let img = render_at_playhead(&s, scale).unwrap_or_else(|e| fail(e));
let (img, offline) = render_at_playhead(&s, scale).unwrap_or_else(|e| fail(e));
let dt = t0.elapsed();
let png = filmcraft_automation::png_rgba(img.w as u32, img.h as u32, img.over_black_rgba8(), 0).unwrap_or_else(|e| fail(e));
std::fs::write(out, png).unwrap_or_else(|e| fail(format!("{out}: {e}")));
let offline: Vec<Value> = offline.iter().filter_map(|o| serde_json::to_value(o).ok()).collect();
check_offline(&a, &offline, &format!("render: {out}"));
diag(format_args!("rendered {}x{} in {:.1} ms → {out}", img.w, img.h, dt.as_secs_f64() * 1000.0));
}
"mcp" => {
@ -467,7 +483,7 @@ async fn cli() {
usage(format!("unknown mcp option `{opt}`"));
}
let server = match a.opt("--bridge") {
Some(addr) => filmcraft_automation::FilmcraftMcp::bridge(addr).unwrap_or_else(|e| fail(e)),
Some(addr) => filmcraft_automation::FilmcraftMcp::bridge(addr).unwrap_or_else(|e| usage(e)),
None => match Backend::open(&a) {
Backend::Local(s) => filmcraft_automation::FilmcraftMcp::headless(*s),
Backend::Bridge(_) => usage("mcp: use --bridge ADDR to drive a running app"),
@ -499,9 +515,46 @@ fn written_paths(result: &Value, requested: &str) -> Vec<String> {
if paths.is_empty() { vec![requested.to_string()] } else { paths }
}
/// The active sequence at the playhead, or the reason it cannot be rendered (no sequence, bad scale).
fn render_at_playhead(s: &Session, scale: f32) -> Result<filmcraft_engine::render::Image, String> {
s.try_render_program_at(scale, s.playhead()).map_err(|e| e.to_string())
/// The offline items an export result reports (`offlineMedia` of a direct export, or of each
/// queued item), once each.
fn offline_media(result: &Value) -> Vec<Value> {
let lists = std::iter::once(result).chain(result.get("items").and_then(Value::as_array).into_iter().flatten());
let mut out: Vec<Value> = Vec::new();
for o in lists.filter_map(|r| r.get("offlineMedia").and_then(Value::as_array)).flatten() {
if !out.iter().any(|x| x.get("item") == o.get("item")) {
out.push(o.clone());
}
}
out
}
/// One stderr line per offline item that rendered as placeholder frames (#1054).
fn offline_warning(offline: &[Value]) -> String {
let s = |o: &Value, k: &str| o.get(k).and_then(Value::as_str).unwrap_or("").to_string();
let mut msg = format!("filmcraft-cli: warning: {} offline media item(s) rendered as placeholder frames:", offline.len());
for o in offline {
let id = o.get("item").and_then(Value::as_u64).map(|i| i.to_string()).unwrap_or_default();
msg.push_str(&format!("\n item {id} {} ({}): {}", s(o, "name"), s(o, "status"), s(o, "path")));
}
msg
}
/// Warn about offline media on stderr; without `--allow-offline` that fails the run (exit 1), after
/// the output (`what`) was written, so scripts don't take placeholder frames for the real picture.
fn check_offline(a: &Args, offline: &[Value], what: &str) {
if offline.is_empty() {
return;
}
diag(format_args!("{}", offline_warning(offline)));
if !a.flag("--allow-offline") {
fail(format!("{what} has placeholder frames for offline media; relink it (exec media.relink …) or pass --allow-offline"));
}
}
/// The active sequence at the playhead with the offline items it drew, or the reason it cannot be
/// rendered (no sequence, bad scale).
fn render_at_playhead(s: &Session, scale: f32) -> Result<(filmcraft_engine::render::Image, Vec<filmcraft_engine::media_pool::OfflineMedia>), String> {
s.try_render_program_with_offline_at(scale, s.playhead()).map_err(|e| e.to_string())
}
/// `--start` / `--end` ask for a custom range. A lone bound is forwarded too, so the engine reports the missing one
@ -530,7 +583,7 @@ mod format_tests {
let e = super::render_at_playhead(&s, scale).unwrap_err();
assert!(e.contains("finite and positive"), "{e}");
}
assert_eq!(super::render_at_playhead(&s, 0.5).unwrap().w, 8);
assert_eq!(super::render_at_playhead(&s, 0.5).unwrap().0.w, 8);
}
/// The CLI (and MCP / headless runs, which share its entry point) registers the hardware
@ -572,6 +625,19 @@ mod format_tests {
assert_eq!(super::written_paths(&json!({"items": []}), "x.mov"), ["x.mov"]);
}
#[test]
fn offline_media_from_direct_and_queued_results() {
use serde_json::json;
let a = json!({"item": 8, "name": "a.mp4", "path": "/m/a.mp4", "status": "missing", "error": ""});
let b = json!({"item": 9, "name": "b.mov", "path": "/m/b.mov", "status": "offline", "error": ""});
assert_eq!(super::offline_media(&json!({"path": "x.mp4", "offlineMedia": [a]})), std::slice::from_ref(&a));
let queued = json!({"items": [{"offlineMedia": [a, b]}, {"offlineMedia": [a]}, {"offlineMedia": []}]});
assert_eq!(super::offline_media(&queued), [a.clone(), b.clone()]);
assert!(super::offline_media(&json!({"path": "x.mp4"})).is_empty());
let w = super::offline_warning(&[a, b]);
assert!(w.contains("2 offline media item(s)") && w.contains("item 8 a.mp4 (missing): /m/a.mp4") && w.contains("item 9 b.mov (offline)"), "{w}");
}
#[test]
fn export_lone_bound_is_forwarded_as_custom_range() {
use serde_json::json;
@ -589,6 +655,27 @@ mod format_tests {
assert!(p.get("range").is_none());
}
/// `import` used `canonicalize`, which resolves symlinks and on Windows stores `\\?\C:\…`
/// verbatim paths in the project (#850).
#[test]
fn imported_paths_are_absolute_without_resolving_links() {
let cwd = std::env::current_dir().unwrap();
assert_eq!(super::absolute("clip.mov"), cwd.join("clip.mov").to_string_lossy());
assert!(!super::absolute("clip.mov").starts_with(r"\\?\"));
assert_eq!(super::absolute(""), "");
#[cfg(unix)]
{
let dir = std::env::temp_dir().join(format!("filmcraft-cli-abs-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("real.mov"), b"x").unwrap();
let link = dir.join("link.mov");
let _ = std::fs::remove_file(&link);
std::os::unix::fs::symlink(dir.join("real.mov"), &link).unwrap();
assert_eq!(super::absolute(&link.to_string_lossy()), link.to_string_lossy());
std::fs::remove_dir_all(&dir).unwrap();
}
}
#[test]
fn export_format_from_extension() {
assert_eq!(super::format_for("out/clip.MXF"), Some("mxf-op1a"));

55
apps/filmcraft-cli/tests/cli.rs
View File

@ -22,6 +22,9 @@ fn help_and_usage_errors() {
assert_eq!(cli(&[]).status.code(), Some(2));
assert_eq!(cli(&["frobnicate"]).status.code(), Some(2));
assert_eq!(cli(&["exec", "file.newBin", "notkv"]).status.code(), Some(2));
// Invalid --bridge is a usage error (exit 2) for both exec and mcp (#921).
assert_eq!(cli(&["--bridge", "0.0.0.0:9876", "exec", "inspect.project"]).status.code(), Some(2));
assert_eq!(cli(&["mcp", "--bridge", "0.0.0.0:9876"]).status.code(), Some(2));
}
#[test]
@ -291,3 +294,55 @@ fn closed_stdout_still_runs_the_script_and_saves() {
assert_eq!(out.status.code(), Some(1));
let _ = std::fs::remove_dir_all(&dir);
}
/// #1054: exporting or rendering a project whose media is missing fails (exit 1) and names the
/// offline items on stderr and in the result's `offlineMedia`, unless `--allow-offline` accepts the
/// placeholder frames; then it still warns.
#[test]
fn export_with_missing_media_fails_unless_allowed() {
let dir = std::env::temp_dir().join(format!("fc-cli-1054-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let f = |n: &str| dir.join(n).to_str().unwrap().to_string();
let (still, proj) = (f("still.png"), f("p.fcproj"));
let o = cli(&["--demo", "render", "--seconds", "1", "--scale", "0.1", "--out", &still]);
assert!(o.status.success(), "{}", String::from_utf8_lossy(&o.stderr));
json_out(&cli(&["exec", "file.newSequence", "name=s", "width=64", "height=36", "fps=24", "--save-as", &proj]));
let paths = format!("paths=[\"{}\"]", still.replace('\\', "\\\\"));
let item = json_out(&cli(&["--project", &proj, "--save", "exec", "file.import", &paths]))["items"][0].as_u64().unwrap();
json_out(&cli(&["--project", &proj, "--save", "exec", "timeline.place", &format!("item={item}"), "track=V1", "frame=0", "audio=false"]));
let export = |out: &str, extra: &[&str]| {
let mut args = vec!["--project", proj.as_str(), "--compact", "export", out, "--format", "png", "--start", "0", "--end", "0.05"];
args.extend_from_slice(extra);
cli(&args)
};
let on = export(&f("on.png"), &[]);
let v = json_out(&on);
assert_eq!(v["offlineMedia"], serde_json::json!([]), "{v}");
assert!(!String::from_utf8_lossy(&on.stderr).contains("warning"), "{}", String::from_utf8_lossy(&on.stderr));
std::fs::remove_file(&still).unwrap();
let off = export(&f("off.png"), &[]);
let err = String::from_utf8_lossy(&off.stderr);
assert_eq!(off.status.code(), Some(1), "{err}");
assert!(err.contains("still.png") && err.contains("missing") && err.contains("--allow-offline"), "{err}");
let v: Value = serde_json::from_slice(&off.stdout).expect("the result JSON is still printed");
assert_eq!(v["offlineMedia"][0]["item"], item, "{v}");
assert_eq!(v["offlineMedia"][0]["status"], "missing", "{v}");
let allowed = export(&f("allowed.png"), &["--allow-offline"]);
let err = String::from_utf8_lossy(&allowed.stderr);
assert!(allowed.status.success(), "{err}");
assert!(err.contains("still.png") && err.contains("missing"), "still warns: {err}");
assert_eq!(json_out(&allowed)["offlineMedia"][0]["status"], "missing");
let render = cli(&["--project", &proj, "render", "--seconds", "0", "--out", &f("r.png")]);
let err = String::from_utf8_lossy(&render.stderr);
assert_eq!(render.status.code(), Some(1), "{err}");
assert!(err.contains("still.png") && err.contains("--allow-offline"), "{err}");
let render = cli(&["--project", &proj, "render", "--seconds", "0", "--out", &f("r.png"), "--allow-offline"]);
assert!(render.status.success(), "{}", String::from_utf8_lossy(&render.stderr));
assert!(String::from_utf8_lossy(&render.stderr).contains("still.png"));
let _ = std::fs::remove_dir_all(&dir);
}

488
apps/filmcraft-cli/tests/workflows.rs Normal file
View File

@ -0,0 +1,488 @@
//! Workflow acceptance tests (docs/gaps.md G6): real editing jobs done end to end the way an agent
//! does them, through the headless MCP server of the `filmcraft-cli` binary (`filmcraft-cli mcp`,
//! JSON-RPC over stdio) and the CLI itself, on media that looks like what people shoot:
//!
//! - a phone clip: H.264 + AAC in MP4 with a variable frame rate (dropped frames);
//! - a camera clip: long-GOP H.264 (one keyframe) in Matroska with two audio tracks (AAC, AC-3).
//!
//! Each test imports, edits (extract, audio gain, title; ripple trims, colour matte, cross
//! dissolve), exports (H.264 MP4; save the project, reopen it in a new process and export ProRes)
//! and checks the file with the ffprobe / ffmpeg oracles: streams, frame count, duration, which
//! source frame lands where, A/V sync, the title's pixels, the dissolve's mix, audio levels.
//!
//! The media is generated with ffmpeg into `target/fixtures/cli/workflows` at test time (never
//! committed); without ffmpeg the tests skip (`FILMCRAFT_REQUIRE_ORACLES=1` makes that a failure).
//! ffmpeg and ffprobe are external test oracles only.
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use serde_json::{Value, json};
// ---------------------------------------------------------------------------------------------
// Fixtures
/// A phone-like clip: 4 s of 320×240 `testsrc2` at 30 fps with every frame `n % 7 == 3` dropped
/// (variable frame rate, the timestamps of the kept frames unchanged), H.264 + AAC stereo. The
/// whole picture flashes white on the frames at 1.0 s and 3.0 s, and a 1 kHz beep sounds from 1.0
/// to 1.1 s and from 3.0 to 3.1 s (silence otherwise): A/V sync markers.
fn phone_vfr_mp4(ffmpeg: &Path) -> Option<PathBuf> {
let out = filmcraft_testkit::fixtures_dir("cli/workflows").join("phone_vfr_h264_aac.mp4");
filmcraft_testkit::fixtures::generate(&out, |tmp| {
Command::new(ffmpeg)
.args(["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=30:duration=4"])
.args(["-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=48000:duration=4"])
.args([
"-filter_complex",
"[0:v]drawbox=x=0:y=0:w=iw:h=ih:color=white:t=fill:enable='between(t,1,1.02)+between(t,3,3.02)',\
select='not(eq(mod(n,7),3))'[v];\
[1:a]volume='if(between(t,1,1.1)+between(t,3,3.1),1,0)':eval=frame[a]",
])
.args(["-map", "[v]", "-map", "[a]", "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-fps_mode", "passthrough"])
.args(["-video_track_timescale", "30000", "-c:a", "aac", "-b:a", "128k", "-ac", "2"])
.arg(tmp)
.status()
.is_ok_and(|s| s.success())
})
}
/// A camera-like clip: 6 s of 320×240 `testsrc2` at 25 fps as long-GOP H.264 (a single keyframe,
/// B-frames) in Matroska, with two stereo audio tracks: 440 Hz in AAC and 1320 Hz in AC-3.
fn camera_long_gop_mkv(ffmpeg: &Path) -> Option<PathBuf> {
let out = filmcraft_testkit::fixtures_dir("cli/workflows").join("camera_long_gop_2audio.mkv");
filmcraft_testkit::fixtures::generate(&out, |tmp| {
Command::new(ffmpeg)
.args(["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=25:duration=6"])
.args(["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=6"])
.args(["-f", "lavfi", "-i", "sine=frequency=1320:sample_rate=48000:duration=6"])
.args(["-map", "0", "-map", "1", "-map", "2", "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p"])
.args(["-g", "300", "-keyint_min", "300", "-sc_threshold", "0", "-bf", "3"])
.args(["-c:a:0", "aac", "-ac:a:0", "2", "-c:a:1", "ac3", "-ac:a:1", "2"])
.arg(tmp)
.status()
.is_ok_and(|s| s.success())
})
}
/// A scratch directory under the workspace `target/`, removed when dropped.
struct Scratch(PathBuf);
impl Scratch {
fn new(name: &str) -> Self {
let d = filmcraft_testkit::workspace_root().join("target").join("workflow-tests").join(format!("{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
Scratch(d)
}
fn path(&self, name: &str) -> String {
self.0.join(name).to_string_lossy().into_owned()
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
// ---------------------------------------------------------------------------------------------
// A minimal MCP client for `filmcraft-cli mcp` (newline-delimited JSON-RPC over stdio)
struct Mcp {
child: Child,
stdin: ChildStdin,
stdout: BufReader<ChildStdout>,
next_id: u64,
}
impl Mcp {
/// Start the headless server (an empty project) and initialise the session. `data_dir` keeps
/// user presets away from the real data directory.
fn start(data_dir: &str) -> Self {
let mut child = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli"))
.args(["--data-dir", data_dir, "mcp"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.expect("spawn filmcraft-cli mcp");
let stdin = child.stdin.take().unwrap();
let stdout = BufReader::new(child.stdout.take().unwrap());
let mut m = Mcp { child, stdin, stdout, next_id: 1 };
let init =
m.request("initialize", json!({"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "workflow-tests", "version": "1"}}));
assert_eq!(init["result"]["serverInfo"]["name"], "filmcraft", "{init}");
m.send(&json!({"jsonrpc": "2.0", "method": "notifications/initialized"}));
m
}
fn send(&mut self, msg: &Value) {
writeln!(self.stdin, "{msg}").unwrap();
self.stdin.flush().unwrap();
}
/// Send a request and return its response (notifications in between are skipped).
fn request(&mut self, method: &str, params: Value) -> Value {
let id = self.next_id;
self.next_id += 1;
self.send(&json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}));
loop {
let mut line = String::new();
assert!(self.stdout.read_line(&mut line).unwrap() > 0, "the MCP server closed stdout while `{method}` was pending");
let v: Value = serde_json::from_str(&line).unwrap_or_else(|e| panic!("not JSON-RPC: {e}: {line}"));
if v["id"] == id {
return v;
}
}
}
/// Call a tool; its text result parsed as JSON. Panics on a protocol or tool error.
fn tool(&mut self, name: &str, arguments: Value) -> Value {
let r = self.request("tools/call", json!({"name": name, "arguments": arguments}));
let text = r["result"]["content"][0]["text"].as_str().unwrap_or_default().to_string();
assert!(r["error"].is_null() && r["result"]["isError"] == false, "{name} {arguments}: {r}");
serde_json::from_str(&text).unwrap_or(Value::String(text))
}
/// `command_run`: one engine command by id.
fn run(&mut self, id: &str, params: Value) -> Value {
self.tool("command_run", json!({"id": id, "params": params}))
}
}
impl Drop for Mcp {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
// ---------------------------------------------------------------------------------------------
// Oracles
fn probe(ffprobe: &Path, path: &str) -> Value {
let out = Command::new(ffprobe).args(["-v", "error", "-of", "json", "-count_frames", "-show_format", "-show_streams", path]).output().unwrap();
assert!(out.status.success(), "ffprobe {path}: {}", String::from_utf8_lossy(&out.stderr));
serde_json::from_slice(&out.stdout).unwrap()
}
fn streams<'a>(j: &'a Value, kind: &str) -> Vec<&'a Value> {
j["streams"].as_array().unwrap().iter().filter(|s| s["codec_type"] == kind).collect()
}
fn num(v: &Value) -> f64 {
v.as_str().and_then(|s| s.parse().ok()).or_else(|| v.as_f64()).unwrap_or(f64::NAN)
}
/// Run ffmpeg decoding `input` (with `pre` input options) to raw data on stdout.
fn ffmpeg_raw(ffmpeg: &Path, pre: &[&str], input: &str, args: &[&str]) -> Vec<u8> {
let out = Command::new(ffmpeg).args(["-v", "error"]).args(pre).args(["-i", input]).args(args).arg("-").output().unwrap();
assert!(out.status.success(), "ffmpeg {input}: {}", String::from_utf8_lossy(&out.stderr));
out.stdout
}
/// Size of the frames [`frames`] returns.
const FW: usize = 80;
const FH: usize = 60;
/// Every video frame of `input`, scaled to 80×60 in `pix_fmt` (`gray` or `rgb24`). `vf` goes
/// first in the filter chain (e.g. `fps=30` to see a VFR file as an editor shows it).
fn frames(ffmpeg: &Path, input: &str, vf: Option<&str>, pix_fmt: &str) -> Vec<Vec<u8>> {
let chain = match vf {
Some(f) => format!("{f},scale={FW}:{FH}:flags=area,format={pix_fmt}"),
None => format!("scale={FW}:{FH}:flags=area,format={pix_fmt}"),
};
let bpp = if pix_fmt == "gray" { 1 } else { 3 };
let raw = ffmpeg_raw(ffmpeg, &[], input, &["-map", "0:v:0", "-vf", &chain, "-fps_mode", "passthrough", "-f", "rawvideo"]);
raw.chunks_exact(FW * FH * bpp).map(<[u8]>::to_vec).collect()
}
/// Mean absolute difference of two equally sized frames.
fn diff(a: &[u8], b: &[u8]) -> f64 {
a.iter().zip(b).map(|(x, y)| f64::from(x.abs_diff(*y))).sum::<f64>() / a.len().max(1) as f64
}
/// Assert that `got` shows source frame `want`: it is (one of) the closest source frames, and close.
fn assert_shows(got: &[u8], src: &[Vec<u8>], want: usize, what: &str) {
let d: Vec<f64> = src.iter().map(|s| diff(got, s)).collect();
let best = d.iter().copied().fold(f64::INFINITY, f64::min);
let at = d.iter().position(|x| *x == best).unwrap_or(0);
assert!(d[want] < 6.0 && d[want] <= best + 0.25, "{what}: expected source frame {want} (diff {:.2}), closest is {at} (diff {best:.2})", d[want]);
}
/// The first channel of audio stream `stream` of `input`, mono-mixed, as f32 at 48 kHz.
fn pcm(ffmpeg: &Path, input: &str, stream: usize) -> Vec<f32> {
let map = format!("0:a:{stream}");
let raw = ffmpeg_raw(ffmpeg, &[], input, &["-map", &map, "-ac", "1", "-ar", "48000", "-f", "f32le"]);
raw.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
}
/// Times (s) where the signal rises above a quarter of its peak, at least 0.2 s apart.
fn onsets(x: &[f32]) -> Vec<f64> {
let peak = x.iter().fold(0f32, |m, v| m.max(v.abs()));
let mut out = Vec::new();
let mut last = f64::NEG_INFINITY;
for (i, block) in x.chunks(48).enumerate() {
let t = i as f64 * 48.0 / 48_000.0;
if block.iter().any(|v| v.abs() > peak / 4.0) && t - last > 0.2 {
out.push(t);
}
if block.iter().any(|v| v.abs() > peak / 4.0) {
last = t;
}
}
out
}
fn rms(x: &[f32], from: f64, to: f64) -> f64 {
let s = x.get((from * 48_000.0) as usize..((to * 48_000.0) as usize).min(x.len())).unwrap_or_default();
(s.iter().map(|v| f64::from(*v) * f64::from(*v)).sum::<f64>() / s.len().max(1) as f64).sqrt()
}
/// Amplitude of the `freq` component of 100 ms of `x` from `at` seconds (Goertzel).
fn tone(x: &[f32], at: f64, freq: f64) -> f64 {
let start = (at * 48_000.0) as usize;
let s = x.get(start..(start + 4800).min(x.len())).unwrap_or_default();
let c = 2.0 * (2.0 * std::f64::consts::PI * freq / 48_000.0).cos();
let (mut s1, mut s2) = (0.0, 0.0);
for v in s {
let s0 = f64::from(*v) + c * s1 - s2;
s2 = s1;
s1 = s0;
}
(s1 * s1 + s2 * s2 - c * s1 * s2).max(0.0).sqrt() * 2.0 / s.len().max(1) as f64
}
fn db(ratio: f64) -> f64 {
20.0 * ratio.log10()
}
fn clips_on<'a>(seq: &'a Value, kind: &str, track: usize) -> &'a Vec<Value> {
seq[kind][track]["items"].as_array().unwrap()
}
// ---------------------------------------------------------------------------------------------
// Workflows
/// Phone footage: import a VFR H.264/AAC clip, make a sequence from it, extract a second from the
/// middle (Mark In / Mark Out / Extract), lower the audio 6 dB, put a title over the start, and
/// export H.264 MP4 through MCP. The VFR clip plays at its 30 fps base rate (a dropped frame
/// holds the one before), the cut lands on the right frames, picture and sound stay in sync
/// across the cut, the title is in the picture only while it is on the timeline.
#[test]
fn phone_vfr_clip_extract_gain_title_to_h264_mp4() {
let (ffmpeg, ffprobe) = filmcraft_testkit::require_oracles!();
let Some(src) = phone_vfr_mp4(&ffmpeg) else {
return filmcraft_testkit::skip("phone workflow", "ffmpeg could not make the VFR H.264/AAC fixture (libx264?)");
};
let src = src.to_string_lossy().into_owned();
let dir = Scratch::new("phone");
let mut m = Mcp::start(&dir.path("data"));
let imported = m.tool("media_import", json!({"paths": [src]}));
assert_eq!(imported["errors"], json!([]), "{imported}");
let item = imported["items"][0].as_u64().unwrap();
let made = m.run("file.newSequenceFromClip", json!({"items": [item]}));
assert!(made["sequence"].is_u64(), "{made}");
let seq = m.tool("sequence_inspect", json!({}));
assert_eq!((seq["settings"]["width"].as_u64(), seq["settings"]["height"].as_u64()), (Some(320), Some(240)), "{}", seq["settings"]);
assert_eq!(seq["settings"]["frame_rate"], json!({"num": 30, "den": 1}), "a VFR phone clip makes a sequence at its base rate");
assert_eq!(seq["durationFrames"], 120, "4 s at 30 fps");
// Extract 1.5 s .. 2.5 s; Mark Out includes its frame, so 31 frames go and 89 remain.
m.run("markers.markIn", json!({"seconds": 1.5}));
m.run("markers.markOut", json!({"seconds": 2.5}));
m.run("sequence.extract", json!({}));
m.run("edit.selectAll", json!({}));
let gain = m.run("clip.audioGain", json!({"mode": "set", "db": -6.0}));
assert_eq!(gain["clips"], 2, "both audio pieces: {gain}");
m.run("edit.deselectAll", json!({}));
m.run("graphics.newText", json!({"text": "HELLO", "size": 40, "time": 0, "seconds": 0.5}));
let seq = m.tool("sequence_inspect", json!({}));
assert_eq!(seq["durationFrames"], 89, "{seq}");
assert_eq!(clips_on(&seq, "video", 0).len(), 2);
assert_eq!(clips_on(&seq, "audio", 0).len(), 2);
let out = dir.path("phone_edit.mp4");
let r = m.run("file.exportMedia", json!({"path": out, "format": "h264", "range": "entire", "wait": true}));
assert_eq!(r["result"]["frames"], 89, "{r}");
// container and streams
let j = probe(&ffprobe, &out);
let (v, a) = (streams(&j, "video"), streams(&j, "audio"));
assert_eq!((v.len(), a.len()), (1, 1), "{j}");
assert_eq!((v[0]["codec_name"].as_str(), v[0]["width"].as_u64(), v[0]["height"].as_u64()), (Some("h264"), Some(320), Some(240)));
assert_eq!(v[0]["r_frame_rate"], "30/1");
assert_eq!(v[0]["nb_read_frames"], "89");
assert_eq!((a[0]["codec_name"].as_str(), a[0]["sample_rate"].as_str(), a[0]["channels"].as_u64()), (Some("aac"), Some("48000"), Some(2)));
let duration = num(&j["format"]["duration"]);
assert!((duration - 89.0 / 30.0).abs() < 0.06, "duration {duration}");
assert!((num(&a[0]["duration"]) - 89.0 / 30.0).abs() < 0.06, "audio as long as the picture: {}", a[0]["duration"]);
// picture: frame k of the export is frame k (before the cut) or k + 31 (after) of the source
// as an editor plays it (CFR 30, a dropped frame holding the previous one)
let src_frames = frames(&ffmpeg, &src, Some("fps=30"), "gray");
let got = frames(&ffmpeg, &out, None, "gray");
assert!(src_frames.len() >= 120, "{} source frames", src_frames.len());
assert_eq!(got.len(), 89);
for k in [20, 32, 40, 44, 45, 50, 70, 88] {
let want = if k < 45 { k } else { k + 31 };
assert_shows(&got[k], &src_frames, want, &format!("export frame {k}"));
}
// the flash frames: 30 (with 31 holding it, as the source dropped its frame 31) and 90 - 31
let mean = |f: &Vec<u8>| f.iter().map(|v| f64::from(*v)).sum::<f64>() / f.len() as f64;
let flashes: Vec<usize> = got.iter().enumerate().filter(|(_, f)| mean(f) > 200.0).map(|(i, _)| i).collect();
assert_eq!(flashes, [30, 31, 59], "white frames");
// the title: over frames 0..15 only (its pixels differ from the source there, nowhere after)
let blocks_changed = |k: usize| {
let mut n = 0;
for by in (0..FH).step_by(10) {
for bx in (0..FW).step_by(10) {
let mut d = 0.0;
for y in by..by + 10 {
for x in bx..bx + 10 {
d += f64::from(got[k][y * FW + x].abs_diff(src_frames[k][y * FW + x]));
}
}
n += usize::from(d / 100.0 > 15.0);
}
}
n
};
for k in [2, 8, 14] {
assert!(blocks_changed(k) >= 2, "title missing on frame {k}");
}
for k in [16, 25] {
assert_eq!(blocks_changed(k), 0, "title still on frame {k}");
}
// sound: beeps where the flashes are (A/V sync across the extract), 6 dB down
let src_pcm = pcm(&ffmpeg, &src, 0);
let got_pcm = pcm(&ffmpeg, &out, 0);
let beeps = onsets(&got_pcm);
assert_eq!(beeps.len(), 2, "beeps at {beeps:?}");
for (beep, flash) in beeps.iter().zip([30.0 / 30.0, 59.0 / 30.0]) {
assert!((beep - flash).abs() <= 1.0 / 30.0, "beep at {beep:.3} s, flash at {flash:.3} s");
}
let change = db(rms(&got_pcm, 1.01, 1.09) / rms(&src_pcm, 1.01, 1.09));
assert!((change + 6.0).abs() < 1.0, "audio gain -6 dB measured {change:.2} dB");
let change = db(rms(&got_pcm, 1.98, 2.06) / rms(&src_pcm, 3.01, 3.09));
assert!((change + 6.0).abs() < 1.0, "second beep {change:.2} dB");
}
/// Camera footage: import a long-GOP H.264 Matroska file with two audio tracks through MCP, make
/// a sequence (both tracks land on A1 and A2), ripple-trim 2.6 s off the head (decoding starts at
/// the only keyframe) and 15 frames off the tail, add a red colour matte with a centred cross
/// dissolve, save the project; then a new `filmcraft-cli` process opens it and exports ProRes.
/// The first exported frame is source frame 65, the dissolve mixes the two pictures, both audio
/// tracks are in the mix at their own level and stop where the clip does.
#[test]
fn camera_long_gop_two_audio_tracks_trim_dissolve_to_prores() {
let (ffmpeg, ffprobe) = filmcraft_testkit::require_oracles!();
let Some(src) = camera_long_gop_mkv(&ffmpeg) else {
return filmcraft_testkit::skip("camera workflow", "ffmpeg could not make the long-GOP Matroska fixture (libx264?)");
};
let src = src.to_string_lossy().into_owned();
let dir = Scratch::new("camera");
let data = dir.path("data");
let project = dir.path("camera.fcproj");
{
let mut m = Mcp::start(&data);
let imported = m.tool("media_import", json!({"paths": [src]}));
assert_eq!(imported["errors"], json!([]), "{imported}");
let item = imported["items"][0].as_u64().unwrap();
let made = m.run("file.newSequenceFromClip", json!({"items": [item]}));
let clips: Vec<u64> = made["clips"].as_array().unwrap().iter().filter_map(Value::as_u64).collect();
assert_eq!(clips.len(), 3, "a video clip and one clip per audio track: {made}");
let seq = m.tool("sequence_inspect", json!({}));
assert_eq!(seq["settings"]["frame_rate"], json!({"num": 25, "den": 1}));
assert_eq!(seq["audio"].as_array().map(Vec::len), Some(2), "two audio tracks");
let video = clips_on(&seq, "video", 0)[0]["clip"].as_u64().unwrap();
// 65 frames off the head and 15 off the tail, the linked audio following
m.run("timeline.trim", json!({"clip": video, "edge": "in", "mode": "ripple", "deltaFrames": 65}));
m.run("timeline.trim", json!({"clip": video, "edge": "out", "mode": "ripple", "deltaFrames": -15}));
let seq = m.tool("sequence_inspect", json!({}));
let end = clips_on(&seq, "video", 0)[0]["endFrame"].as_u64().unwrap();
for t in 0..2 {
let a = &clips_on(&seq, "audio", t)[0];
assert_eq!((a["startFrame"].as_u64(), a["endFrame"].as_u64(), a["sourceIn"].as_i64()), (Some(0), Some(end), Some(660_441_600_000)), "A{}", t + 1);
}
let matte = m.run("file.newColorMatte", json!({"color": "#ff0000", "seconds": 2.0}))["item"].as_u64().unwrap();
let placed = m.run("timeline.place", json!({"item": matte, "track": "V1", "frame": end, "insert": false}));
let matte_clip = placed["clips"][0].as_u64().unwrap();
m.run("sequence.applyVideoTransition", json!({"clip": matte_clip, "edge": "in", "frames": 10, "effect": "cross_dissolve"}));
let seq = m.tool("sequence_inspect", json!({}));
let tr = &seq["video"][0]["transitions"][0];
assert_eq!((tr["effect"].as_str(), tr["align"].as_str()), (Some("cross_dissolve"), Some("center")), "{tr}");
m.run("file.saveAs", json!({"path": project}));
}
assert_eq!(end_frame_of(&project), Some(71), "the saved project holds the edit");
// a fresh process opens the saved project and renders it
let out = dir.path("camera_edit.mov");
let o = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(["--data-dir", &data, "--project", &project, "export", &out]).output().unwrap();
assert!(o.status.success(), "export: {}", String::from_utf8_lossy(&o.stderr));
let r: Value = serde_json::from_slice(&o.stdout).unwrap();
let total = 71 + 50;
assert_eq!(r["result"]["frames"], total, "{r}");
let j = probe(&ffprobe, &out);
let (v, a) = (streams(&j, "video"), streams(&j, "audio"));
assert_eq!((v.len(), a.len()), (1, 1), "{j}");
assert_eq!((v[0]["codec_name"].as_str(), v[0]["width"].as_u64(), v[0]["height"].as_u64()), (Some("prores"), Some(320), Some(240)));
assert_eq!(v[0]["r_frame_rate"], "25/1");
assert_eq!(v[0]["nb_read_frames"], total.to_string());
assert_eq!((a[0]["sample_rate"].as_str(), a[0]["channels"].as_u64()), (Some("48000"), Some(2)));
let duration = num(&j["format"]["duration"]);
assert!((duration - f64::from(total) / 25.0).abs() < 0.05, "duration {duration}");
// picture: frame k is source frame 65 + k up to the dissolve (frames 66..76), red after it
let src_frames = frames(&ffmpeg, &src, None, "rgb24");
let got = frames(&ffmpeg, &out, None, "rgb24");
assert_eq!((src_frames.len(), got.len()), (150, total as usize));
for k in [0, 1, 2, 30, 64] {
assert_shows(&got[k], &src_frames, 65 + k, &format!("export frame {k}"));
}
let red = &got[100];
assert!(red.as_chunks::<3>().0.iter().all(|[r, g, b]| *r > 230 && *g < 25 && *b < 25), "the matte is red");
// the dissolve: frame = (1 - a)·camera + a·red, `a` rising through about 0.5 at the cut
let mix = |k: usize| {
let (cam, e) = (&src_frames[65 + k], &got[k]);
let (mut num, mut den) = (0.0, 0.0);
for ((c, r), e) in cam.iter().zip(red).zip(e) {
let (c, r, e) = (f64::from(*c), f64::from(*r), f64::from(*e));
num += (e - c) * (r - c);
den += (r - c) * (r - c);
}
num / den
};
let alphas: Vec<f64> = (64..=78).map(mix).collect();
assert!(alphas[0].abs() < 0.05 && (alphas[14] - 1.0).abs() < 0.05, "outside the dissolve: {alphas:?}");
assert!(alphas.windows(2).all(|w| w[1] >= w[0] - 0.02), "the dissolve progresses: {alphas:?}");
assert!((alphas[7] - 0.5).abs() < 0.12, "half way at the cut (frame 71): {alphas:?}");
// sound: both tracks in the mix at their own level while the clip plays, silence on the matte
let got_pcm = pcm(&ffmpeg, &out, 0);
let (t1, t2) = (pcm(&ffmpeg, &src, 0), pcm(&ffmpeg, &src, 1));
for at in [0.2, 1.5, 2.6] {
let (low, high) = (tone(&got_pcm, at, 440.0), tone(&got_pcm, at, 1320.0));
let (want_low, want_high) = (tone(&t1, at + 2.6, 440.0), tone(&t2, at + 2.6, 1320.0));
assert!(
db(low / want_low).abs() < 1.0 && db(high / want_high).abs() < 1.0,
"at {at} s: 440 Hz {low:.4} (want {want_low:.4}), 1320 Hz {high:.4} (want {want_high:.4})"
);
}
assert!(rms(&got_pcm, 3.0, 4.8) < 1e-3, "silence under the matte");
}
/// `endFrame` of the first clip on V1 of the saved project's active sequence, read back through
/// `filmcraft-cli --project … inspect sequence`.
fn end_frame_of(project: &str) -> Option<u64> {
let o = Command::new(env!("CARGO_BIN_EXE_filmcraft-cli")).args(["--project", project, "inspect", "sequence"]).output().ok()?;
let v: Value = serde_json::from_slice(&o.stdout).ok()?;
v["video"][0]["items"][0]["endFrame"].as_u64()
}

2
apps/filmcraft/Cargo.toml
View File

@ -41,6 +41,8 @@ open = "5"
[dev-dependencies]
glob = "0.3"
# graphics.rs: enumerate the real adapters in a test.
pollster = "0.4"
[target.'cfg(target_os = "macos")'.dependencies]
muda = "0.21"

174
apps/filmcraft/src/args.rs
View File

@ -1,5 +1,8 @@
//! The desktop app's command line (`filmcraft --help`).
use std::ffi::OsString;
use std::path::{Path, PathBuf};
/// `filmcraft --help`.
pub const USAGE: &str = "\
Usage: filmcraft [options] [project.fcproj | media files...]
@ -7,12 +10,17 @@ Usage: filmcraft [options] [project.fcproj | media files...]
Options:
--control <port> start the localhost JSON-lines control server
(also: FILMCRAFT_CONTROL_PORT)
--control-input-only accept widget input only from the control server
(requires --control or FILMCRAFT_CONTROL_PORT)
--demo open the demo project
--empty start with an empty project
--recover recover the newest unsaved changes without asking
--no-recover start without asking about unsaved changes
--data-dir <dir> folder for auto-save, crash recovery and logs
(also: FILMCRAFT_DATA_DIR)
--graphics <mode> start with this graphics mode and keep it: default,
dx12 (Windows), software (Windows, Linux);
`--graphics default` undoes an automatic fallback
--version print the version and exit
-h, --help print this help and exit
";
@ -26,9 +34,10 @@ pub enum Cli {
}
/// Options of a normal start.
#[derive(Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq)]
pub struct Launch {
pub control_port: Option<u16>,
pub control_input_only: bool,
/// A project and / or media files to open.
pub files: Vec<String>,
pub demo: bool,
@ -36,6 +45,8 @@ pub struct Launch {
pub startup_flag: bool,
pub recover: Option<bool>,
pub data_dir: Option<std::path::PathBuf>,
/// `--graphics <mode>`: start with this graphics mode and keep it (see `graphics.rs`).
pub graphics: Option<crate::graphics::Mode>,
}
/// Parse the arguments after the program name. `env_control_port` is the value of
@ -45,29 +56,49 @@ pub struct Launch {
/// mistyped flag helps nobody, and starting without the control server that was asked for leaves
/// a script waiting for it. That holds for the whole line, so `--version --bogus` is an error
/// too, and for a port from the environment exactly as for one from `--control`.
pub fn parse(env_control_port: Option<String>, args: impl IntoIterator<Item = String>) -> Result<Cli, String> {
let mut l = Launch { control_port: None, files: Vec::new(), demo: true, startup_flag: false, recover: None, data_dir: None };
///
/// The arguments come from `std::env::args_os`: not every argument is Unicode (a Linux file name
/// in Latin-1), and `std::env::args` panics on one. A file name that is not is passed on lossily
/// (its import then reports it missing); `--data-dir` keeps the folder exactly.
pub fn parse(env_control_port: Option<String>, args: impl IntoIterator<Item = OsString>) -> Result<Cli, String> {
let mut l = Launch {
control_port: None,
control_input_only: false,
files: Vec::new(),
demo: true,
startup_flag: false,
recover: None,
data_dir: None,
graphics: None,
};
let (mut help, mut version) = (false, false);
let mut args = args.into_iter();
while let Some(a) = args.next() {
match a.as_str() {
while let Some(raw) = args.next() {
let a = raw.to_string_lossy();
match a.as_ref() {
"-h" | "--help" => help = true,
"--version" => version = true,
"--control" => {
let port = args.next().ok_or("--control needs a port number")?;
let port = port.to_string_lossy();
l.control_port = Some(port.parse().map_err(|_| format!("--control needs a port number, got `{port}`"))?);
}
"--demo" => (l.demo, l.startup_flag) = (true, true),
"--control-input-only" => l.control_input_only = true,
"--empty" => (l.demo, l.startup_flag) = (false, true),
"--recover" => l.recover = Some(true),
"--no-recover" => l.recover = Some(false),
"--data-dir" => l.data_dir = Some(args.next().map(std::path::PathBuf::from).ok_or("--data-dir needs a folder")?),
"--data-dir" => l.data_dir = Some(args.next().map(PathBuf::from).ok_or("--data-dir needs a folder")?),
"--graphics" => {
let name = args.next().ok_or("--graphics needs a mode: default, dx12 or software")?;
l.graphics = Some(graphics_mode(&name.to_string_lossy())?);
}
// everything after `--` is a file, whatever it looks like
"--" => l.files.extend(args.by_ref()),
"--" => l.files.extend(args.by_ref().map(|f| f.to_string_lossy().into_owned())),
// macOS adds a process serial number when it launches an app bundle
_ if a.starts_with("-psn_") => {}
_ if a.len() > 1 && a.starts_with('-') => return Err(format!("unknown option `{a}`")),
_ => l.files.push(a),
_ => l.files.push(a.into_owned()),
}
}
if help {
@ -81,15 +112,51 @@ pub fn parse(env_control_port: Option<String>, args: impl IntoIterator<Item = St
{
l.control_port = Some(port.parse().map_err(|_| format!("FILMCRAFT_CONTROL_PORT needs a port number, got `{port}`"))?);
}
if l.control_input_only && l.control_port.is_none() {
return Err("--control-input-only requires --control or FILMCRAFT_CONTROL_PORT".into());
}
Ok(Cli::Run(l))
}
/// Whether a launch file is a FilmCraft project. The extension's case does not matter: Windows file
/// associations and the Linux MIME glob start the app for `CUT.FCPROJ` too (#850).
pub fn is_project(path: &str) -> bool {
Path::new(path).extension().is_some_and(|e| e.eq_ignore_ascii_case("fcproj"))
}
/// `path` made absolute against the working directory, so a project or media file named relative
/// to a terminal's folder is still found once the app runs from another one (Recent Projects, the
/// media paths saved in the project). Unlike `canonicalize`, `std::path::absolute` keeps symlinks
/// and adds no Windows verbatim `\\?\` prefix. A path it cannot resolve is kept as given.
pub fn absolute(path: &str) -> String {
std::path::absolute(path).ok().and_then(|p| p.into_os_string().into_string().ok()).unwrap_or_else(|| path.to_string())
}
/// The launch files as the project to open (the first one, see [`is_project`]) and the media files
/// to import (every file that is not a project), all made [`absolute`].
pub fn project_and_media(files: &[String]) -> (Option<String>, Vec<String>) {
let project = files.iter().find(|f| is_project(f)).map(|f| absolute(f));
let media = files.iter().filter(|f| !is_project(f)).map(|f| absolute(f)).collect();
(project, media)
}
/// A `--graphics` value: one of this platform's modes.
fn graphics_mode(name: &str) -> Result<crate::graphics::Mode, String> {
use crate::graphics::{LADDER, Mode};
let here = || LADDER.iter().map(|m| m.name()).collect::<Vec<_>>().join(", ");
let mode = Mode::parse(name).ok_or_else(|| format!("--graphics needs one of {}, got `{name}`", here()))?;
if !LADDER.contains(&mode) {
return Err(format!("--graphics {name} is not available on this platform (use one of {})", here()));
}
Ok(mode)
}
#[cfg(test)]
mod tests {
use super::*;
fn parse_env(env: Option<&str>, args: &[&str]) -> Result<Cli, String> {
parse(env.map(str::to_string), args.iter().map(|a| a.to_string()))
parse(env.map(str::to_string), args.iter().map(OsString::from))
}
fn parse_args(args: &[&str]) -> Result<Cli, String> {
@ -157,7 +224,19 @@ mod tests {
#[test]
fn known_options_and_files_parse_as_before() {
let none = launch(&[]);
assert_eq!(none, Launch { control_port: None, files: vec![], demo: true, startup_flag: false, recover: None, data_dir: None });
assert_eq!(
none,
Launch {
control_port: None,
control_input_only: false,
files: vec![],
demo: true,
startup_flag: false,
recover: None,
data_dir: None,
graphics: None
}
);
let l = launch(&["--control", "9876", "--empty", "--no-recover", "--data-dir", "/tmp/fc", "cut.fcproj", "a.mp4"]);
assert_eq!(l.control_port, Some(9876));
assert_eq!((l.demo, l.startup_flag, l.recover), (false, true, Some(false)));
@ -168,4 +247,79 @@ mod tests {
assert_eq!(launch(&["--", "--odd name.mov", "-h"]).files, ["--odd name.mov", "-h"]);
assert_eq!(launch(&["-", "-psn_0_12345"]).files, ["-"]);
}
/// `std::env::args` panics on an argument that is not Unicode, so such a file name (Latin-1 on
/// Linux) stopped the app before its window opened.
#[cfg(unix)]
#[test]
fn arguments_that_are_not_unicode_still_start_the_app() {
use std::os::unix::ffi::OsStringExt;
let dir = OsString::from_vec(b"/tmp/d\xe9".to_vec());
let file = OsString::from_vec(b"caf\xe9.mov".to_vec());
let Ok(Cli::Run(l)) = parse(None, [OsString::from("--data-dir"), dir.clone(), file.clone(), OsString::from("--"), file]) else {
panic!("should start the app");
};
assert_eq!(l.data_dir, Some(PathBuf::from(dir)), "the data folder is kept exactly");
assert_eq!(l.files, ["caf\u{fffd}.mov", "caf\u{fffd}.mov"]);
}
/// Windows file associations and the Linux MIME glob match `CUT.FCPROJ` too; it used to be
/// imported as a media file instead of opened (#850).
#[test]
fn a_project_is_recognised_whatever_the_case_of_its_extension() {
for p in ["cut.fcproj", "CUT.FCPROJ", "Cut.FcProj", "/edits/a b.fcproj", r"C:\Edits\CUT.FCPROJ"] {
assert!(is_project(p), "{p}");
}
for p in ["cut.mp4", "fcproj", ".fcproj", "a.fcproj.mov", "a.fcprojx", ""] {
assert!(!is_project(p), "{p}");
}
}
/// Relative launch paths were opened and imported as given, so the project's media paths and
/// Recent Projects pointed nowhere once the app ran from another folder (#850).
#[test]
fn launch_files_are_made_absolute_and_split_case_insensitively() {
let cwd = std::env::current_dir().unwrap();
let nested = Path::new("clips").join("a.mp4");
let already = cwd.join("b.wav").to_string_lossy().into_owned();
let files = ["Cut.FCPROJ".to_string(), nested.to_string_lossy().into_owned(), "other.fcproj".to_string(), already.clone()];
let (project, media) = project_and_media(&files);
assert_eq!(project, Some(cwd.join("Cut.FCPROJ").to_string_lossy().into_owned()));
assert_eq!(media, [cwd.join(&nested).to_string_lossy().into_owned(), already]);
for p in media.iter().chain(&project) {
assert!(Path::new(p).is_absolute() && !p.starts_with(r"\\?\"), "{p}");
}
assert_eq!(project_and_media(&[]), (None, vec![]));
// nothing to resolve against: kept as given rather than dropped
assert_eq!(absolute(""), "");
}
#[test]
fn isolated_input_is_explicit_and_requires_a_control_server() {
assert!(!launch(&["--control", "9876"]).control_input_only);
assert!(parse_args(&["--control-input-only"]).is_err());
assert!(launch(&["--control-input-only", "--control", "9876"]).control_input_only);
assert!(matches!(parse_env(Some("9876"), &["--control-input-only"]), Ok(Cli::Run(Launch { control_input_only: true, control_port: Some(9876), .. }))));
assert_eq!(parse_args(&["--control-input-only", "--help"]), Ok(Cli::Help));
assert!(USAGE.contains("--control-input-only"));
}
#[test]
fn graphics_picks_one_of_this_platforms_modes() {
use crate::graphics::{LADDER, Mode};
assert_eq!(launch(&[]).graphics, None);
assert_eq!(launch(&["--graphics", "default"]).graphics, Some(Mode::Default));
for mode in LADDER {
assert_eq!(launch(&["--graphics", mode.name()]).graphics, Some(*mode));
}
// a mode this platform has not is refused, not ignored
for mode in [Mode::Dx12, Mode::Software].into_iter().filter(|m| !LADDER.contains(m)) {
assert!(parse_args(&["--graphics", mode.name()]).unwrap_err().contains("not available"));
}
assert!(parse_args(&["--graphics"]).is_err(), "missing mode");
assert!(parse_args(&["--graphics", "vulkan"]).unwrap_err().contains("vulkan"));
assert!(parse_args(&["--graphics", "DEFAULT"]).is_err());
assert!(parse_args(&["--graphics", &"x".repeat(1 << 16)]).is_err());
assert!(USAGE.contains("--graphics"));
}
}

347
apps/filmcraft/src/audio.rs
View File

@ -1,18 +1,33 @@
//! cpal audio output: the playback master clock. Settings ▸ Audio Hardware picks the host
//! ("Device Class"), the output device, the I/O buffer size and the sample rate.
//!
//! A stream the device drops while playing (an ALSA xrun the backend does not recover from, which
//! cpal reports as `poll() returned POLLERR`; an unplugged device) is restarted by
//! [`AudioOut::recover`], keeping the played-frame clock and the play-ahead mixer, so sound
//! resumes where it stopped instead of playback going silent (#840).
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, Mutex, PoisonError, TryLockError};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use filmcraft_engine::settings::AudioHardwarePrefs;
use filmcraft_ui_egui::{AudioDevices, AudioOut};
/// The playback mixer callback handed to [`AudioOut::start`].
type Fill = Box<dyn FnMut(&mut [f32], usize) + Send>;
pub struct CpalOut {
stream: Option<cpal::Stream>,
/// The running playback's mixer, shared with the stream so a restarted stream continues it.
fill: Option<Arc<Mutex<Fill>>>,
played: Arc<AtomicU64>,
failed: Arc<AtomicBool>,
health: Arc<Health>,
watchdog: Watchdog,
/// Time base for the watchdog.
epoch: Instant,
rate: u32,
channels: u16,
hw: AudioHardwarePrefs,
@ -128,8 +143,11 @@ impl CpalOut {
};
Self {
stream: None,
fill: None,
played: Arc::new(AtomicU64::new(0)),
failed: Arc::new(AtomicBool::new(false)),
health: Arc::new(Health::default()),
watchdog: Watchdog::default(),
epoch: Instant::now(),
rate: cfg.as_ref().map_or(48_000, |c| c.sample_rate().0),
channels: cfg.as_ref().map_or(2, |c| c.channels()),
hw: AudioHardwarePrefs::default(),
@ -153,19 +171,14 @@ impl CpalOut {
Ok(exact.unwrap_or(default))
}
fn converted_stream<T>(
&self,
dev: &cpal::Device,
config: &cpal::StreamConfig,
mut fill: Box<dyn FnMut(&mut [f32], usize) + Send>,
) -> Result<cpal::Stream, cpal::BuildStreamError>
fn converted_stream<T>(&self, dev: &cpal::Device, config: &cpal::StreamConfig, fill: Arc<Mutex<Fill>>) -> Result<cpal::Stream, cpal::BuildStreamError>
where
T: cpal::SizedSample + cpal::FromSample<f32>,
{
let channels = usize::from(config.channels);
let played = self.played.clone();
let failed = self.failed.clone();
let error_flag = failed.clone();
let health = self.health.clone();
let error_health = health.clone();
let mut scratch = Vec::new();
dev.build_output_stream(
config,
@ -173,39 +186,23 @@ impl CpalOut {
// Reuse the conversion buffer. A pathological driver buffer must not panic the audio thread.
if buf.len() > 1_048_576 || scratch.try_reserve(buf.len().saturating_sub(scratch.len())).is_err() {
buf.fill(T::EQUILIBRIUM);
if !failed.swap(true, Ordering::SeqCst) {
if !health.failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: unable to allocate the audio conversion buffer");
}
return;
}
scratch.resize(buf.len(), 0.0);
fill(&mut scratch, channels);
render(&fill, &mut scratch, channels, &played, &health);
convert_samples(&scratch, buf);
played.fetch_add((buf.len() / channels) as u64, Ordering::SeqCst);
},
move |e| stream_error(&error_flag, e),
move |e| stream_error(&error_health, e),
None,
)
}
}
fn stream_error(failed: &AtomicBool, e: cpal::StreamError) {
if !failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: audio stream error: {e}");
}
}
fn convert_samples<T: cpal::Sample + cpal::FromSample<f32>>(input: &[f32], output: &mut [T]) {
// Integer conversion represents [-1, 1). In particular I24 cannot represent +1 exactly.
let upper = f32::from_bits(1.0_f32.to_bits() - 1);
for (sample, converted) in input.iter().zip(output) {
*converted = T::from_sample(if sample.is_finite() { sample.clamp(-1.0, upper) } else { 0.0 });
}
}
impl AudioOut for CpalOut {
fn start(&mut self, mut fill: Box<dyn FnMut(&mut [f32], usize) + Send>) -> Result<u32, String> {
self.stop();
/// Open the configured output device and play `fill` on it. `expect` is the (sample rate,
/// channels) a restarted stream must keep: the mixer was set up for them.
fn open(&mut self, fill: Arc<Mutex<Fill>>, expect: Option<(u32, u16)>) -> Result<(), String> {
let host = host(&self.hw.device_class);
let DeviceChoice { device: dev, fallback } = output_device(&host, &self.hw.default_output, &self.fallback).ok_or("no output device")?;
self.note = fallback;
@ -214,6 +211,11 @@ impl AudioOut for CpalOut {
if channels == 0 || cfg.sample_rate().0 == 0 {
return Err("the output device returned an invalid channel count or sample rate".into());
}
if let Some((rate, ch)) = expect
&& (rate, ch) != (cfg.sample_rate().0, cfg.channels())
{
return Err(format!("the output device now runs at {} Hz with {} channels", cfg.sample_rate().0, cfg.channels()));
}
let mut config: cpal::StreamConfig = cfg.clone().into();
if let cpal::SupportedBufferSize::Range { min, max } = cfg.buffer_size()
&& (*min..=*max).contains(&self.hw.buffer_size)
@ -222,20 +224,18 @@ impl AudioOut for CpalOut {
}
self.rate = cfg.sample_rate().0;
self.channels = cfg.channels();
self.played.store(0, Ordering::SeqCst);
self.failed.store(false, Ordering::SeqCst);
let played = self.played.clone();
let failed = self.failed.clone();
let stream = match cfg.sample_format() {
cpal::SampleFormat::F32 => dev.build_output_stream(
&config,
move |buf: &mut [f32], _| {
fill(buf, channels);
played.fetch_add((buf.len() / channels) as u64, Ordering::SeqCst);
},
move |e| stream_error(&failed, e),
None,
),
cpal::SampleFormat::F32 => {
let played = self.played.clone();
let health = self.health.clone();
let error_health = health.clone();
dev.build_output_stream(
&config,
move |buf: &mut [f32], _| render(&fill, buf, channels, &played, &health),
move |e| stream_error(&error_health, e),
None,
)
}
cpal::SampleFormat::I8 => self.converted_stream::<i8>(&dev, &config, fill),
cpal::SampleFormat::I16 => self.converted_stream::<i16>(&dev, &config, fill),
cpal::SampleFormat::I24 => self.converted_stream::<cpal::I24>(&dev, &config, fill),
@ -251,10 +251,172 @@ impl AudioOut for CpalOut {
.map_err(|e| e.to_string())?;
stream.play().map_err(|e| e.to_string())?;
self.stream = Some(stream);
Ok(())
}
/// Replace a dropped stream with a new one on the same mixer, clock and format.
fn restart(&mut self) -> Result<(), String> {
let fill = self.fill.clone().ok_or("nothing is playing")?;
let expect = (self.rate, self.channels);
// The old stream (and its audio thread) goes first: only one stream ever runs the mixer.
self.stream = None;
self.health.clear_errors();
self.open(fill, Some(expect))
}
}
/// What the audio thread reports to the UI thread.
#[derive(Default)]
struct Health {
/// The device went away (unplugged): restart the stream at once.
lost: AtomicBool,
/// A backend error since the last check (cpal keeps the stream alive after these): restart
/// only if the stream also stopped consuming samples.
errored: AtomicBool,
/// Unrecoverable: the mixer panicked, a buffer could not be allocated, or restarts ran out.
/// [`AudioOut::played_frames`] reports `None` and playback goes on without sound.
failed: AtomicBool,
/// The first error of a stream has been printed.
reported: AtomicBool,
}
impl Health {
fn clear_errors(&self) {
self.lost.store(false, Ordering::SeqCst);
self.errored.store(false, Ordering::SeqCst);
self.reported.store(false, Ordering::SeqCst);
}
}
/// How long a stream that reported an error may go without consuming samples before it is
/// restarted. Well under the UI's `AUDIO_STALL_S`, which gives up on the device clock.
const STALL_RESTART: Duration = Duration::from_millis(300);
/// At most this many restarts within [`RESTART_WINDOW`]; a device that keeps failing gives up.
const MAX_RESTARTS: usize = 3;
const RESTART_WINDOW: Duration = Duration::from_secs(10);
/// What [`Watchdog::check`] decides about the running stream.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Verdict {
Healthy,
Restart,
GiveUp,
}
/// Decides when a stream that reported errors is restarted (UI thread, every frame).
#[derive(Default)]
struct Watchdog {
/// Played-frame count last seen, and when it last changed.
seen: (u64, Duration),
/// An error was reported and the stream has not consumed samples since.
pending_error: bool,
/// When recent restarts happened (at most [`MAX_RESTARTS`] kept).
restarts: Vec<Duration>,
}
impl Watchdog {
fn reset(&mut self, now: Duration) {
*self = Self { seen: (0, now), ..Self::default() };
}
fn check(&mut self, now: Duration, played: u64, errored: bool, lost: bool) -> Verdict {
if played != self.seen.0 {
self.seen = (played, now);
self.pending_error = false;
}
// An error that arrives with the stream still consuming samples is only a hiccup.
self.pending_error |= errored;
let stalled = now.saturating_sub(self.seen.1) >= STALL_RESTART;
if !lost && !(self.pending_error && stalled) {
return Verdict::Healthy;
}
self.restarts.retain(|t| now.saturating_sub(*t) < RESTART_WINDOW);
if self.restarts.len() >= MAX_RESTARTS {
return Verdict::GiveUp;
}
self.restarts.push(now);
self.seen = (played, now);
self.pending_error = false;
Verdict::Restart
}
}
/// One device callback: run the shared mixer into `buf` and advance the clock. Never panics: a
/// panicking mixer plays silence and fails the stream, and a mixer held by another stream (never
/// expected: a restart drops the old stream first) plays silence without advancing the clock.
fn render(fill: &Mutex<Fill>, buf: &mut [f32], channels: usize, played: &AtomicU64, health: &Health) {
let mut guard = match fill.try_lock() {
Ok(guard) => guard,
Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
Err(TryLockError::WouldBlock) => {
buf.fill(0.0);
return;
}
};
if health.failed.load(Ordering::SeqCst) || catch_unwind(AssertUnwindSafe(|| (*guard)(buf, channels))).is_err() {
buf.fill(0.0);
if !health.failed.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: the audio mixer failed; playing without sound");
}
return;
}
played.fetch_add((buf.len() / channels.max(1)) as u64, Ordering::SeqCst);
}
fn stream_error(health: &Health, e: cpal::StreamError) {
match e {
cpal::StreamError::DeviceNotAvailable => health.lost.store(true, Ordering::SeqCst),
cpal::StreamError::BackendSpecific { .. } => health.errored.store(true, Ordering::SeqCst),
}
if !health.reported.swap(true, Ordering::SeqCst) {
eprintln!("filmcraft: audio stream error: {e}");
}
}
fn convert_samples<T: cpal::Sample + cpal::FromSample<f32>>(input: &[f32], output: &mut [T]) {
// Integer conversion represents [-1, 1). In particular I24 cannot represent +1 exactly.
let upper = f32::from_bits(1.0_f32.to_bits() - 1);
for (sample, converted) in input.iter().zip(output) {
*converted = T::from_sample(if sample.is_finite() { sample.clamp(-1.0, upper) } else { 0.0 });
}
}
impl AudioOut for CpalOut {
fn start(&mut self, fill: Box<dyn FnMut(&mut [f32], usize) + Send>) -> Result<u32, String> {
self.stop();
self.played.store(0, Ordering::SeqCst);
self.health = Arc::new(Health::default());
self.watchdog.reset(self.epoch.elapsed());
let fill = Arc::new(Mutex::new(fill));
self.open(fill.clone(), None)?;
self.fill = Some(fill);
Ok(self.rate)
}
fn stop(&mut self) {
self.stream = None;
self.fill = None;
}
fn recover(&mut self) {
if self.stream.is_none() || self.health.failed.load(Ordering::SeqCst) {
return;
}
let lost = self.health.lost.swap(false, Ordering::SeqCst);
let errored = self.health.errored.swap(false, Ordering::SeqCst);
match self.watchdog.check(self.epoch.elapsed(), self.played.load(Ordering::SeqCst), errored, lost) {
Verdict::Healthy => {}
Verdict::Restart => {
log::warn!("the audio output stream {}; restarting it", if lost { "lost its device" } else { "stopped after an error" });
if let Err(e) = self.restart() {
log::warn!("the audio output stream could not be restarted: {e}");
self.stream = None;
self.health.failed.store(true, Ordering::SeqCst);
}
}
Verdict::GiveUp => {
log::warn!("the audio output stream keeps failing ({MAX_RESTARTS} restarts within {} s)", RESTART_WINDOW.as_secs());
self.health.failed.store(true, Ordering::SeqCst);
}
}
}
fn sample_rate(&self) -> u32 {
self.rate
@ -263,7 +425,7 @@ impl AudioOut for CpalOut {
self.channels as usize
}
fn played_frames(&self) -> Option<u64> {
self.stream.as_ref().filter(|_| !self.failed.load(Ordering::SeqCst)).map(|_| self.played.load(Ordering::SeqCst))
self.stream.as_ref().filter(|_| !self.health.failed.load(Ordering::SeqCst)).map(|_| self.played.load(Ordering::SeqCst))
}
fn devices(&self) -> AudioDevices {
let h = host(&self.hw.device_class);
@ -327,11 +489,92 @@ mod tests {
}
#[test]
fn failed_stream_relinquishes_the_playback_clock() {
let failed = AtomicBool::new(false);
stream_error(&failed, cpal::StreamError::DeviceNotAvailable);
assert!(failed.load(Ordering::SeqCst));
fn stream_errors_ask_for_a_restart_instead_of_failing_playback() {
let health = Health::default();
stream_error(&health, cpal::StreamError::BackendSpecific { err: cpal::BackendSpecificError { description: "`alsa::poll()` returned POLLERR".into() } });
assert!(health.errored.load(Ordering::SeqCst) && !health.lost.load(Ordering::SeqCst));
stream_error(&health, cpal::StreamError::DeviceNotAvailable);
assert!(health.lost.load(Ordering::SeqCst));
assert!(!health.failed.load(Ordering::SeqCst), "a stream error alone must not silence playback");
let output = CpalOut::new();
assert_eq!(output.played_frames(), None);
assert_eq!(output.played_frames(), None, "no stream, no clock");
}
fn ms(n: u64) -> Duration {
Duration::from_millis(n)
}
#[test]
fn an_error_while_samples_keep_flowing_is_only_a_hiccup() {
let mut w = Watchdog::default();
w.reset(ms(0));
assert_eq!(w.check(ms(10), 480, true, false), Verdict::Healthy);
// the stream keeps consuming samples after the error: never restarted
for (i, t) in (20..2_000).step_by(20).enumerate() {
assert_eq!(w.check(ms(t), 480 * (i as u64 + 2), false, false), Verdict::Healthy);
}
}
#[test]
fn a_stream_that_errors_and_stops_consuming_is_restarted() {
let mut w = Watchdog::default();
w.reset(ms(0));
assert_eq!(w.check(ms(10), 480, false, false), Verdict::Healthy);
// POLLERR: the backend reports errors and no longer calls back
assert_eq!(w.check(ms(20), 480, true, false), Verdict::Healthy);
assert_eq!(w.check(ms(200), 480, true, false), Verdict::Healthy);
assert_eq!(w.check(ms(310), 480, false, false), Verdict::Restart);
// the restarted stream gets a fresh grace period
assert_eq!(w.check(ms(400), 480, false, false), Verdict::Healthy);
// a stall without any reported error is left to the UI's own watchdog
assert_eq!(w.check(ms(2_000), 480, false, false), Verdict::Healthy);
}
#[test]
fn a_lost_device_restarts_at_once_and_repeated_failures_give_up() {
let mut w = Watchdog::default();
w.reset(ms(0));
assert_eq!(w.check(ms(5), 0, false, true), Verdict::Restart);
assert_eq!(w.check(ms(10), 0, false, true), Verdict::Restart);
assert_eq!(w.check(ms(15), 0, false, true), Verdict::Restart);
assert_eq!(w.check(ms(20), 0, false, true), Verdict::GiveUp);
// sporadic failures far apart keep being recovered
assert_eq!(w.check(ms(10_006), 0, false, true), Verdict::Restart);
}
#[test]
fn the_device_callback_never_panics_and_counts_played_frames() {
let (played, health) = (AtomicU64::new(0), Health::default());
let fill: Mutex<Fill> = Mutex::new(Box::new(|buf: &mut [f32], _| buf.fill(0.25)));
let mut buf = [1.0_f32; 8];
render(&fill, &mut buf, 2, &played, &health);
assert_eq!((buf, played.load(Ordering::SeqCst)), ([0.25; 8], 4));
// a mixer still held elsewhere plays silence without moving the clock
{
let _held = fill.lock().unwrap();
render(&fill, &mut buf, 2, &played, &health);
}
assert_eq!((buf, played.load(Ordering::SeqCst)), ([0.0; 8], 4));
// zero channels from a broken driver must not divide by zero
render(&fill, &mut buf, 0, &played, &health);
assert_eq!(played.load(Ordering::SeqCst), 12);
}
#[test]
fn a_panicking_mixer_plays_silence_and_fails_the_stream() {
let (played, health) = (AtomicU64::new(0), Health::default());
let fill: Mutex<Fill> = Mutex::new(Box::new(|buf: &mut [f32], _| {
buf.fill(0.5);
panic!("mixer bug");
}));
let mut buf = [1.0_f32; 4];
render(&fill, &mut buf, 2, &played, &health);
assert_eq!(buf, [0.0; 4]);
assert_eq!(played.load(Ordering::SeqCst), 0);
assert!(health.failed.load(Ordering::SeqCst));
// later callbacks stay silent instead of re-entering the broken mixer
buf = [1.0; 4];
render(&fill, &mut buf, 2, &played, &health);
assert_eq!(buf, [0.0; 4]);
}
}

831
apps/filmcraft/src/graphics.rs
View File

@ -1,33 +1,487 @@
//! Windows graphics instance configuration, applied before wgpu can initialize any driver.
//! Graphics start-up: the wgpu backends and adapter the window is created with, and the fallback
//! when a start fails (docs/gaps.md G4, #820).
//!
//! A graphics driver can crash inside wgpu instance or device creation (Intel's Vulkan driver
//! `igvk64.dll`, #512 #413; AMD's OpenGL driver `atio6axx.dll`, #477), where no Rust error
//! handling runs, so every launch died the same way until the user set `WGPU_BACKEND` by hand.
//!
//! # When a start counts as failed
//! The data folder keeps the mode to start with and the failed starts in a row in it
//! ([`STATE_FILE`]). Each start also leaves its own attempt file (`graphics-start/<pid>-<n>.json`,
//! so concurrent launches never write the same file) that is removed once the window has painted
//! two frames, when the process ends normally, or [`DEATH_WINDOW`] after launch, whatever happened
//! by then. An attempt file found by a later launch therefore means a process that died within
//! [`DEATH_WINDOW`] of its start without painting: a driver crash, typically. A start that hangs,
//! or is force-quit or crashes later, is not counted; neither is another instance that is still
//! starting (its attempt is younger than [`DEATH_WINDOW`] and is left alone). After
//! [`FAILURES_TO_MOVE_ON`] such failures in a row in the same mode, the launch moves on to the next
//! [`Mode`] of the platform's [`LADDER`] and keeps it; a start that paints resets the count. A
//! start-up error that wgpu does report (no adapter, device creation failed) moves on at once,
//! within the same process.
//!
//! # Getting back
//! A fallback mode shows a notice with a reset button in the window (`graphics.notice.reset`,
//! [`GraphicsStart::reset_to_default`]); `--graphics default` does the same from the command
//! line, and `--graphics dx12|software` picks a mode by hand. An explicit `WGPU_BACKEND` leaves
//! the saved state alone: the user is in charge.
//!
//! All files are written beside and renamed into place, so a reader never sees half a file; two
//! instances updating [`STATE_FILE`] at the same moment can lose one update (a failure counted
//! once less, a confirmation repeated by the next start), never corrupt it.
use eframe::{NativeOptions, egui_wgpu::WgpuSetup, wgpu::Backends};
use std::io::Read as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// Restrict the wgpu instance to the primary backends (Vulkan, DirectX 12, Metal), i.e. leave
/// OpenGL out, unless `backend_override` (normally `Backends::from_env()`, i.e. `WGPU_BACKEND`)
/// names other backends.
pub fn configure(options: &mut NativeOptions, backend_override: Option<Backends>) {
if let WgpuSetup::CreateNew(create) = &mut options.wgpu_options.wgpu_setup {
// eframe's default includes GL. Creating that backend can crash inside AMD's
// atio6axx.dll before adapter selection or any Rust error handling runs, so the
// window never appears. Choosing an adapter afterwards is too late: only leaving GL
// out of the instance avoids it. Keep Vulkan and DX12 both: forcing DX12 alone sends
// shaders through FXC (which rejects fx.wgsl, silently falling back to the CPU) and
// leaves machines without DX12 with no backend at all.
create.instance_descriptor.backends = backend_override.unwrap_or(Backends::PRIMARY);
use eframe::{
NativeOptions,
egui_wgpu::WgpuSetup,
wgpu::{self, Backends},
};
/// The saved graphics state in the data folder: `{"mode": "<mode>", "failures": <n>}`.
pub const STATE_FILE: &str = "graphics-start.json";
/// The folder in the data folder with one attempt file per start that has not painted yet.
pub const ATTEMPTS_DIR: &str = "graphics-start";
/// How long after launch a start that has not painted may die and still count as failed. A
/// driver crash in instance or device creation happens within a second or two.
pub const DEATH_WINDOW: Duration = Duration::from_secs(15);
/// Failed starts in a row, in the same mode, that move on to the next mode.
pub const FAILURES_TO_MOVE_ON: u32 = 2;
/// Attempt files looked at per launch; anything beyond is left for a later launch.
const MAX_ATTEMPTS: usize = 64;
/// The state and attempt files are a few bytes; anything longer is not ours.
const MAX_FILE: u64 = 4096;
/// How the window's graphics device is created.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Mode {
/// The platform default: Vulkan or DirectX 12 on Windows (no OpenGL, see [`apply`]), Metal on
/// macOS, Vulkan or OpenGL elsewhere.
Default,
/// DirectX 12 only (Windows), for Vulkan drivers that crash at start-up.
Dx12,
/// A software adapter (WARP on Windows, lavapipe / llvmpipe elsewhere); the preview then
/// composites on the CPU (`FilmcraftApp::set_wgpu`).
Software,
}
impl Mode {
/// The name in [`STATE_FILE`] and on the command line (`--graphics <name>`).
pub fn name(self) -> &'static str {
match self {
Mode::Default => "default",
Mode::Dx12 => "dx12",
Mode::Software => "software",
}
}
/// The English name shown in the interface (translated there).
pub fn label(self) -> &'static str {
match self {
Mode::Default => "default graphics",
Mode::Dx12 => "DirectX 12 only",
Mode::Software => "software rendering",
}
}
pub fn parse(name: &str) -> Option<Mode> {
[Mode::Default, Mode::Dx12, Mode::Software].into_iter().find(|m| m.name() == name)
}
}
/// The modes tried in order on this platform. macOS has only Metal, and no software adapter.
pub const LADDER: &[Mode] = if cfg!(target_os = "windows") {
&[Mode::Default, Mode::Dx12, Mode::Software]
} else if cfg!(target_os = "macos") {
&[Mode::Default]
} else {
&[Mode::Default, Mode::Software]
};
/// The mode after `mode` in `ladder`, if any.
fn next_in(ladder: &[Mode], mode: Mode) -> Option<Mode> {
let i = ladder.iter().position(|m| *m == mode)?;
ladder.get(i.checked_add(1)?).copied()
}
/// [`STATE_FILE`]: the mode to start with and the failed starts in a row in it.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Saved {
mode: Mode,
failures: u32,
}
/// An attempt file left by a start: its process id (from the file name), mode and launch time.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Attempt {
pid: u32,
mode: Mode,
/// Seconds since the Unix epoch.
started: u64,
}
/// What a launch makes of the saved state and the attempt files it found.
#[derive(Debug, PartialEq, Eq)]
struct Decision {
/// The state to start with and save.
saved: Saved,
/// Moved on to the next mode because the last starts failed.
moved_on: bool,
/// Per attempt: done with (counted, or not ours to count), delete it.
delete: Vec<bool>,
}
/// This launch's mode from the saved state and the attempts other starts left. `now` is in Unix
/// seconds, `pid` is this process (an attempt with our process id is a dead process whose id was
/// reused), `window` is [`DEATH_WINDOW`] in seconds.
fn decide(saved: Option<Saved>, attempts: &[Attempt], now: u64, pid: u32, window: u64, ladder: &[Mode]) -> Decision {
let first = ladder.first().copied().unwrap_or(Mode::Default);
// Nothing saved, or a mode this platform does not use (a data folder copied from another OS).
let saved = saved.filter(|s| ladder.contains(&s.mode)).unwrap_or(Saved { mode: first, failures: 0 });
let mut failures = saved.failures;
let mut delete = Vec::with_capacity(attempts.len());
for a in attempts {
if a.started > now.saturating_add(window) {
// From the future (a clock that jumped back): unknown, and not worth keeping.
delete.push(true);
continue;
}
let dead = a.pid == pid || now.saturating_sub(a.started) >= window;
// A younger attempt is another instance starting right now: leave it to that instance.
delete.push(dead);
// Only failures of the mode we would start in count; older ones are history.
if dead && a.mode == saved.mode {
failures = failures.saturating_add(1);
}
}
if failures >= FAILURES_TO_MOVE_ON {
return match next_in(ladder, saved.mode) {
Some(next) => Decision { saved: Saved { mode: next, failures: 0 }, moved_on: true, delete },
// Nothing left to try: the last mode is still the best chance.
None => Decision { saved: Saved { mode: saved.mode, failures: 0 }, moved_on: false, delete },
};
}
Decision { saved: Saved { mode: saved.mode, failures }, moved_on: false, delete }
}
fn read_small(path: &Path) -> Option<String> {
let mut text = String::new();
std::fs::File::open(path).ok()?.take(MAX_FILE).read_to_string(&mut text).ok()?;
Some(text)
}
/// [`Saved`] from its text; `None` for anything else. A missing failure count is none.
fn parse_state(text: &str) -> Option<Saved> {
let v: serde_json::Value = serde_json::from_str(text).ok()?;
let mode = Mode::parse(v.get("mode")?.as_str()?)?;
let failures = match v.get("failures") {
None => 0,
Some(n) => u32::try_from(n.as_u64()?).ok()?,
};
Some(Saved { mode, failures })
}
fn state_json(s: Saved) -> String {
serde_json::json!({ "mode": s.mode.name(), "failures": s.failures }).to_string()
}
/// An attempt from its file name (`<pid>-<n>.json`) and text; `None` for anything else.
fn parse_attempt(file_name: &str, text: &str) -> Option<Attempt> {
let pid = file_name.strip_suffix(".json")?.split_once('-')?.0.parse().ok()?;
let v: serde_json::Value = serde_json::from_str(text).ok()?;
Some(Attempt { pid, mode: Mode::parse(v.get("mode")?.as_str()?)?, started: v.get("started")?.as_u64()? })
}
fn attempt_json(mode: Mode, started: u64) -> String {
serde_json::json!({ "mode": mode.name(), "started": started }).to_string()
}
/// Write `text` beside `path` (a name only this process uses) and rename it into place, so a
/// start that dies mid-write, or another instance reading, never sees half a file.
fn write_atomic(path: &Path, text: &str) -> std::io::Result<()> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
let mut tmp = path.as_os_str().to_owned();
tmp.push(format!(".{}.tmp", std::process::id()));
let tmp = PathBuf::from(tmp);
std::fs::write(&tmp, text)?;
std::fs::rename(&tmp, path).inspect_err(|_| {
let _ = std::fs::remove_file(&tmp);
})
}
fn remove_quietly(path: &Path) {
match std::fs::remove_file(path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => log::warn!("can't remove {}: {e}", path.display()),
_ => {}
}
}
/// The attempt files in `dir` (at most [`MAX_ATTEMPTS`]): path and contents, `None` when unreadable.
fn list_attempts(dir: &Path) -> Vec<(PathBuf, Option<Attempt>)> {
let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
entries
.filter_map(Result::ok)
.filter_map(|e| {
let name = e.file_name().into_string().ok()?;
name.ends_with(".json").then(|| {
let path = e.path();
let attempt = read_small(&path).and_then(|text| parse_attempt(&name, &text));
(path, attempt)
})
})
.take(MAX_ATTEMPTS)
.collect()
}
fn unix_now() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_secs())
}
/// Attempt numbers within this process, so each attempt file has its own name.
static NEXT_ATTEMPT: AtomicU32 = AtomicU32::new(0);
/// This launch's graphics mode and its files in the data folder.
#[derive(Clone, Debug)]
pub struct GraphicsStart {
/// [`STATE_FILE`]; `None` without a data folder or with `WGPU_BACKEND` set.
state: Option<PathBuf>,
/// This start's attempt file (with `state`).
attempt: Option<PathBuf>,
mode: Mode,
/// [`Self::reset_to_default`] was used: a later [`Self::confirm`] must not save this mode over it.
reset: Arc<AtomicBool>,
}
impl GraphicsStart {
/// Pick this launch's mode from the data folder and record the attempt. `forced`
/// (`--graphics`) starts in that mode and saves it. `explicit_backend` (`WGPU_BACKEND` set)
/// leaves the data folder alone and starts in `forced`, else [`Mode::Default`].
pub fn begin(data_dir: Option<&Path>, explicit_backend: bool, forced: Option<Mode>) -> Self {
let start = Self::begin_at(data_dir, explicit_backend, forced, unix_now(), std::process::id());
if let Some(path) = start.attempt.clone() {
expire_later(path);
}
start
}
/// [`Self::begin`] at `now` (Unix seconds) as process `pid`, without the expiry timer.
fn begin_at(data_dir: Option<&Path>, explicit_backend: bool, forced: Option<Mode>, now: u64, pid: u32) -> Self {
let first = LADDER.first().copied().unwrap_or(Mode::Default);
let forced = forced.map(|m| if LADDER.contains(&m) { m } else { first });
let reset = Arc::new(AtomicBool::new(false));
let Some(dir) = data_dir.filter(|_| !explicit_backend) else {
return Self { state: None, attempt: None, mode: forced.unwrap_or(Mode::Default), reset };
};
let state = dir.join(STATE_FILE);
let attempts_dir = dir.join(ATTEMPTS_DIR);
let saved = read_small(&state).and_then(|t| parse_state(&t));
let found = list_attempts(&attempts_dir);
let known: Vec<Attempt> = found.iter().filter_map(|(_, a)| *a).collect();
let decision = decide(saved, &known, now, pid, DEATH_WINDOW.as_secs(), LADDER);
let mut to_delete = decision.delete.iter();
for (path, attempt) in &found {
// unreadable files in our folder are debris (a different version, a damaged disk)
if attempt.is_none() || to_delete.next().copied().unwrap_or(false) {
remove_quietly(path);
}
}
let file = state.display();
let next = match forced {
Some(mode) => {
log::info!("graphics mode {} chosen with --graphics", mode.name());
Saved { mode, failures: 0 }
}
None => {
if decision.moved_on {
let failed = saved.map_or(first, |s| s.mode).name();
log::warn!(
"the last {FAILURES_TO_MOVE_ON} starts with graphics mode {failed} died before their window painted; starting with {} (`--graphics default` or the notice in the window goes back)",
decision.saved.mode.name()
);
} else if decision.saved.failures > 0 {
log::warn!("the last start with graphics mode {} died before its window painted", decision.saved.mode.name());
} else if decision.saved.mode != Mode::Default {
log::info!("graphics mode {} after earlier failed starts ({file}; `--graphics default` goes back)", decision.saved.mode.name());
}
decision.saved
}
};
if saved != Some(next)
&& let Err(e) = write_atomic(&state, &state_json(next))
{
log::warn!("can't write {file}: {e}");
}
let start = Self { state: Some(state), attempt: None, mode: next.mode, reset };
start.with_new_attempt(&attempts_dir, now, pid)
}
/// Record a new attempt file for the current mode.
fn with_new_attempt(mut self, dir: &Path, now: u64, pid: u32) -> Self {
let n = NEXT_ATTEMPT.fetch_add(1, Ordering::Relaxed);
let path = dir.join(format!("{pid}-{n}.json"));
match write_atomic(&path, &attempt_json(self.mode, now)) {
Ok(()) => self.attempt = Some(path),
Err(e) => log::warn!("can't write {}: {e}", path.display()),
}
self
}
pub fn mode(&self) -> Mode {
self.mode
}
/// Running in a fallback mode, i.e. not the platform default.
pub fn degraded(&self) -> bool {
self.mode != Mode::Default
}
/// Move on to the next mode after a start-up error wgpu reported; `false` when none is left.
/// The error is certain, so the next mode is saved at once.
pub fn step_down(&mut self) -> bool {
let Some(next) = next_in(LADDER, self.mode) else {
return false;
};
log::warn!("graphics mode {} failed to start; trying {}", self.mode.name(), next.name());
self.mode = next;
if let Some(state) = self.state.clone() {
self.save(&state, Saved { mode: next, failures: 0 });
if let Some(old) = self.attempt.take() {
remove_quietly(&old);
if let Some(dir) = old.parent() {
let path = self.clone().with_new_attempt(dir, unix_now(), std::process::id()).attempt;
if let Some(path) = &path {
expire_later(path.clone());
}
self.attempt = path;
}
}
}
true
}
/// The window is up: keep this mode, and the failed starts in a row are over.
pub fn confirm(&self) {
self.finish();
let Some(state) = &self.state else { return };
if self.reset.load(Ordering::Relaxed) {
return;
}
let ok = Saved { mode: self.mode, failures: 0 };
if read_small(state).and_then(|t| parse_state(&t)) != Some(ok) {
self.save(state, ok);
}
}
/// This start is over, whether it painted or not (the window closed, or every mode failed
/// with an error that was reported): its attempt no longer means a crash.
pub fn finish(&self) {
if let Some(path) = &self.attempt {
remove_quietly(path);
}
}
/// Start with the default mode next time (the notice's reset button). This process keeps
/// its mode: the graphics device can't be swapped under a running window.
pub fn reset_to_default(&self) -> Result<(), String> {
self.reset.store(true, Ordering::Relaxed);
let Some(state) = &self.state else { return Ok(()) };
let first = LADDER.first().copied().unwrap_or(Mode::Default);
write_atomic(state, &state_json(Saved { mode: first, failures: 0 })).map_err(|e| format!("can't write {}: {e}", state.display()))?;
log::info!("graphics mode reset to {} for the next start", first.name());
Ok(())
}
fn save(&self, path: &Path, s: Saved) {
if let Err(e) = write_atomic(path, &state_json(s)) {
log::warn!("can't write {}: {e}", path.display());
}
}
}
/// Remove the attempt file at `path` [`DEATH_WINDOW`] from now: a start still alive then is not
/// counted as a failed one, painted or not.
fn expire_later(path: PathBuf) {
let spawned = std::thread::Builder::new().name("graphics-start".into()).spawn(move || {
let _ = std::panic::catch_unwind(move || {
std::thread::sleep(DEATH_WINDOW);
remove_quietly(&path);
});
});
if let Err(e) = spawned {
log::warn!("can't start the graphics start timer: {e}");
}
}
/// Configure `options` for `mode`, before eframe creates the wgpu instance. `backend_override`
/// is normally `Backends::from_env()`, i.e. `WGPU_BACKEND`.
pub fn apply(options: &mut NativeOptions, mode: Mode, backend_override: Option<Backends>) {
apply_for(options, mode, backend_override, cfg!(target_os = "windows"));
}
fn apply_for(options: &mut NativeOptions, mode: Mode, backend_override: Option<Backends>, windows: bool) {
let WgpuSetup::CreateNew(create) = &mut options.wgpu_options.wgpu_setup else {
return;
};
match mode {
// eframe's default (`WGPU_BACKEND`, else PRIMARY | GL), except on Windows: there it
// includes GL. Creating that backend can crash inside AMD's atio6axx.dll before adapter
// selection or any Rust error handling runs, so the window never appears. Choosing an
// adapter afterwards is too late: only leaving GL out of the instance avoids it. Keep
// Vulkan and DX12 both: forcing DX12 alone sends shaders through FXC (which rejects
// fx.wgsl, silently falling back to the CPU) and leaves machines without DX12 with no
// backend at all.
Mode::Default if windows => create.instance_descriptor.backends = backend_override.unwrap_or(Backends::PRIMARY),
Mode::Default => {}
// Without Vulkan, whose driver is what crashes; GPU effects may fall back to the CPU.
Mode::Dx12 => create.instance_descriptor.backends = Backends::DX12,
Mode::Software => {
create.instance_descriptor.backends = software_backends(windows);
create.native_adapter_selector = Some(Arc::new(pick_software));
}
}
}
/// The backends a software adapter is looked for in: WARP is a DX12 adapter, and GL stays out on
/// Windows for the reason in [`apply_for`].
fn software_backends(windows: bool) -> Backends {
if windows { Backends::DX12 } else { Backends::VULKAN | Backends::GL }
}
/// The first software adapter that can draw to the window's surface.
fn pick_software(adapters: &[wgpu::Adapter], surface: Option<&wgpu::Surface<'_>>) -> Result<wgpu::Adapter, String> {
adapters
.iter()
.find(|a| filmcraft_ui_egui::software_rasterizer(&a.get_info()).is_some() && surface.is_none_or(|s| a.is_surface_supported(s)))
.cloned()
.ok_or_else(|| format!("no software graphics adapter (WARP, lavapipe, llvmpipe) among {} adapters", adapters.len()))
}
/// Whether `err` from `eframe::run_native` is a graphics start-up failure that another mode may
/// avoid (as opposed to a display, event-loop or app error).
pub fn is_graphics_failure(err: &eframe::Error) -> bool {
matches!(err, eframe::Error::Wgpu(_))
}
#[cfg(test)]
mod tests {
use super::*;
fn configured_backends(backend_override: Option<Backends>) -> Backends {
const WINDOWS: &[Mode] = &[Mode::Default, Mode::Dx12, Mode::Software];
const LINUX: &[Mode] = &[Mode::Default, Mode::Software];
const MACOS: &[Mode] = &[Mode::Default];
fn configured(mode: Mode, backend_override: Option<Backends>, windows: bool) -> (Backends, bool) {
let mut options = NativeOptions::default();
configure(&mut options, backend_override);
apply_for(&mut options, mode, backend_override, windows);
let WgpuSetup::CreateNew(create) = options.wgpu_options.wgpu_setup else {
panic!("default native options must create a wgpu instance");
};
create.instance_descriptor.backends
(create.instance_descriptor.backends, create.native_adapter_selector.is_some())
}
#[test]
@ -37,16 +491,349 @@ mod tests {
panic!("default native options must create a wgpu instance");
};
assert!(eframe_default.instance_descriptor.backends.contains(Backends::GL));
let configured = configured_backends(None);
assert_eq!(configured, Backends::PRIMARY);
assert!(!configured.contains(Backends::GL));
assert!(configured.contains(Backends::DX12) && configured.contains(Backends::VULKAN));
let (backends, selector) = configured(Mode::Default, None, true);
assert_eq!(backends, Backends::PRIMARY);
assert!(!backends.contains(Backends::GL));
assert!(backends.contains(Backends::DX12) && backends.contains(Backends::VULKAN));
assert!(!selector);
}
#[test]
fn explicit_backend_override_is_preserved() {
for backends in [Backends::VULKAN, Backends::GL, Backends::DX12, Backends::VULKAN | Backends::DX12, Backends::empty()] {
assert_eq!(configured_backends(Some(backends)), backends);
assert_eq!(configured(Mode::Default, Some(backends), true).0, backends);
}
}
#[test]
fn fallback_modes_leave_out_the_crashing_backends() {
// DX12 only: Intel's and AMD's Vulkan drivers are what crashed (#512, #413).
assert_eq!(configured(Mode::Dx12, None, true), (Backends::DX12, false));
// Software on Windows is WARP (DX12), never GL (atio6axx.dll, #477).
assert_eq!(configured(Mode::Software, None, true), (Backends::DX12, true));
assert_eq!(configured(Mode::Software, None, false), (Backends::VULKAN | Backends::GL, true));
// Off Windows the default is eframe's own.
let WgpuSetup::CreateNew(eframe_default) = NativeOptions::default().wgpu_options.wgpu_setup else {
panic!("default native options must create a wgpu instance");
};
assert_eq!(configured(Mode::Default, None, false), (eframe_default.instance_descriptor.backends, false));
}
const W: u64 = 15;
const NOW: u64 = 1_000_000;
const ME: u32 = 7;
fn saved(mode: Mode, failures: u32) -> Option<Saved> {
Some(Saved { mode, failures })
}
fn dead(mode: Mode) -> Attempt {
Attempt { pid: 1, mode, started: NOW - 100 }
}
fn mode_after(s: Option<Saved>, attempts: &[Attempt], ladder: &[Mode]) -> (Mode, u32, bool) {
let d = decide(s, attempts, NOW, ME, W, ladder);
(d.saved.mode, d.saved.failures, d.moved_on)
}
#[test]
fn two_failed_starts_in_a_row_move_on_to_the_next_mode_and_keep_it() {
// first launch
assert_eq!(mode_after(None, &[], WINDOWS), (Mode::Default, 0, false));
// one start died: counted, but the same mode is tried again
assert_eq!(mode_after(None, &[dead(Mode::Default)], WINDOWS), (Mode::Default, 1, false));
// the second in a row moves on: DX12, then software
assert_eq!(mode_after(saved(Mode::Default, 1), &[dead(Mode::Default)], WINDOWS), (Mode::Dx12, 0, true));
assert_eq!(mode_after(saved(Mode::Dx12, 1), &[dead(Mode::Dx12)], WINDOWS), (Mode::Software, 0, true));
assert_eq!(mode_after(saved(Mode::Default, 1), &[dead(Mode::Default)], LINUX), (Mode::Software, 0, true));
// two dead instances found at once count as two
assert_eq!(mode_after(None, &[dead(Mode::Default), dead(Mode::Default)], WINDOWS), (Mode::Dx12, 0, true));
// a saved mode is kept
assert_eq!(mode_after(saved(Mode::Dx12, 0), &[], WINDOWS), (Mode::Dx12, 0, false));
assert_eq!(mode_after(saved(Mode::Default, 1), &[], WINDOWS), (Mode::Default, 1, false));
// nothing left to try: stay
assert_eq!(mode_after(saved(Mode::Software, 1), &[dead(Mode::Software)], WINDOWS), (Mode::Software, 0, false));
assert_eq!(mode_after(saved(Mode::Default, 5), &[dead(Mode::Default)], MACOS), (Mode::Default, 0, false));
// a mode this platform has not: start over
assert_eq!(mode_after(saved(Mode::Dx12, 1), &[dead(Mode::Dx12)], LINUX), (Mode::Default, 0, false));
assert_eq!(mode_after(saved(Mode::Software, 0), &[], MACOS), (Mode::Default, 0, false));
assert_eq!(mode_after(None, &[], &[]), (Mode::Default, 0, false));
// a hostile count doesn't overflow
assert_eq!(mode_after(saved(Mode::Default, u32::MAX), &[dead(Mode::Default)], WINDOWS), (Mode::Dx12, 0, true));
}
#[test]
fn only_starts_that_died_soon_after_launch_in_the_current_mode_count() {
let starting = Attempt { pid: 2, mode: Mode::Default, started: NOW - 3 };
let just_now = Attempt { pid: 3, mode: Mode::Default, started: NOW };
let future = Attempt { pid: 4, mode: Mode::Default, started: NOW + 3600 };
let older_mode = dead(Mode::Dx12);
let mine = Attempt { pid: ME, mode: Mode::Default, started: NOW - 1 };
let d = decide(saved(Mode::Default, 0), &[starting, just_now, future, older_mode, mine], NOW, ME, W, WINDOWS);
// another instance still starting is left alone; one from the future is dropped; a
// failure in another mode is history; our own process id can only be a dead process
assert_eq!(d.delete, [false, false, true, true, true]);
assert_eq!((d.saved, d.moved_on), (Saved { mode: Mode::Default, failures: 1 }, false));
// the clock a little behind another instance's: still starting
let ahead = Attempt { pid: 5, mode: Mode::Default, started: NOW + 2 };
assert_eq!(decide(None, &[ahead], NOW, ME, W, WINDOWS).delete, [false]);
}
#[test]
fn the_files_round_trip_and_reject_anything_else() {
for mode in WINDOWS {
for failures in [0, 1, 2, u32::MAX] {
assert_eq!(parse_state(&state_json(Saved { mode: *mode, failures })), saved(*mode, failures));
}
assert_eq!(parse_attempt("12-0.json", &attempt_json(*mode, NOW)), Some(Attempt { pid: 12, mode: *mode, started: NOW }));
}
// without a count (an earlier build of this change wrote `pending`): none
assert_eq!(parse_state(r#"{"mode":"dx12","pending":true}"#), saved(Mode::Dx12, 0));
for bad in [
"",
"{",
"null",
"[]",
r#"{"mode":"vulkan","failures":1}"#,
r#"{"failures":1}"#,
r#"{"mode":1,"failures":1}"#,
r#"{"mode":"dx12","failures":"one"}"#,
r#"{"mode":"dx12","failures":-1}"#,
r#"{"mode":"dx12","failures":4294967296}"#,
] {
assert_eq!(parse_state(bad), None, "{bad}");
}
let good = attempt_json(Mode::Default, NOW);
for (name, text) in [
("12-0.json", "{"),
("12-0.json", r#"{"mode":"default"}"#),
("12-0.json", r#"{"mode":"default","started":-5}"#),
("x-0.json", good.as_str()),
("12.json", good.as_str()),
("-0.json", good.as_str()),
("99999999999-0.json", good.as_str()),
("12-0.txt", good.as_str()),
] {
assert_eq!(parse_attempt(name, text), None, "{name} {text}");
}
}
fn temp_dir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("filmcraft-graphics-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
dir
}
fn state_of(dir: &Path) -> Option<Saved> {
read_small(&dir.join(STATE_FILE)).and_then(|t| parse_state(&t))
}
fn attempt_files(dir: &Path) -> usize {
std::fs::read_dir(dir.join(ATTEMPTS_DIR)).map_or(0, |d| d.filter_map(Result::ok).count())
}
/// A launch as another process (`pid`) at `now`.
fn launch(dir: &Path, pid: u32, now: u64) -> GraphicsStart {
GraphicsStart::begin_at(Some(dir), false, None, now, pid)
}
#[test]
fn a_start_that_paints_keeps_its_mode_and_clears_its_attempt() {
let dir = temp_dir("paint");
let start = launch(&dir, 10, NOW);
assert_eq!(start.mode(), Mode::Default);
assert!(!start.degraded());
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert_eq!(attempt_files(&dir), 1);
start.confirm();
assert_eq!(attempt_files(&dir), 0);
// a window closed before it painted is no failure either
let start = launch(&dir, 11, NOW + 60);
start.finish();
assert_eq!(attempt_files(&dir), 0);
assert_eq!(launch(&dir, 12, NOW + 120).mode(), Mode::Default);
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn two_dead_starts_move_on_and_a_painted_one_resets_the_count() {
let dir = temp_dir("dead");
let second = LADDER.get(1).copied().unwrap_or(Mode::Default);
// dies without painting
let _ = launch(&dir, 20, NOW);
// the next launch counts it, tries the same mode again and dies too
let again = launch(&dir, 21, NOW + 60);
assert_eq!(again.mode(), Mode::Default);
assert_eq!(state_of(&dir), saved(Mode::Default, 1));
// two in a row: the next mode, kept
let moved = launch(&dir, 22, NOW + 120);
assert_eq!(moved.mode(), second);
assert_eq!(moved.degraded(), second != Mode::Default);
moved.confirm();
assert_eq!(launch(&dir, 23, NOW + 180).mode(), second);
// one failure, then a start that paints: the count starts over
let _ = std::fs::remove_dir_all(&dir);
let _ = launch(&dir, 30, NOW);
let painted = launch(&dir, 31, NOW + 60);
painted.confirm();
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
let _ = launch(&dir, 32, NOW + 120);
assert_eq!(launch(&dir, 33, NOW + 180).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn concurrent_launches_neither_count_nor_disturb_each_other() {
let dir = temp_dir("race");
// two instances start within the same seconds; neither has painted yet
let a = launch(&dir, 40, NOW);
let b = launch(&dir, 41, NOW + 1);
let c = launch(&dir, 42, NOW + 2);
assert_eq!((a.mode(), b.mode(), c.mode()), (Mode::Default, Mode::Default, Mode::Default));
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert_eq!(attempt_files(&dir), 3, "each instance has its own attempt file");
b.confirm();
a.confirm();
c.confirm();
assert_eq!(attempt_files(&dir), 0);
assert_eq!(launch(&dir, 43, NOW + 60).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn corrupt_huge_or_foreign_files_start_with_the_default() {
let dir = temp_dir("corrupt");
let _ = std::fs::create_dir_all(dir.join(ATTEMPTS_DIR));
let huge = format!(r#"{{"mode":"software","failures":1,"pad":"{}"}}"#, "x".repeat(10_000));
for text in ["garbage", huge.as_str()] {
let _ = std::fs::write(dir.join(STATE_FILE), text);
let _ = std::fs::write(dir.join(ATTEMPTS_DIR).join("50-0.json"), text);
let _ = std::fs::write(dir.join(ATTEMPTS_DIR).join("notes.txt"), text);
let start = launch(&dir, 51, NOW);
assert_eq!(start.mode(), Mode::Default);
// replaced with a well-formed state; the unreadable attempt is gone, other files stay
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert!(!dir.join(ATTEMPTS_DIR).join("50-0.json").exists());
assert!(dir.join(ATTEMPTS_DIR).join("notes.txt").exists());
start.finish();
}
// a data folder that is a file: nothing can be written, the default still starts
let file = dir.join("not-a-folder");
let _ = std::fs::write(&file, "x");
let mut start = GraphicsStart::begin_at(Some(&file), false, None, NOW, 52);
assert_eq!(start.mode(), Mode::Default);
start.confirm();
let _ = start.step_down();
assert!(start.reset_to_default().is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn many_attempt_files_are_bounded_per_launch() {
let dir = temp_dir("many");
let attempts = dir.join(ATTEMPTS_DIR);
let _ = std::fs::create_dir_all(&attempts);
let n = MAX_ATTEMPTS + 10;
for pid in 0..n {
let _ = std::fs::write(attempts.join(format!("{}-0.json", pid + 1000)), attempt_json(Mode::Software, NOW + 1));
}
// all of them belong to instances still starting: none is counted or removed
let start = launch(&dir, 60, NOW);
assert_eq!(start.mode(), Mode::Default);
assert_eq!(attempt_files(&dir), n + 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn graphics_flag_picks_the_mode_and_default_undoes_a_fallback() {
let dir = temp_dir("forced");
let last = LADDER.last().copied().unwrap_or(Mode::Default);
let _ = write_atomic(&dir.join(STATE_FILE), &state_json(Saved { mode: last, failures: 1 }));
launch(&dir, 70, NOW).confirm();
// `--graphics default`
let start = GraphicsStart::begin_at(Some(&dir), false, Some(Mode::Default), NOW + 60, 71);
assert_eq!(start.mode(), Mode::Default);
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
start.confirm();
assert_eq!(launch(&dir, 72, NOW + 120).mode(), Mode::Default);
// `--graphics <the last mode>` is kept for the next launches
GraphicsStart::begin_at(Some(&dir), false, Some(last), NOW + 180, 73).confirm();
assert_eq!(launch(&dir, 74, NOW + 240).mode(), last);
// a mode this platform has not starts with the first one
if !LADDER.contains(&Mode::Dx12) {
assert_eq!(GraphicsStart::begin_at(Some(&dir), false, Some(Mode::Dx12), NOW + 300, 75).mode(), Mode::Default);
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn reset_to_default_wins_over_a_later_confirm() {
let dir = temp_dir("reset");
let last = LADDER.last().copied().unwrap_or(Mode::Default);
let _ = write_atomic(&dir.join(STATE_FILE), &state_json(Saved { mode: last, failures: 0 }));
let start = launch(&dir, 80, NOW);
assert_eq!(start.mode(), last);
// the notice's button, from a clone of the start (the app holds one)
assert_eq!(start.clone().reset_to_default(), Ok(()));
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
start.confirm();
assert_eq!(state_of(&dir), saved(Mode::Default, 0));
assert_eq!(launch(&dir, 81, NOW + 60).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_explicit_backend_leaves_the_data_folder_alone() {
let dir = temp_dir("explicit");
let _ = write_atomic(&dir.join(STATE_FILE), &state_json(Saved { mode: Mode::Software, failures: 1 }));
let mut start = GraphicsStart::begin_at(Some(&dir), true, None, NOW, 90);
assert_eq!(start.mode(), Mode::Default);
start.confirm();
let _ = start.step_down();
assert_eq!(start.reset_to_default(), Ok(()));
assert_eq!(state_of(&dir), saved(Mode::Software, 1));
assert_eq!(attempt_files(&dir), 0);
// with --graphics too, that mode starts, still without touching the folder
let last = LADDER.last().copied().unwrap_or(Mode::Default);
assert_eq!(GraphicsStart::begin_at(Some(&dir), true, Some(last), NOW, 91).mode(), last);
assert_eq!(state_of(&dir), saved(Mode::Software, 1));
// no data folder: nothing is written, the default still starts
assert_eq!(GraphicsStart::begin_at(None, false, None, NOW, 92).mode(), Mode::Default);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn step_down_walks_the_ladder_and_saves_each_mode_at_once() {
let dir = temp_dir("step");
let mut start = launch(&dir, ME, NOW);
for expected in LADDER.iter().skip(1) {
assert!(start.step_down());
assert_eq!(start.mode(), *expected);
assert_eq!(state_of(&dir), saved(*expected, 0));
assert_eq!(attempt_files(&dir), 1, "one attempt file per process");
}
assert!(!start.step_down(), "nothing after the last mode");
start.finish();
assert_eq!(attempt_files(&dir), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_software_selector_only_picks_a_software_adapter() {
if wgpu::Instance::enabled_backend_features().is_empty() {
return;
}
// only the backends software mode uses: no driver this test loads is one the app avoids
let backends = software_backends(cfg!(target_os = "windows"));
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { backends, ..wgpu::InstanceDescriptor::new_without_display_handle() });
let adapters = pollster::block_on(instance.enumerate_adapters(backends));
match pick_software(&adapters, None) {
Ok(adapter) => assert!(filmcraft_ui_egui::software_rasterizer(&adapter.get_info()).is_some()),
Err(why) => {
assert!(adapters.iter().all(|a| filmcraft_ui_egui::software_rasterizer(&a.get_info()).is_none()), "{why}");
}
}
assert!(pick_software(&[], None).is_err());
}
}

384
apps/filmcraft/src/main.rs
View File

@ -31,7 +31,6 @@ mod control_server;
#[cfg(target_os = "linux")]
mod dev_icon;
mod file_filters;
#[cfg(any(target_os = "windows", test))]
mod graphics;
mod logging;
#[cfg(target_os = "macos")]
@ -44,6 +43,8 @@ use filmcraft_engine::Session;
use filmcraft_engine::autosave::{AutosaveConfig, default_data_dir};
use filmcraft_ui_egui::FilmcraftApp;
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
/// Reverse-DNS app id: Linux `.desktop` file, hicolor icon name and Wayland app id.
const APP_ID: &str = "ai.storyteller.filmcraft";
@ -80,7 +81,7 @@ fn main() -> eframe::Result {
let logger = logging::install();
// lossy, so a value that is not Unicode is reported like any other bad port instead of ignored
let env_port = std::env::var_os("FILMCRAFT_CONTROL_PORT").map(|p| p.to_string_lossy().into_owned());
let Launch { control_port, files, demo, startup_flag, recover, data_dir } = match args::parse(env_port, std::env::args().skip(1)) {
let launch = match args::parse(env_port, std::env::args_os().skip(1)) {
Ok(Cli::Run(l)) => l,
Ok(Cli::Help) => {
tell(args::USAGE, false);
@ -95,7 +96,8 @@ fn main() -> eframe::Result {
std::process::exit(2);
}
};
let log_dir = data_dir.clone().or_else(default_data_dir).map(|d| d.join(logging::LOG_DIR));
let data_root = launch.data_dir.clone().or_else(default_data_dir);
let log_dir = data_root.as_ref().map(|d| d.join(logging::LOG_DIR));
log::info!("FilmCraft {} ({} {})", env!("CARGO_PKG_VERSION"), std::env::consts::OS, std::env::consts::ARCH);
// The log file lives next to the crash logs; opened after the arguments, so `--help`,
// `--version` and usage errors leave no file behind. Records logged until now go in first.
@ -118,172 +120,38 @@ fn main() -> eframe::Result {
// OS hardware video decoders (VideoToolbox on macOS) in front of our own; Settings ▸ Playback ▸
// Hardware decoding switches them off. Unsupported streams and failures use our decoders.
register_hardware_decoders();
register_gpu_frame_renderer();
let options = eframe::NativeOptions {
viewport: egui::ViewportBuilder::default()
.with_title("FilmCraft")
.with_inner_size([1600.0, 980.0])
.with_min_inner_size([900.0, 560.0])
.with_drag_and_drop(true)
.with_fullsize_content_view(true)
.with_titlebar_shown(false)
.with_title_shown(false)
// Started for an agent (`--control`): open without taking the user's keyboard focus.
.with_active(control_port.is_none())
.with_icon(app_icon())
// Wayland matches the window to `packaging/linux/ai.storyteller.filmcraft.desktop`.
.with_app_id(APP_ID),
event_loop_builder: agent_event_loop(control_port.is_some()),
..Default::default()
// Which graphics backends and adapter the window starts with: `--graphics`, else the saved
// mode, which moves on after starts that died before their window painted (a driver crash),
// see graphics.rs.
let backend_override = eframe::wgpu::Backends::from_env();
let mut graphics = graphics::GraphicsStart::begin(data_root.as_deref(), backend_override.is_some(), launch.graphics);
// The GPU export renderer creates its own wgpu instance with every backend; after a start
// fell back, that would load the driver that failed. Exports then composite on the CPU.
if graphics.mode() == graphics::Mode::Default {
register_gpu_frame_renderer();
}
let started = loop {
let mut options = native_options(launch.control_port.is_some());
// Before eframe creates the wgpu instance (e.g. no OpenGL on Windows, see graphics.rs).
graphics::apply(&mut options, graphics.mode(), backend_override);
let app_created = Arc::new(AtomicBool::new(false));
let result = eframe::run_native("FilmCraft", options, app_creator(launch.clone(), graphics.clone(), Arc::clone(&app_created)));
let app_created = app_created.load(Ordering::Relaxed);
match &result {
// wgpu reported the failure (no adapter, device creation failed) before the app
// existed: try the next mode in this process.
Err(e) if !app_created && graphics::is_graphics_failure(e) && graphics.step_down() => {
log::warn!("graphics start failed: {e}");
continue;
}
// Closed before the second frame: the window still came up.
Ok(()) if app_created => graphics.confirm(),
_ => {}
}
break result;
};
// Before eframe creates the wgpu instance: leave OpenGL out on Windows (see graphics.rs).
#[cfg(target_os = "windows")]
let options = {
let mut options = options;
graphics::configure(&mut options, eframe::wgpu::Backends::from_env());
options
};
let started = eframe::run_native(
"FilmCraft",
options,
Box::new(move |cc| {
let mut session = Session::default();
if let Some(dir) = data_dir.clone().or_else(default_data_dir) {
let mut cfg = AutosaveConfig::new(dir);
cfg.local_offset = local_offset;
if let Err(e) = session.start_autosave(cfg) {
log::warn!("auto-save and crash recovery unavailable: {e}");
}
}
// voice-over recording reads the microphone through cpal
session.voiceover.input = Some(Box::new(audio_in::CpalIn::new(&session.prefs.audio_hardware.device_class)));
let project = files.iter().find(|f| f.ends_with(".fcproj")).cloned();
if let Some(p) = project {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
} else if !startup_flag && session.prefs.general.at_startup == "openMostRecent" {
// Settings ▸ General ▸ At Startup ▸ Open Most Recent
let recent = session.prefs.general.recent_projects.iter().find(|p| std::path::Path::new(p).exists()).cloned();
match recent {
Some(p) => {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
}
None => {
let _ = session.execute("file.openDemoProject", json!({}));
}
}
} else if !startup_flag && session.prefs.general.at_startup == "emptyProject" {
} else if demo {
let _ = session.execute("file.openDemoProject", json!({}));
}
let media: Vec<String> = files.iter().filter(|f| !f.ends_with(".fcproj")).cloned().collect();
if !media.is_empty() {
let _ = session.execute("file.import", json!({"paths": media}));
}
if recover == Some(true) && !session.recovery_candidates().is_empty() {
let id = session.recovery_candidates()[0].id.clone();
match session.execute("file.recover", json!({"id": id})) {
Ok(r) => log::info!("recovered {r}"),
Err(e) => log::error!("recovery failed: {e}"),
}
}
let mut app = FilmcraftApp::new(session);
if recover == Some(false) {
app.dialog = None;
}
app.integrated_titlebar = cfg!(target_os = "macos");
if let Some(rs) = cc.wgpu_render_state.clone()
&& std::env::var_os("FILMCRAFT_CPU_COMPOSITE").is_none()
{
app.set_wgpu(rs);
}
// Keep device selection available even when the default device is unavailable.
// Settings ▸ Audio Hardware is applied on the first frame (`apply_prefs`).
app.audio = Some(Box::new(audio::CpalOut::new()));
app.hooks.pick_files = Some(Box::new(|exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_files()
.unwrap_or_default()
.into_iter()
.map(|p| p.to_string_lossy().to_string())
.collect()
}));
// Link Media ▸ Locate…, Attach Proxies, Reconnect Full Resolution: one path, not imported.
app.hooks.pick_file_for_relink = Some(Box::new(|exts: &[&str], _hint| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save = Some(Box::new(|name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save_as = Some(Box::new(|filter: &str, exts: &[&str], name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_folder = Some(Box::new(|| rfd::FileDialog::new().pick_folder().map(|p| p.to_string_lossy().to_string())));
app.hooks.pick_folder_at =
Some(Box::new(|dir: &str| rfd::FileDialog::new().set_directory(dir).pick_folder().map(|p| p.to_string_lossy().into_owned())));
// Linux: the XDG portal call blocks until the user chooses, which would stop the frame
// loop (GNOME offers to force-quit the window after about five seconds, #260). Show the
// dialog on its own thread instead; the app keeps running and acts on the answer.
#[cfg(target_os = "linux")]
{
app.hooks.run_dialog = Some(Box::new(run_dialog_off_thread));
}
app.hooks.open_path = Some(Box::new(open_path));
// Help menu, About and Discord links (#642): eframe is built without its `links`
// feature, so `ctx.open_url` does nothing on the desktop. Detached, so a launcher
// that waits for the browser never freezes the UI.
app.hooks.open_url = Some(Box::new(|url: &str| open::that_detached(url).map_err(|e| e.to_string())));
app.hooks.clipboard_media = Some(Box::new(clipboard_media));
// Settings ▸ General ▸ Interface Language ▸ System Language (#218).
app.hooks.cursor_screen_position = Some(Box::new(filmcraft_platform::cursor::cursor_screen_position));
app.hooks.system_languages = Some(Box::new(|| sys_locale::get_locales().collect()));
// Settings ▸ Appearance ▸ Appearance Mode ▸ Sync with system on Linux desktops whose
// compositor reports no theme to winit (no polling: see appearance.rs).
app.hooks.system_theme = appearance::service();
app.hooks.raise_without_focus = Some(Box::new(|| {
window_raise::raise_without_focus();
}));
app.hooks.pick_open_file = Some(Box::new(|filter: &str, exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_open_project = Some(Box::new(|| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
#[cfg(target_os = "macos")]
{
let (rx, update) = native_menu::install(&app, cc.egui_ctx.clone());
app.command_inbox = Some(rx);
app.hooks.shortcuts_changed = Some(update);
app.ui.show_menu_bar = false;
}
if let Some(port) = control_port {
let rx = control_server::start(port, cc.egui_ctx.clone());
app = app.with_control(rx);
}
Ok(Box::new(app))
}),
);
// However it ended, this start no longer counts as one that died.
graphics.finish();
// The window could not be created (no usable GPU / graphics driver, no display): say so
// instead of exiting silently, and keep the reason in the crash log.
if let Err(e) = &started {
@ -295,6 +163,185 @@ fn main() -> eframe::Result {
started
}
/// The main window's options; built for each start attempt (they hold a one-shot hook).
fn native_options(agent: bool) -> eframe::NativeOptions {
eframe::NativeOptions {
viewport: egui::ViewportBuilder::default()
.with_title("FilmCraft")
.with_inner_size([1600.0, 980.0])
.with_min_inner_size([900.0, 560.0])
.with_drag_and_drop(true)
.with_fullsize_content_view(true)
.with_titlebar_shown(false)
.with_title_shown(false)
// Started for an agent (`--control`): open without taking the user's keyboard focus.
.with_active(!agent)
.with_icon(app_icon())
// Wayland matches the window to `packaging/linux/ai.storyteller.filmcraft.desktop`.
.with_app_id(APP_ID),
event_loop_builder: agent_event_loop(agent),
..Default::default()
}
}
/// Creates the app once eframe has its window and graphics device; sets `created` when it runs,
/// and confirms the graphics mode once two frames have painted.
fn app_creator(launch: Launch, graphics: graphics::GraphicsStart, created: Arc<AtomicBool>) -> eframe::AppCreator<'static> {
let Launch { control_port, control_input_only, files, demo, startup_flag, recover, data_dir, graphics: _ } = launch;
Box::new(move |cc| {
created.store(true, Ordering::Relaxed);
// A fallback graphics mode says so in the window, with a button back to the default.
let notice = graphics.degraded().then(|| {
let start = graphics.clone();
filmcraft_ui_egui::graphics_notice::GraphicsNotice::new(graphics.mode().label(), Box::new(move || start.reset_to_default()))
});
let confirmed = AtomicBool::new(false);
cc.egui_ctx.on_end_pass(
"graphics-start",
Arc::new(move |ui| {
if ui.ctx().cumulative_frame_nr() >= 2 && !confirmed.swap(true, Ordering::Relaxed) {
graphics.confirm();
}
}),
);
let mut session = Session::default();
if let Some(dir) = data_dir.clone().or_else(default_data_dir) {
let mut cfg = AutosaveConfig::new(dir);
cfg.local_offset = local_offset;
if let Err(e) = session.start_autosave(cfg) {
log::warn!("auto-save and crash recovery unavailable: {e}");
}
}
// voice-over recording reads the microphone through cpal
session.voiceover.input = Some(Box::new(audio_in::CpalIn::new(&session.prefs.audio_hardware.device_class)));
let (project, media) = args::project_and_media(&files);
if let Some(p) = project {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
} else if !startup_flag && session.prefs.general.at_startup == "openMostRecent" {
// Settings ▸ General ▸ At Startup ▸ Open Most Recent
let recent = session.prefs.general.recent_projects.iter().find(|p| std::path::Path::new(p).exists()).cloned();
match recent {
Some(p) => {
if let Err(e) = session.execute("file.open", json!({"path": p})) {
log::error!("{e}");
}
}
None => {
let _ = session.execute("file.openDemoProject", json!({}));
}
}
} else if !startup_flag && session.prefs.general.at_startup == "emptyProject" {
} else if demo {
let _ = session.execute("file.openDemoProject", json!({}));
}
if !media.is_empty() {
let _ = session.execute("file.import", json!({"paths": media}));
}
if recover == Some(true) && !session.recovery_candidates().is_empty() {
let id = session.recovery_candidates()[0].id.clone();
match session.execute("file.recover", json!({"id": id})) {
Ok(r) => log::info!("recovered {r}"),
Err(e) => log::error!("recovery failed: {e}"),
}
}
let mut app = FilmcraftApp::new(session);
app.graphics_notice = notice;
if recover == Some(false) {
app.dialog = None;
}
app.integrated_titlebar = cfg!(target_os = "macos");
if let Some(rs) = cc.wgpu_render_state.clone()
&& std::env::var_os("FILMCRAFT_CPU_COMPOSITE").is_none()
{
app.set_wgpu(rs);
}
// Keep device selection available even when the default device is unavailable.
// Settings ▸ Audio Hardware is applied on the first frame (`apply_prefs`).
app.audio = Some(Box::new(audio::CpalOut::new()));
app.hooks.pick_files = Some(Box::new(|exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_files()
.unwrap_or_default()
.into_iter()
.map(|p| p.to_string_lossy().to_string())
.collect()
}));
// Link Media ▸ Locate…, Attach Proxies, Reconnect Full Resolution: one path, not imported.
app.hooks.pick_file_for_relink = Some(Box::new(|exts: &[&str], _hint| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save = Some(Box::new(|name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_save_as = Some(Box::new(|filter: &str, exts: &[&str], name: &str| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.set_file_name(name)
.save_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_folder = Some(Box::new(|| rfd::FileDialog::new().pick_folder().map(|p| p.to_string_lossy().to_string())));
app.hooks.pick_folder_at =
Some(Box::new(|dir: &str| rfd::FileDialog::new().set_directory(dir).pick_folder().map(|p| p.to_string_lossy().into_owned())));
// Linux: the XDG portal call blocks until the user chooses, which would stop the frame
// loop (GNOME offers to force-quit the window after about five seconds, #260). Show the
// dialog on its own thread instead; the app keeps running and acts on the answer.
#[cfg(target_os = "linux")]
{
app.hooks.run_dialog = Some(Box::new(run_dialog_off_thread));
}
app.hooks.open_path = Some(Box::new(open_path));
// Help menu, About and Discord links (#642): eframe is built without its `links`
// feature, so `ctx.open_url` does nothing on the desktop. Detached, so a launcher
// that waits for the browser never freezes the UI.
app.hooks.open_url = Some(Box::new(|url: &str| open::that_detached(url).map_err(|e| e.to_string())));
app.hooks.clipboard_media = Some(Box::new(clipboard_media));
// Settings ▸ General ▸ Interface Language ▸ System Language (#218).
app.hooks.cursor_screen_position = Some(Box::new(filmcraft_platform::cursor::cursor_screen_position));
app.hooks.system_languages = Some(Box::new(|| sys_locale::get_locales().collect()));
// Settings ▸ Appearance ▸ Appearance Mode ▸ Sync with system on Linux desktops whose
// compositor reports no theme to winit (no polling: see appearance.rs).
app.hooks.system_theme = appearance::service();
app.hooks.raise_without_focus = Some(Box::new(|| {
window_raise::raise_without_focus();
}));
app.hooks.pick_open_file = Some(Box::new(|filter: &str, exts: &[&str]| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
app.hooks.pick_open_project = Some(Box::new(|| {
rfd::FileDialog::new()
.add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"]))
.pick_file()
.map(|p| p.to_string_lossy().to_string())
}));
#[cfg(target_os = "macos")]
{
let (rx, update) = native_menu::install(&app, cc.egui_ctx.clone());
app.command_inbox = Some(rx);
app.hooks.shortcuts_changed = Some(update);
app.ui.show_menu_bar = false;
}
if let Some(port) = control_port {
let rx = control_server::start(port, cc.egui_ctx.clone());
app = if control_input_only { app.with_control_input_only(rx) } else { app.with_control(rx) };
}
Ok(Box::new(app))
})
}
/// Files copied in the file manager, else an image, on the system clipboard (Paste, #611).
fn clipboard_media() -> Option<filmcraft_ui_egui::ClipboardMedia> {
let mut clipboard = arboard::Clipboard::new().ok()?;
@ -326,6 +373,7 @@ fn open_path(path: &str, reveal: bool) -> Result<(), String> {
}
c
} else {
// Linux file managers have no select-file flag: open the parent folder instead.
let mut c = std::process::Command::new("xdg-open");
let p = std::path::Path::new(path);
c.arg(if reveal { p.parent().unwrap_or(p) } else { p });

41
apps/filmcraft/src/native_menu.rs
View File

@ -4,7 +4,7 @@
use std::sync::mpsc::{Receiver, channel};
use filmcraft_ui_egui::FilmcraftApp;
use filmcraft_ui_egui::menus::{MENUS, MenuItem as Item, menu_items};
use filmcraft_ui_egui::menus::{MENUS, MenuItem as Item, NATIVE_APP_MENU, menu_items};
use muda::accelerator::Accelerator;
use muda::{Menu, MenuEvent, MenuItem, PredefinedMenuItem, Submenu};
@ -28,25 +28,39 @@ pub fn install(app: &FilmcraftApp, ctx: egui::Context) -> (Receiver<String>, Sho
let mut titles: Vec<(String, Submenu)> = Vec::new();
let bar = Menu::new();
let app_menu = Submenu::new("FilmCraft", true);
let [settings_label, hide_label, hide_others_label, quit_label] = NATIVE_APP_MENU;
// FilmCraft ▸ Settings ▸ <category> (Premiere's app-menu layout; General is Cmd+,)
let settings = Submenu::new(app.ui.language.tr("Settings"), true);
titles.push(("Settings".into(), settings.clone()));
let settings = Submenu::new(app.ui.language.tr(settings_label), true);
titles.push((settings_label.into(), settings.clone()));
for it in items.iter().filter(|i| i.id.starts_with("app.settings.")) {
let label = it.label.trim_end_matches('…').to_string();
let mi = MenuItem::with_id(it.id.clone(), label, true, it.shortcut.as_deref().and_then(accel));
native.push((it.id.clone(), mi.clone()));
let _ = settings.append(&mi);
}
// About and Discord repeat Help menu commands: same translated labels, updated with them
let label = |id: &str, english: &str| items.iter().find(|i| i.id == id).map_or_else(|| english.to_string(), |i| i.label.clone());
let about = MenuItem::with_id("app.about", label("app.about", "About FilmCraft"), true, None);
let discord = MenuItem::with_id("help.discord", label("help.discord", "Join the ArtCraft Discord…"), true, None);
native.push(("app.about".into(), about.clone()));
native.push(("help.discord".into(), discord.clone()));
// the predefined items keep their native actions; our text lets them follow the language
let predefined = [
(hide_label, PredefinedMenuItem::hide(Some(app.ui.language.tr(hide_label)))),
(hide_others_label, PredefinedMenuItem::hide_others(Some(app.ui.language.tr(hide_others_label)))),
(quit_label, PredefinedMenuItem::quit(Some(app.ui.language.tr(quit_label)))),
];
let [(_, hide), (_, hide_others), (_, quit)] = &predefined;
let _ = app_menu.append_items(&[
&MenuItem::with_id("app.about", "About FilmCraft", true, None),
&MenuItem::with_id("help.discord", "Join the ArtCraft Discord…", true, None),
&about,
&discord,
&PredefinedMenuItem::separator(),
&settings,
&PredefinedMenuItem::separator(),
&PredefinedMenuItem::hide(None),
&PredefinedMenuItem::hide_others(None),
hide,
hide_others,
&PredefinedMenuItem::separator(),
&PredefinedMenuItem::quit(None),
quit,
]);
let _ = bar.append(&app_menu);
for top in MENUS {
@ -103,12 +117,23 @@ pub fn install(app: &FilmcraftApp, ctx: egui::Context) -> (Receiver<String>, Sho
filmcraft_ui_egui::i18n::Language::Uk
} else if checked("app.language.chinese") {
filmcraft_ui_egui::i18n::Language::ZhCn
} else if checked("app.language.german") {
filmcraft_ui_egui::i18n::Language::De
} else if checked("app.language.russian") {
filmcraft_ui_egui::i18n::Language::Ru
} else if checked("app.language.french") {
filmcraft_ui_egui::i18n::Language::Fr
} else if checked("app.language.indonesian") {
filmcraft_ui_egui::i18n::Language::Id
} else {
filmcraft_ui_egui::i18n::Language::En
};
for (title, submenu) in &titles {
submenu.set_text(language.tr(title));
}
for (label, item) in &predefined {
item.set_text(language.tr(label));
}
for (id, mi) in &native {
if let Some(it) = items.iter().find(|i| &i.id == id) {
mi.set_text(&it.label);

614
contributors/contributors.json
View File

@ -2,23 +2,23 @@
"schema": 1,
"repo": "storytold/filmcraft",
"branch": "main",
"head": "7b6c134472287db4367fca80bcb498240ba36567",
"head_date": "2026-10-09T15:02:38Z",
"commits": 539,
"head": "9efe8f3112194b58a30cf0449b569306077c67d4",
"head_date": "2026-10-10T22:34:28Z",
"commits": 812,
"unattributed_commits": 0,
"contributors": [
{
"login": "echelon",
"people_entry": true,
"github_name": null,
"prs": 48,
"commits": 337,
"lines_added": 311167,
"lines_deleted": 13200,
"prs": 49,
"commits": 366,
"lines_added": 313219,
"lines_deleted": 14128,
"binary_added": 74,
"binary_deleted": 0,
"first_commit": "2026-09-30T09:22:00Z",
"last_commit": "2026-10-09T15:02:38Z"
"last_commit": "2026-10-10T20:08:58Z"
},
{
"login": "storybored379",
@ -154,9 +154,9 @@
"login": "carlitosteso3",
"people_entry": false,
"github_name": null,
"prs": 15,
"commits": 15,
"lines_added": 1740,
"prs": 16,
"commits": 16,
"lines_added": 2009,
"lines_deleted": 207,
"binary_added": 0,
"binary_deleted": 0,
@ -232,27 +232,53 @@
"login": "DWAA1660",
"people_entry": false,
"github_name": null,
"prs": 2,
"commits": 2,
"lines_added": 209,
"lines_deleted": 0,
"prs": 3,
"commits": 3,
"lines_added": 441,
"lines_deleted": 10,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-07T22:57:38Z",
"last_commit": "2026-10-08T23:52:30Z"
"last_commit": "2026-10-09T22:01:08Z"
},
{
"login": "txiki739",
"people_entry": false,
"github_name": null,
"prs": 5,
"commits": 8,
"lines_added": 7976,
"lines_deleted": 3333,
"prs": 8,
"commits": 11,
"lines_added": 8160,
"lines_deleted": 3347,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-07T23:41:48Z",
"last_commit": "2026-10-09T09:28:24Z"
"last_commit": "2026-10-09T14:06:49Z"
},
{
"login": "mutombopr",
"people_entry": false,
"github_name": null,
"prs": 4,
"commits": 19,
"lines_added": 10421,
"lines_deleted": 337,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-08T03:57:26Z",
"last_commit": "2026-10-10T17:46:20Z"
},
{
"login": "ahmedhosnypro",
"people_entry": true,
"github_name": null,
"prs": 2,
"commits": 5,
"lines_added": 473,
"lines_deleted": 80,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-08T05:32:08Z",
"last_commit": "2026-10-09T22:00:30Z"
},
{
"login": "wstock",
@ -284,14 +310,14 @@
"login": "MrCloudy2",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 132,
"lines_deleted": 4,
"prs": 6,
"commits": 7,
"lines_added": 1293,
"lines_deleted": 46,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-08T08:53:54Z",
"last_commit": "2026-10-08T08:53:54Z"
"last_commit": "2026-10-10T15:14:34Z"
},
{
"login": "switch87",
@ -336,27 +362,27 @@
"login": "MarekSpodBiedry",
"people_entry": false,
"github_name": null,
"prs": 2,
"commits": 2,
"lines_added": 74,
"lines_deleted": 6,
"prs": 29,
"commits": 29,
"lines_added": 1048,
"lines_deleted": 137,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-08T13:44:56Z",
"last_commit": "2026-10-09T08:45:28Z"
"last_commit": "2026-10-10T21:22:52Z"
},
{
"login": "Tom1tk",
"people_entry": false,
"github_name": null,
"prs": 2,
"commits": 2,
"lines_added": 47,
"lines_deleted": 26,
"prs": 3,
"commits": 3,
"lines_added": 70,
"lines_deleted": 30,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-08T13:47:40Z",
"last_commit": "2026-10-08T14:56:39Z"
"last_commit": "2026-10-09T14:19:08Z"
},
{
"login": "Pathos0925",
@ -375,14 +401,14 @@
"login": "mjohnson7cd",
"people_entry": false,
"github_name": null,
"prs": 2,
"commits": 1,
"lines_added": 100,
"lines_deleted": 2,
"prs": 3,
"commits": 2,
"lines_added": 121,
"lines_deleted": 5,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-08T20:36:28Z",
"last_commit": "2026-10-08T20:36:28Z"
"last_commit": "2026-10-10T00:47:33Z"
},
{
"login": "Oldrich333",
@ -401,14 +427,14 @@
"login": "cshapiro",
"people_entry": false,
"github_name": null,
"prs": 3,
"commits": 5,
"lines_added": 3128,
"lines_deleted": 111,
"prs": 4,
"commits": 6,
"lines_added": 3165,
"lines_deleted": 116,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-08T23:28:32Z",
"last_commit": "2026-10-09T02:57:30Z"
"last_commit": "2026-10-09T15:59:56Z"
},
{
"login": "rossaai",
@ -440,14 +466,14 @@
"login": "JordyUSA",
"people_entry": false,
"github_name": null,
"prs": 2,
"commits": 3,
"lines_added": 4158,
"lines_deleted": 283,
"prs": 6,
"commits": 9,
"lines_added": 4948,
"lines_deleted": 405,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T01:35:57Z",
"last_commit": "2026-10-09T02:53:28Z"
"last_commit": "2026-10-10T15:06:45Z"
},
{
"login": "jerrykimo",
@ -466,14 +492,27 @@
"login": "swillunderscore",
"people_entry": true,
"github_name": null,
"prs": 1,
"commits": 3,
"lines_added": 1008,
"lines_deleted": 13,
"prs": 2,
"commits": 6,
"lines_added": 2407,
"lines_deleted": 63,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T01:58:37Z",
"last_commit": "2026-10-09T02:19:21Z"
"last_commit": "2026-10-10T00:20:25Z"
},
{
"login": "jnohclee-rgb",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 19,
"lines_deleted": 1,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T02:14:06Z",
"last_commit": "2026-10-09T02:14:06Z"
},
{
"login": "IppSec",
@ -492,14 +531,14 @@
"login": "sanyamsingh001",
"people_entry": true,
"github_name": null,
"prs": 2,
"commits": 2,
"lines_added": 323,
"lines_deleted": 25,
"prs": 5,
"commits": 5,
"lines_added": 1169,
"lines_deleted": 93,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T04:06:37Z",
"last_commit": "2026-10-09T05:44:39Z"
"last_commit": "2026-10-09T23:00:34Z"
},
{
"login": "ItzBloxy",
@ -513,6 +552,435 @@
"binary_deleted": 0,
"first_commit": "2026-10-09T09:27:48Z",
"last_commit": "2026-10-09T10:17:08Z"
},
{
"login": "dmatviichuk",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 464,
"lines_deleted": 6,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T12:18:42Z",
"last_commit": "2026-10-09T12:18:42Z"
},
{
"login": "David-T-Campos",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 7,
"lines_added": 61,
"lines_deleted": 8,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T15:25:06Z",
"last_commit": "2026-10-09T15:27:55Z"
},
{
"login": "lloyd-nich",
"people_entry": false,
"github_name": null,
"prs": 0,
"commits": 3,
"lines_added": 1477,
"lines_deleted": 121,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T15:58:49Z",
"last_commit": "2026-10-10T07:02:57Z"
},
{
"login": "haranishi",
"people_entry": false,
"github_name": null,
"prs": 4,
"commits": 4,
"lines_added": 3700,
"lines_deleted": 92,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T17:14:19Z",
"last_commit": "2026-10-10T08:34:16Z"
},
{
"login": "vbasky",
"people_entry": false,
"github_name": null,
"prs": 10,
"commits": 11,
"lines_added": 1314,
"lines_deleted": 215,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T17:37:44Z",
"last_commit": "2026-10-10T16:41:59Z"
},
{
"login": "ossini",
"people_entry": true,
"github_name": "Oskar Sobania",
"prs": 2,
"commits": 12,
"lines_added": 1381,
"lines_deleted": 149,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T18:43:22Z",
"last_commit": "2026-10-10T17:17:47Z"
},
{
"login": "liangforstudy",
"people_entry": true,
"github_name": null,
"prs": 3,
"commits": 3,
"lines_added": 575,
"lines_deleted": 20,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T21:22:05Z",
"last_commit": "2026-10-10T09:40:58Z"
},
{
"login": "ReReddo",
"people_entry": true,
"github_name": null,
"prs": 12,
"commits": 14,
"lines_added": 1918,
"lines_deleted": 126,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T21:30:29Z",
"last_commit": "2026-10-10T19:45:20Z"
},
{
"login": "MasakatsuFunaki",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 261,
"lines_deleted": 13,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T21:49:24Z",
"last_commit": "2026-10-09T21:49:24Z"
},
{
"login": "maccagnit",
"people_entry": false,
"github_name": null,
"prs": 4,
"commits": 4,
"lines_added": 221,
"lines_deleted": 30,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T23:00:05Z",
"last_commit": "2026-10-10T15:22:56Z"
},
{
"login": "digitaldecaycreations-lgtm",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 11,
"lines_deleted": 0,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-09T23:20:58Z",
"last_commit": "2026-10-09T23:20:58Z"
},
{
"login": "dylanmtaylor",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 37,
"lines_deleted": 42,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T00:04:17Z",
"last_commit": "2026-10-10T00:04:17Z"
},
{
"login": "sawizzle",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 2,
"lines_added": 23,
"lines_deleted": 6,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T00:06:15Z",
"last_commit": "2026-10-10T04:50:49Z"
},
{
"login": "luniiya",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 697,
"lines_deleted": 6,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T00:14:55Z",
"last_commit": "2026-10-10T00:14:55Z"
},
{
"login": "chuygarcia92",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 38,
"lines_deleted": 33,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T02:20:17Z",
"last_commit": "2026-10-10T02:20:17Z"
},
{
"login": "Hathor875",
"people_entry": true,
"github_name": "Krzysztof Cieślik",
"prs": 1,
"commits": 1,
"lines_added": 109,
"lines_deleted": 4,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T02:21:06Z",
"last_commit": "2026-10-10T02:21:06Z"
},
{
"login": "Guilhermeb-Ferrarezi",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 2,
"lines_added": 161,
"lines_deleted": 1,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T02:43:20Z",
"last_commit": "2026-10-10T02:48:16Z"
},
{
"login": "mitchmckinstry",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 3,
"lines_added": 399,
"lines_deleted": 11,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T03:14:19Z",
"last_commit": "2026-10-10T03:25:58Z"
},
{
"login": "gyroninja",
"people_entry": false,
"github_name": null,
"prs": 2,
"commits": 2,
"lines_added": 178,
"lines_deleted": 12,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T04:32:42Z",
"last_commit": "2026-10-10T10:14:18Z"
},
{
"login": "hadmark",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 4,
"lines_added": 3608,
"lines_deleted": 4,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T06:08:41Z",
"last_commit": "2026-10-10T06:32:49Z"
},
{
"login": "SilentYeti",
"people_entry": false,
"github_name": null,
"prs": 2,
"commits": 3,
"lines_added": 1562,
"lines_deleted": 72,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T06:17:11Z",
"last_commit": "2026-10-10T09:17:16Z"
},
{
"login": "adurr21",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 287,
"lines_deleted": 68,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T07:03:03Z",
"last_commit": "2026-10-10T07:03:03Z"
},
{
"login": "maxm11",
"people_entry": true,
"github_name": null,
"prs": 3,
"commits": 6,
"lines_added": 487,
"lines_deleted": 38,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T07:19:38Z",
"last_commit": "2026-10-10T14:18:27Z"
},
{
"login": "Ni-zav",
"people_entry": false,
"github_name": null,
"prs": 10,
"commits": 13,
"lines_added": 498,
"lines_deleted": 65,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T08:47:24Z",
"last_commit": "2026-10-10T21:55:59Z"
},
{
"login": "ivandrozdov1505-coder",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 404,
"lines_deleted": 5,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T10:40:22Z",
"last_commit": "2026-10-10T10:40:22Z"
},
{
"login": "KattKnight",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 3,
"lines_added": 335,
"lines_deleted": 9,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T10:56:07Z",
"last_commit": "2026-10-10T13:00:21Z"
},
{
"login": "keller-lukas",
"people_entry": true,
"github_name": null,
"prs": 1,
"commits": 2,
"lines_added": 480,
"lines_deleted": 15,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T11:06:02Z",
"last_commit": "2026-10-10T11:06:02Z"
},
{
"login": "4papuce",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 39,
"lines_deleted": 0,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T12:43:28Z",
"last_commit": "2026-10-10T12:43:28Z"
},
{
"login": "vangel76",
"people_entry": true,
"github_name": null,
"prs": 49,
"commits": 53,
"lines_added": 11727,
"lines_deleted": 752,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T13:47:11Z",
"last_commit": "2026-10-10T22:34:28Z"
},
{
"login": "abdelkabirouadoukou",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 30,
"lines_deleted": 8,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T15:54:00Z",
"last_commit": "2026-10-10T15:54:00Z"
},
{
"login": "impostazioni",
"people_entry": false,
"github_name": null,
"prs": 3,
"commits": 3,
"lines_added": 596,
"lines_deleted": 34,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T17:29:09Z",
"last_commit": "2026-10-10T20:52:28Z"
},
{
"login": "alex-brazh",
"people_entry": false,
"github_name": null,
"prs": 1,
"commits": 1,
"lines_added": 159,
"lines_deleted": 4,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": "2026-10-10T19:55:52Z",
"last_commit": "2026-10-10T19:55:52Z"
},
{
"login": "LloydNicholson",
"people_entry": false,
"github_name": null,
"prs": 3,
"commits": 0,
"lines_added": 0,
"lines_deleted": 0,
"binary_added": 0,
"binary_deleted": 0,
"first_commit": null,
"last_commit": null
}
],
"models": [
@ -521,27 +989,27 @@
"company": "Anthropic",
"model": "Claude Opus",
"version": "5.5",
"commits": 474,
"lines_added": 346706,
"lines_deleted": 19057
"commits": 602,
"lines_added": 374868,
"lines_deleted": 20972
},
{
"id": "anthropic/claude-sonnet/5.5",
"company": "Anthropic",
"model": "Claude Sonnet",
"version": "5.5",
"commits": 18,
"lines_added": 13935,
"lines_deleted": 854
"commits": 25,
"lines_added": 18034,
"lines_deleted": 957
},
{
"id": "openai/codex",
"company": "OpenAI",
"model": "Codex",
"version": "",
"commits": 11,
"lines_added": 2057,
"lines_deleted": 361
"commits": 16,
"lines_added": 3654,
"lines_deleted": 503
},
{
"id": "anthropic/claude-fable/5.1",
@ -557,9 +1025,9 @@
"company": "Anthropic",
"model": "Claude",
"version": "",
"commits": 1,
"lines_added": 235,
"lines_deleted": 22
"commits": 2,
"lines_added": 496,
"lines_deleted": 35
},
{
"id": "anthropic/claude-opus/4.8",

11
contributors/people.toml
View File

@ -57,11 +57,22 @@ sync_github_name = false
[people.liangforstudy]
sync_github_name = true
[people.keller-lukas]
real_name = "Lukas Keller"
display_name = "Lukas Keller"
sync_github_name = false
[people.maxm11]
real_name = "Max Murphy"
display_name = "Max Murphy"
sync_github_name = false
[people.ossini]
sync_github_name = true
[people.opentechnepal]
sync_github_name = true
[people.vangel76]
real_name = "Valentin Klein"
display_name = "Valentin Klein"

3
crates/aac/README.md
View File

@ -48,6 +48,9 @@ from bitrate/quality), `tns`, `ms_stereo`, `window_shape` (sine/KBD), `adts` (pr
| 7 | 0 + PCE | C L R Ls Rs Cs LFE (PCE in the ASC; repeated in every AU when ADTS-framed) |
| 8 | 7 | C Lc Rc L R Ls Rs LFE |
The decoder also reads configurations 11 (6.1: C L R Ls Rs Cs LFE) and 12 (7.1: C L R Ls Rs Lsr Rsr
LFE) from ISO/IEC 14496-3 Amd 4; ffmpeg 9 writes 7.1 AAC as configuration 12.
`Decoder::layout()` reports the element layout (it follows a PCE for configuration 0).
## Encoder

14
crates/aac/src/config.rs
View File

@ -136,7 +136,8 @@ impl ProgramConfig {
}
}
/// Element layout of `channelConfiguration` 1..=7 (tags as conventionally assigned).
/// Element layout of `channelConfiguration` 1..=7, 11 and 12 (tags as conventionally assigned; 11 and
/// 12 from ISO/IEC 14496-3 Amd 4: 6.1 with a back centre, 7.1 with back surrounds).
pub fn config_layout(channel_config: u8) -> Option<Vec<(ElementType, u8)>> {
use ElementType::*;
Some(match channel_config {
@ -146,7 +147,8 @@ pub fn config_layout(channel_config: u8) -> Option<Vec<(ElementType, u8)>> {
4 => vec![(Sce, 0), (Cpe, 0), (Sce, 1)],
5 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1)],
6 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Lfe, 0)],
7 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)],
7 | 12 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)],
11 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)],
_ => return None,
})
}
@ -388,6 +390,14 @@ mod tests {
assert_eq!(AudioSpecificConfig::parse(&a.to_bytes()).unwrap().channels(), 6);
}
#[test]
fn amd4_channel_configurations() {
// 6.1 and 7.1 back (#922); 22.2 and 7.1 with heights stay unsupported
let channels = |c| AudioSpecificConfig::parse(&AudioSpecificConfig::lc(3, c).to_bytes()).map(|a| a.channels()).ok();
assert_eq!([7, 11, 12].map(channels), [Some(8), Some(7), Some(8)]);
assert_eq!([13, 14].map(config_layout), [None, None]);
}
#[test]
fn asc_with_pce() {
let pce = ProgramConfig {

2
crates/ac3/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "filmcraft-ac3"
description = "Clean-room AC-3 (Dolby Digital, ATSC A/52) audio decoder"
description = "Clean-room AC-3 and E-AC-3 (Dolby Digital / Dolby Digital Plus, ATSC A/52) audio decoder"
version.workspace = true
edition.workspace = true
license.workspace = true

48
crates/ac3/README.md
View File

@ -1,6 +1,6 @@
# filmcraft-ac3
Clean-room AC-3 (Dolby Digital) audio decoder in pure Rust. Layer L0: depends only on
Clean-room AC-3 (Dolby Digital) and E-AC-3 (Dolby Digital Plus) audio decoder in pure Rust. Layer L0: depends only on
`thiserror`. Builds for `wasm32-unknown-unknown`; no `unsafe`. No GPL/LGPL code (FFmpeg,
liba52) was consulted; FFmpeg is used only as an external fixture generator and test oracle.
@ -11,14 +11,17 @@ publicly available standard: §5 bit stream syntax and semantics, §6 decoding o
exponents, §7.2 parametric bit allocation (Tables 7.6-7.16 extracted mechanically from the
standard's text), §7.3 mantissas and dither, §7.4 coupling, §7.5 rematrixing, §7.7.1 dynamic
range, §7.9 transforms and block switching. The window is computed as the Kaiser-Bessel derived
window (α = 5) and checked against Table 7.33 to 6·10⁻⁶.
window (α = 5) and checked against Table 7.33 to 6·10⁻⁶. E-AC-3: Annex E §E2 bit stream syntax
(syncinfo, bsi, audfrm, audblk; Tables E2.1-E2.13, frame exponent strategies from Table E2.10),
§E3.3 modified parameters, §E3.4.2 AHT helper variables (to detect AHT), §E3.6 spectral extension
(attenuation Table E3.14, which is 2^(-(code + 1)(bin + 1) / 15), checked against the table).
## API
```rust
let mut dec = filmcraft_ac3::Decoder::new();
let h = filmcraft_ac3::parse_header(frame)?; // rate, channels, frame size
let out = dec.decode(frame)?; // 1536 samples per channel, WAV channel order
let out = dec.decode(frame)?; // h.samples() per channel, WAV channel order
dec.reset(); // before decoding from another position
```
@ -26,7 +29,7 @@ dec.reset(); // before decoding from another
- Every audio coding mode (1+1, 1/0, 2/0, 3/0, 2/1, 3/1, 2/2, 3/2) with or without LFE; 32 /
44.1 / 48 kHz and every frame size; bsid ≤ 8 (9 and 10, the half / quarter-rate variants, are
accepted). E-AC-3 (bsid 11-16) is refused with `Error::Unsupported`.
accepted).
- D15 / D25 / D45 exponents with reuse across blocks; the parametric bit allocation in the
standard's fixed-point steps, delta bit allocation, the all-zero SNR offset special case;
grouped 3-, 5- and 11-level mantissas (groups shared across exponent sets), 7- and 15-level
@ -43,6 +46,26 @@ dec.reset(); // before decoding from another
- Errors (bad exponents, reserved codes, blocks running past the frame) return `Err`; garbage
never panics.
## E-AC-3 (bsid 11-16)
- Independent substream 0 is decoded (stream types 0 and 2); dependent substreams (the channels
beyond 5.1 of a 7.1 program) and further independent programs decode to no channels, so a 7.1
stream plays as its 5.1 core. `Header::is_primary` tells them apart.
- 1, 2, 3 or 6 blocks per syncframe (`Header::samples`), 32 / 44.1 / 48 kHz and the reduced rates
16 / 22.05 / 24 kHz, every acmod with or without LFE.
- The whole bsi (mixing and informational metadata, custom channel maps, addbsi) is parsed and
skipped; audfrm: per-block or frame-based exponent strategies (exponents may be reused across
syncframes of fewer than six blocks; the decoder keeps them until `reset`), the syntax-enable
flags and their defaults, SNR offset strategies 1-3, converter fields, block start information.
- Audio blocks share the AC-3 path: coupling with the default or transmitted band structure
(`firstcplcos` / `firstcplleak` states), rematrixing with the Annex E band count, end mantissas
derived from spectral extension.
- Spectral extension: band structure, coordinates, blending factors, translation with wrap,
banded RMS, the border / wrap-point notch filter, noise blending and scaling (§E3.6.4).
- Refused with `Error::Unsupported`: the adaptive hybrid transform (when a channel uses it; `ahte`
alone is fine) and enhanced coupling. Transient pre-noise processing data is read and not
applied.
## Accuracy (vs FFmpeg as an external oracle)
AC-3 output is not bit-exact across decoders: zero-bit mantissas are filled with
@ -56,10 +79,17 @@ FFmpeg's decode (worst channel SNR):
| 1/0 32 kHz 96 kb/s | 94.1 dB | 95.7 dB |
| 3/2 + LFE 48 kHz 448 kb/s | 81.4 dB | 90.2 dB |
| 2/0 pink noise 128 kb/s (many zero-bit mantissas) | 27.7 dB | 27.9 dB |
| E-AC-3 2/0 48 kHz 192 kb/s (tones, chirp) | 64.8 dB | 74.7 dB |
| E-AC-3 2/0 44.1 kHz 64 kb/s with coupling and rematrixing | 65.5 dB | 66.1 dB |
| E-AC-3 1/0 32 kHz 64 kb/s | 91.0 dB | 90.1 dB |
| E-AC-3 3/2 + LFE 48 kHz 384 kb/s with mixing / informational metadata | 81.6 dB | 85.3 dB |
| E-AC-3 2/0 pink noise 96 kb/s | 48.6 dB | 47.9 dB |
The noise row shows the difference is dither: two of our own decodes with different dither
sequences differ as much as we differ from FFmpeg. In MPEG-2 TS (AVCHD-style `.mts`) and VOB the
maximum sample difference to FFmpeg is 2.3·10⁻³.
maximum sample difference to FFmpeg is 2.3·10⁻³. E-AC-3 through `filmcraft-codecs`
(`tests/eac3_oracle.rs`): 5.1 in MP4 and Matroska 91.3 dB, stereo in QuickTime 87.0 dB and
MPEG-TS 79.5 dB against FFmpeg's decode.
## Gaps
@ -67,4 +97,10 @@ maximum sample difference to FFmpeg is 2.3·10⁻³.
§7.9.4.2 but not checked against an oracle.
- No downmixing, no heavy compression (`compr`) or dialogue normalisation (both are metadata
for playback systems), no CRC check (corrupt frames are caught by consistency checks).
- E-AC-3 is not decoded.
- FFmpeg's E-AC-3 encoder writes six-block syncframes with frame-based exponent strategies,
coupling with the default band structure and none of the optional per-block syntax. Spectral
extension (no free encoder writes it), fewer blocks per syncframe, the reduced rates and the
per-block syntax (shared with AC-3) are implemented from the standard and covered by unit tests
on synthetic data, not by an oracle.
- E-AC-3: no adaptive hybrid transform, enhanced coupling, transient pre-noise processing, or
channels from dependent substreams (7.1 plays as 5.1).

634
crates/ac3/src/eac3.rs Normal file
View File

@ -0,0 +1,634 @@
//! E-AC-3 (ATSC A/52:2012 Annex E): the syncframe header and bit stream information (§E2.2.2),
//! the audio frame (§E2.2.3) and spectral extension (§E2.2.4, §E3.6). The audio blocks share the
//! AC-3 decoder (`Decoder::audio_block`), which reads the E-AC-3 differences from [`FrameInfo`].
use crate::{Bits, Error, Header, NFCHANS, Result};
/// Sample rates by fscod, and by fscod2 when fscod is 3 (Tables E2.2, E2.3).
const RATES: [u32; 3] = [48_000, 44_100, 32_000];
const REDUCED_RATES: [u32; 3] = [24_000, 22_050, 16_000];
/// Audio blocks per syncframe by numblkscod (Table E2.4).
const BLOCKS: [u8; 4] = [1, 2, 3, 6];
/// Exponent strategies of the six blocks for each frame-based strategy code (Table E2.10):
/// 0 reuse, 1 D15, 2 D25, 3 D45.
const FRAME_EXPSTR: [[u8; 6]; 32] = [
[1, 0, 0, 0, 0, 0],
[1, 0, 0, 0, 0, 3],
[1, 0, 0, 0, 2, 0],
[1, 0, 0, 0, 3, 3],
[2, 0, 0, 2, 0, 0],
[2, 0, 0, 2, 0, 3],
[2, 0, 0, 3, 2, 0],
[2, 0, 0, 3, 3, 3],
[2, 0, 1, 0, 0, 0],
[2, 0, 2, 0, 0, 3],
[2, 0, 2, 0, 2, 0],
[2, 0, 2, 0, 3, 3],
[2, 0, 3, 2, 0, 0],
[2, 0, 3, 2, 0, 3],
[2, 0, 3, 3, 2, 0],
[2, 0, 3, 3, 3, 3],
[3, 1, 0, 0, 0, 0],
[3, 1, 0, 0, 0, 3],
[3, 2, 0, 0, 2, 0],
[3, 2, 0, 0, 3, 3],
[3, 2, 0, 2, 0, 0],
[3, 2, 0, 2, 0, 3],
[3, 2, 0, 3, 2, 0],
[3, 2, 0, 3, 3, 3],
[3, 3, 1, 0, 0, 0],
[3, 3, 2, 0, 0, 3],
[3, 3, 2, 0, 2, 0],
[3, 3, 2, 0, 3, 3],
[3, 3, 3, 2, 0, 0],
[3, 3, 3, 2, 0, 3],
[3, 3, 3, 3, 2, 0],
[3, 3, 3, 3, 3, 3],
];
/// Default coupling band structure by coupling sub-band (Table E2.12, "Default Coupling Banding
/// Structure"; sub-band 0 never starts a combined band).
const DEFAULT_CPLBNDSTRC: [bool; 18] = {
let mut t = [false; 18];
let ones = [8, 10, 11, 13, 14, 15, 16, 17];
let mut i = 0;
while i < ones.len() {
t[ones[i]] = true;
i += 1;
}
t
};
/// Default spectral extension band structure by sub-band (Table E2.11).
const DEFAULT_SPXBNDSTRC: [bool; 17] = {
let mut t = [false; 17];
let mut i = 8;
while i < 17 {
t[i] = true;
i += 2;
}
t
};
/// First transform coefficient of spectral extension sub-band `k` (Table E3.13, k ≤ 17).
const fn spx_band_bin(k: usize) -> usize {
25 + 12 * k
}
/// Spectral extension attenuation (Table E3.14): spxattentab[code][binindex] = 2^(-(code + 1) ·
/// (binindex + 1) / 15), which the table lists to nine decimals.
fn spx_atten(code: usize, binindex: usize) -> f32 {
let e = (code as f64 + 1.0) * (binindex as f64 + 1.0) / 15.0;
(-e).exp2() as f32
}
/// Parse the header of an E-AC-3 syncframe (bsid 11-16) at the start of `b` (at least 7 bytes).
pub(crate) fn parse_header(b: &[u8], bsid: u8) -> Result<Header> {
let mut r = Bits::new(b.get(2..).unwrap_or_default());
let strmtyp = r.read(2) as u8;
let substreamid = r.read(3) as u8;
let words = r.read(11) as usize + 1;
let fscod = r.read(2) as usize;
let (sample_rate, blocks) = match RATES.get(fscod) {
Some(&rate) => (rate, BLOCKS[r.read(2) as usize & 3]),
None => (*REDUCED_RATES.get(r.read(2) as usize).ok_or(Error::Invalid("reserved fscod2"))?, 6),
};
let acmod = r.read(3) as u8;
let lfeon = r.bit();
if strmtyp == 3 {
return Err(Error::Unsupported("reserved E-AC-3 stream type 3".into()));
}
let frame_bytes = words * 2;
if frame_bytes < 8 {
return Err(Error::Invalid("E-AC-3 syncframe too short"));
}
// frame bits per second of audio
let bits = frame_bytes as u64 * 8 * u64::from(sample_rate) / (256 * u64::from(blocks));
let bitrate_kbps = u32::try_from(bits / 1000).unwrap_or(u32::MAX);
Ok(Header { sample_rate, bitrate_kbps, frame_bytes, bsid, acmod, lfeon, blocks, strmtyp, substreamid })
}
/// What the audio frame and blocks need from the bit stream information.
pub(crate) struct Bsi {
pub strmtyp: u8,
/// Sample rate code for the bit allocation's hearing threshold: fscod, or fscod2 at the
/// reduced rates.
pub fscod: usize,
pub numblkscod: usize,
}
/// Read syncinfo and bsi (§E2.2.1-2), leaving `r` at the audio frame.
pub(crate) fn read_bsi(r: &mut Bits) -> Result<Bsi> {
r.skip(16); // syncword
let strmtyp = r.read(2) as u8;
r.skip(3 + 11); // substreamid, frmsiz
let fscod = r.read(2) as usize;
let (rate_code, numblkscod) = if fscod == 3 {
let fscod2 = r.read(2) as usize;
if fscod2 == 3 {
return Err(Error::Invalid("reserved fscod2"));
}
(fscod2, 3)
} else {
(fscod, r.read(2) as usize)
};
let nblocks = BLOCKS[numblkscod & 3] as usize;
let acmod = r.read(3) as usize;
let lfeon = r.bit();
r.skip(5 + 5); // bsid, dialnorm
if r.bit() {
r.skip(8); // compr
}
if acmod == 0 {
r.skip(5); // dialnorm2
if r.bit() {
r.skip(8); // compr2
}
}
if strmtyp == 1 && r.bit() {
r.skip(16); // chanmap
}
if r.bit() {
// mixing metadata
if acmod > 2 {
r.skip(2); // dmixmod
}
if acmod & 1 != 0 && acmod > 2 {
r.skip(6); // ltrtcmixlev, lorocmixlev
}
if acmod & 4 != 0 {
r.skip(6); // ltrtsurmixlev, lorosurmixlev
}
if lfeon && r.bit() {
r.skip(5); // lfemixlevcod
}
if strmtyp == 0 {
if r.bit() {
r.skip(6); // pgmscl
}
if acmod == 0 && r.bit() {
r.skip(6); // pgmscl2
}
if r.bit() {
r.skip(6); // extpgmscl
}
match r.read(2) {
1 => r.skip(5), // premixcmpsel, drcsrc, premixcmpscl
2 => r.skip(12),
// mixdeflen: the mixdata field (through mixdatafill) is mixdeflen + 2 bytes
3 => {
let len = r.read(5) as usize;
r.skip(8 * (len + 2));
}
_ => {}
}
if acmod < 2 {
if r.bit() {
r.skip(14); // panmean, paninfo
}
if acmod == 0 && r.bit() {
r.skip(14); // panmean2, paninfo2
}
}
if r.bit() {
// frame mixing configuration information
if numblkscod == 0 {
r.skip(5);
} else {
for _ in 0..nblocks {
if r.bit() {
r.skip(5);
}
}
}
}
}
}
if r.bit() {
// informational metadata
r.skip(5); // bsmod, copyrightb, origbs
if acmod == 2 {
r.skip(4); // dsurmod, dheadphonmod
}
if acmod >= 6 {
r.skip(2); // dsurexmod
}
if r.bit() {
r.skip(8); // mixlevel, roomtyp, adconvtyp
}
if acmod == 0 && r.bit() {
r.skip(8); // mixlevel2, roomtyp2, adconvtyp2
}
if fscod < 3 {
r.skip(1); // sourcefscod
}
}
if strmtyp == 0 && numblkscod != 3 {
r.skip(1); // convsync
}
if strmtyp == 2 && (numblkscod == 3 || r.bit()) {
r.skip(6); // blkid → frmsizecod
}
if r.bit() {
let l = r.read(6) as usize;
r.skip((l + 1) * 8); // addbsi
}
Ok(Bsi { strmtyp, fscod: rate_code, numblkscod })
}
/// The audio frame (§E2.2.3) and the state that runs from block to block within a syncframe.
pub(crate) struct FrameInfo {
pub strmtyp: u8,
pub blkswe: bool,
pub dithflage: bool,
pub bamode: bool,
pub frmfgaincode: bool,
pub dbaflde: bool,
pub skipflde: bool,
pub snroffststr: u8,
pub frmcsnroffst: i32,
pub frmfsnroffst: i32,
pub cplstre: [bool; 6],
pub cplinu: [bool; 6],
pub cplexpstr: [u8; 6],
pub chexpstr: [[u8; 5]; 6],
pub lfeexpstr: [u8; 6],
/// Spectral extension attenuation code per channel (spxattencod), when in use.
pub spxatten: [Option<usize>; 5],
pub firstcplcos: [bool; 5],
pub firstcplleak: bool,
/// Coupling band structure by absolute coupling sub-band.
pub cplbndstrc: [bool; 18],
pub spx: Spx,
}
impl FrameInfo {
/// Read the audio frame. `words` is the syncframe size in 16-bit words.
pub(crate) fn read(r: &mut Bits, bsi: &Bsi, acmod: usize, lfeon: bool, nblocks: usize, words: usize) -> Result<FrameInfo> {
let nf = NFCHANS[acmod & 7];
let six = bsi.numblkscod == 3;
let (expstre, ahte) = if six { (r.bit(), r.bit()) } else { (true, false) };
let snroffststr = r.read(2) as u8;
if snroffststr == 3 {
return Err(Error::Invalid("reserved SNR offset strategy"));
}
let transproce = r.bit();
let mut e = FrameInfo {
strmtyp: bsi.strmtyp,
blkswe: r.bit(),
dithflage: r.bit(),
bamode: r.bit(),
frmfgaincode: r.bit(),
dbaflde: r.bit(),
skipflde: r.bit(),
snroffststr,
frmcsnroffst: 0,
frmfsnroffst: 0,
cplstre: [false; 6],
cplinu: [false; 6],
cplexpstr: [0; 6],
chexpstr: [[0; 5]; 6],
lfeexpstr: [0; 6],
spxatten: [None; 5],
firstcplcos: [true; 5],
firstcplleak: true,
cplbndstrc: DEFAULT_CPLBNDSTRC,
spx: Spx::new(),
};
let spxattene = r.bit();
if acmod > 1 {
e.cplstre[0] = true;
e.cplinu[0] = r.bit();
for blk in 1..nblocks {
e.cplstre[blk] = r.bit();
e.cplinu[blk] = if e.cplstre[blk] { r.bit() } else { e.cplinu[blk - 1] };
}
}
if expstre {
for blk in 0..nblocks {
if e.cplinu[blk] {
e.cplexpstr[blk] = r.read(2) as u8;
}
for ch in 0..nf {
e.chexpstr[blk][ch] = r.read(2) as u8;
}
}
} else {
// frame-based strategies (six blocks only)
if acmod > 1 && e.cplinu.iter().any(|&c| c) {
e.cplexpstr = FRAME_EXPSTR[r.read(5) as usize & 31];
}
for ch in 0..nf {
let s = FRAME_EXPSTR[r.read(5) as usize & 31];
for (blk, &v) in s.iter().enumerate() {
e.chexpstr[blk][ch] = v;
}
}
}
if lfeon {
for blk in 0..nblocks {
e.lfeexpstr[blk] = r.read(1) as u8;
}
}
if bsi.strmtyp == 0 && (six || r.bit()) {
r.skip(5 * nf); // convexpstr
}
if ahte {
// AHT flags exist for channels whose exponents are sent once in the frame (§E3.4.2)
let unsupported = || Err(Error::Unsupported("E-AC-3 adaptive hybrid transform".into()));
let ncplblks = e.cplinu.iter().filter(|&&c| c).count();
let ncplregs = (0..6).filter(|&b| e.cplstre[b] || e.cplexpstr[b] != 0).count();
if ncplblks == 6 && ncplregs == 1 && r.bit() {
return unsupported();
}
for ch in 0..nf {
if (0..6).filter(|&b| e.chexpstr[b][ch] != 0).count() == 1 && r.bit() {
return unsupported();
}
}
if lfeon && e.lfeexpstr.iter().filter(|&&s| s != 0).count() == 1 && r.bit() {
return unsupported();
}
}
if snroffststr == 0 {
e.frmcsnroffst = r.read(6) as i32;
e.frmfsnroffst = r.read(4) as i32;
}
if transproce {
// transient pre-noise processing (§E3.7) is not applied
for _ in 0..nf {
if r.bit() {
r.skip(10 + 8); // transprocloc, transproclen
}
}
}
if spxattene {
for ch in 0..nf {
if r.bit() {
e.spxatten[ch] = Some(r.read(5) as usize);
}
}
}
if bsi.numblkscod != 0 && r.bit() {
// blkstrtinfo: (blocks - 1) × (4 + ceil(log2(words per frame))) bits (§E2.3.2.27)
let log2 = (usize::BITS - words.saturating_sub(1).leading_zeros()) as usize;
r.skip((nblocks - 1) * (4 + log2));
}
Ok(e)
}
}
/// Spectral extension parameters (§E2.2.4, §E3.6), carried from block to block of a syncframe.
pub(crate) struct Spx {
pub inu: bool,
pub chinspx: [bool; 5],
pub begf: usize,
strtf: usize,
/// First sub-band and one past the last (spx_begin_subbnd, spx_end_subbnd).
begin: usize,
end: usize,
bndstrc: [bool; 17],
/// Band sizes in coefficients (spxbndsztab) and their count (nspxbnds).
sizes: [usize; 17],
nbands: usize,
first: [bool; 5],
co: [[f32; 17]; 5],
nblend: [[f32; 17]; 5],
sblend: [[f32; 17]; 5],
}
impl Spx {
fn new() -> Spx {
Spx {
inu: false,
chinspx: [false; 5],
begf: 0,
strtf: 0,
begin: 0,
end: 0,
bndstrc: DEFAULT_SPXBNDSTRC,
sizes: [0; 17],
nbands: 0,
first: [true; 5],
co: [[0.0; 17]; 5],
nblend: [[0.0; 17]; 5],
sblend: [[1.0; 17]; 5],
}
}
/// First coefficient of the extension region.
pub fn begin_bin(&self) -> usize {
spx_band_bin(self.begin)
}
/// Read the strategy and coordinates of block `blk`.
pub(crate) fn read(&mut self, r: &mut Bits, blk: usize, acmod: usize, nf: usize) -> Result<()> {
if blk == 0 || r.bit() {
self.inu = r.bit();
if self.inu {
if acmod == 1 {
self.chinspx = [true, false, false, false, false];
} else {
for ch in 0..nf {
self.chinspx[ch] = r.bit();
}
}
self.strtf = r.read(2) as usize;
self.begf = r.read(3) as usize;
let endf = r.read(3) as usize;
self.begin = if self.begf < 6 { self.begf + 2 } else { self.begf * 2 - 3 };
self.end = if endf < 3 { endf + 5 } else { endf * 2 + 3 };
if r.bit() {
for b in self.begin + 1..self.end {
self.bndstrc[b] = r.bit();
}
}
if self.begin >= self.end {
return Err(Error::Invalid("spectral extension ends before it begins"));
}
if self.strtf >= self.begin {
return Err(Error::Invalid("spectral extension copies from above its start"));
}
self.nbands = 1;
self.sizes = [0; 17];
self.sizes[0] = 12;
for b in self.begin + 1..self.end {
if self.bndstrc[b] {
self.sizes[self.nbands - 1] += 12;
} else {
self.sizes[self.nbands] = 12;
self.nbands += 1;
}
}
} else {
self.chinspx = [false; 5];
self.first = [true; 5];
}
}
if !self.inu {
return Ok(());
}
for ch in 0..nf {
if !self.chinspx[ch] {
self.first[ch] = true;
continue;
}
let coe = std::mem::replace(&mut self.first[ch], false) || r.bit();
if !coe {
continue;
}
let blend = r.read(5) as f32 / 32.0;
let mstr = r.read(2) as i32;
// coordinates (§E3.6.3) and blending factors (§E3.6.4.2.1)
let mut mant = self.begin_bin() as f32;
let end = spx_band_bin(self.end) as f32;
for b in 0..self.nbands {
let e = r.read(4) as i32;
let m = r.read(2) as f32;
let v = if e == 15 { m / 4.0 } else { (m + 4.0) / 8.0 };
self.co[ch][b] = v * 2f32.powi(-(e + 3 * mstr));
let size = self.sizes[b] as f32;
let nratio = ((mant + 0.5 * size) / end - blend).clamp(0.0, 1.0);
self.nblend[ch][b] = nratio.sqrt();
self.sblend[ch][b] = (1.0 - nratio).sqrt();
mant += size;
}
}
Ok(())
}
/// Synthesize the extension region of one channel's coefficients (§E3.6.4): translate the
/// low band up, notch-filter the borders when attenuation is on, blend with `noise` (zero
/// mean, unit variance) scaled to the band's energy, and scale by the coordinates.
pub(crate) fn synthesize(&self, ch: usize, atten: Option<usize>, tc: &mut [f32; 256], mut noise: impl FnMut() -> f32) {
let copystart = spx_band_bin(self.strtf);
let copyend = self.begin_bin();
let sizes = &self.sizes[..self.nbands];
// translation
let mut wrap = [false; 17];
let (mut copy, mut insert) = (copystart, copyend);
for (b, &size) in sizes.iter().enumerate() {
if copy + size > copyend {
copy = copystart;
wrap[b] = true;
}
for _ in 0..size {
if copy == copyend {
copy = copystart;
}
if let (Some(&v), true) = (tc.get(copy), insert < tc.len()) {
tc[insert] = v;
}
insert += 1;
copy += 1;
}
}
// banded RMS energy of the translated coefficients
let mut rms = [0f32; 17];
let mut at = copyend;
for (b, &size) in sizes.iter().enumerate() {
let band = tc.get(at..at + size).unwrap_or_default();
rms[b] = (band.iter().map(|v| v * v).sum::<f32>() / size as f32).sqrt();
at += size;
}
// notch filter around the baseband / extension border and each wrap point
if let Some(code) = atten {
let g = [spx_atten(code, 0), spx_atten(code, 1), spx_atten(code, 2)];
let mut notch = |center: usize| {
for (k, gain) in [g[0], g[1], g[2], g[1], g[0]].into_iter().enumerate() {
if let Some(v) = (center + k).checked_sub(2).and_then(|i| tc.get_mut(i)) {
*v *= gain;
}
}
};
notch(copyend);
let mut start = copyend;
for (b, &size) in sizes.iter().enumerate() {
if b > 0 && wrap[b] {
notch(start);
}
start += size;
}
}
// noise blending and scaling
let mut at = copyend;
for (b, &size) in sizes.iter().enumerate() {
let nscale = rms[b] * self.nblend[ch][b];
let sscale = self.sblend[ch][b];
let co = self.co[ch][b] * 32.0;
for _ in 0..size {
let n = noise();
if let Some(v) = tc.get_mut(at) {
*v = (*v * sscale + n * nscale) * co;
}
at += 1;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn attenuation_matches_table_e3_14() {
// rows 0, 4, 14 and 31 as printed
let rows: [(usize, [f64; 3]); 4] = [
(0, [0.954841604, 0.911722489, 0.870550563]),
(4, [0.793700526, 0.629960525, 0.500000000]),
(14, [0.500000000, 0.250000000, 0.125000000]),
(31, [0.227930622, 0.051952369, 0.011841536]),
];
for (code, want) in rows {
for (i, w) in want.into_iter().enumerate() {
assert!((f64::from(spx_atten(code, i)) - w).abs() < 1e-7, "{code} {i}");
}
}
}
#[test]
fn default_band_structures() {
assert_eq!(DEFAULT_CPLBNDSTRC.iter().filter(|&&b| b).count(), 8);
assert!(DEFAULT_CPLBNDSTRC[8] && !DEFAULT_CPLBNDSTRC[9] && !DEFAULT_CPLBNDSTRC[12] && DEFAULT_CPLBNDSTRC[17]);
assert_eq!(DEFAULT_SPXBNDSTRC.iter().enumerate().filter(|(_, b)| **b).map(|(i, _)| i).collect::<Vec<_>>(), vec![8, 10, 12, 14, 16]);
// every frame-based strategy sends new exponents in block 0
assert!(FRAME_EXPSTR.iter().all(|s| s[0] != 0));
}
#[test]
fn spectral_extension_translation_and_scaling() {
// begin at sub-band 2 (bin 49), end at 5 (bin 85), copy from sub-band 0 (bin 25): three
// 12-bin bands, the third wraps back to the copy start
let mut s = Spx::new();
s.inu = true;
(s.strtf, s.begin, s.end, s.nbands) = (0, 2, 5, 3);
s.sizes[..3].copy_from_slice(&[12, 12, 12]);
s.co[0][..3].copy_from_slice(&[1.0 / 32.0, 0.5 / 32.0, 0.25 / 32.0]);
s.nblend[0] = [0.0; 17];
s.sblend[0] = [1.0; 17];
let mut tc = [0f32; 256];
for (i, v) in tc[25..49].iter_mut().enumerate() {
*v = i as f32 + 1.0;
}
s.synthesize(0, None, &mut tc, || 0.0);
assert_eq!(tc[49], 1.0);
assert_eq!(tc[61], 13.0 * 0.5);
assert_eq!(tc[73], 1.0 * 0.25);
assert_eq!(tc[84], 12.0 * 0.25);
assert_eq!(tc[85], 0.0);
// pure noise: the band gets the energy of its translated coefficients
s.nblend[0] = [1.0; 17];
s.sblend[0] = [0.0; 17];
s.co[0] = [1.0 / 32.0; 17];
let mut tc2 = [0f32; 256];
tc2[25..49].fill(2.0);
let mut k = 0;
s.synthesize(0, None, &mut tc2, || {
k += 1;
if k % 2 == 0 { 1.0 } else { -1.0 }
});
assert!(tc2[49..85].iter().all(|v| v.abs() == 2.0));
}
}

401
crates/ac3/src/lib.rs
View File

@ -1,5 +1,6 @@
//! Clean-room AC-3 (Dolby Digital) audio decoder, written from ATSC A/52:2012 ("Digital Audio
//! Compression Standard", §5 bit stream syntax, §6-7 decoding).
//! Clean-room AC-3 (Dolby Digital) and E-AC-3 (Dolby Digital Plus) audio decoder, written from
//! ATSC A/52:2012 ("Digital Audio Compression Standard", §5 bit stream syntax, §6-7 decoding,
//! Annex E for E-AC-3).
//!
//! - All audio coding modes (1+1, 1/0 … 3/2) with or without LFE, 32 / 44.1 / 48 kHz, every
//! frame size (Table 5.18).
@ -11,10 +12,15 @@
//! - Output: planar f32 at full scale ±1, channels in WAV / SMPTE order (L R C LFE Ls Rs); no
//! downmix.
//!
//! E-AC-3 (bsid 11-16) is recognised and refused.
//! E-AC-3 (bsid 11-16, Annex E): independent substream 0 (dependent substreams and further
//! programs are skipped), 1 / 2 / 3 / 6 blocks per syncframe, the reduced sample rates, frame-based
//! exponent strategies and the other frame-level syntax, and spectral extension (§E3.6). The
//! adaptive hybrid transform and enhanced coupling are refused with [`Error::Unsupported`];
//! transient pre-noise processing data is read and not applied.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
mod eac3;
mod tables;
use std::sync::OnceLock;
@ -47,12 +53,35 @@ pub struct Header {
pub bsid: u8,
pub acmod: u8,
pub lfeon: bool,
/// Audio blocks (of 256 samples per channel) in the syncframe: 6 for AC-3; 1, 2, 3 or 6 for
/// E-AC-3.
pub blocks: u8,
/// E-AC-3 stream type (Table E2.1): 0 independent, 1 dependent, 2 independent converted from
/// AC-3. 0 for AC-3.
pub strmtyp: u8,
/// E-AC-3 substream id (0 for AC-3).
pub substreamid: u8,
}
impl Header {
/// Output channels (full bandwidth + LFE).
pub fn channels(&self) -> usize {
NFCHANS[self.acmod as usize] + self.lfeon as usize
NFCHANS[self.acmod as usize & 7] + self.lfeon as usize
}
/// Samples per channel the syncframe decodes to.
pub fn samples(&self) -> usize {
256 * self.blocks as usize
}
/// Whether this is an E-AC-3 syncframe (bsid 11-16).
pub fn is_eac3(&self) -> bool {
self.bsid > 10
}
/// Whether the syncframe carries the program [`Decoder::decode`] outputs: AC-3, or E-AC-3
/// independent substream 0 (§E2.3.1.2). [`Decoder::decode`] returns no channels for the
/// others: dependent substreams (the channels beyond 5.1 of a 7.1 program) and further
/// independent programs.
pub fn is_primary(&self) -> bool {
self.strmtyp != 1 && self.substreamid == 0
}
}
@ -64,8 +93,11 @@ pub fn parse_header(b: &[u8]) -> Result<Header> {
let fscod = b[4] >> 6;
let frmsizecod = b[4] & 0x3F;
let bsid = b[5] >> 3;
if bsid > 10 {
return Err(Error::Unsupported(format!("bsid {bsid} (E-AC-3)")));
if (11..=16).contains(&bsid) {
return eac3::parse_header(b, bsid);
}
if bsid > 16 {
return Err(Error::Unsupported(format!("bsid {bsid}")));
}
if fscod == 3 || frmsizecod >= 38 {
return Err(Error::Invalid("reserved fscod / frmsizecod"));
@ -91,7 +123,7 @@ pub fn parse_header(b: &[u8]) -> Result<Header> {
r.skip(2);
}
let lfeon = r.read(1) == 1;
Ok(Header { sample_rate, bitrate_kbps: kbps >> shift, frame_bytes: words as usize * 2, bsid, acmod, lfeon })
Ok(Header { sample_rate, bitrate_kbps: kbps >> shift, frame_bytes: words as usize * 2, bsid, acmod, lfeon, blocks: 6, strmtyp: 0, substreamid: 0 })
}
/// MSB-first bit reader; reads past the end give zeros.
@ -469,7 +501,9 @@ struct Frame {
chincpl: [bool; 5],
phsflginu: bool,
cplbegf: usize,
cplendf: usize,
/// cplendf + 3: one past the last coupling sub-band (with spectral extension cplendf may be
/// negative, §E3.3.1).
cplend: usize,
/// Coupling band of each coupling sub-band (relative to cplbegf).
sub_to_band: [usize; 18],
ncplbnd: usize,
@ -500,7 +534,7 @@ impl Frame {
chincpl: [false; 5],
phsflginu: false,
cplbegf: 0,
cplendf: 0,
cplend: 0,
sub_to_band: [0; 18],
ncplbnd: 0,
cplco: [[0.0; 18]; 5],
@ -523,7 +557,7 @@ impl Frame {
37 + 12 * self.cplbegf
}
fn cplendmant(&self) -> usize {
37 + 12 * (self.cplendf + 3)
37 + 12 * self.cplend
}
}
@ -622,6 +656,9 @@ pub struct Decoder {
delay: [[f32; 256]; 6],
dither: u32,
seed: u32,
/// E-AC-3: the block state of the last syncframe (acmod, lfeon, sample rate code), whose
/// exponents a syncframe of fewer than six blocks may reuse.
carry: Option<(usize, bool, usize, Box<Frame>)>,
}
impl Default for Decoder {
@ -630,7 +667,8 @@ impl Default for Decoder {
}
}
/// A decoded syncframe: 1536 samples per channel, WAV channel order.
/// A decoded syncframe: [`Header::samples`] samples per channel (1536 for AC-3), WAV channel
/// order. No channels for an E-AC-3 syncframe that is not [`Header::is_primary`].
#[derive(Clone, Debug)]
pub struct Decoded {
pub header: Header,
@ -639,7 +677,7 @@ pub struct Decoded {
impl Decoder {
pub fn new() -> Decoder {
Decoder { delay: [[0.0; 256]; 6], dither: 0, seed: 0x1234_5678 }
Decoder { delay: [[0.0; 256]; 6], dither: 0, seed: 0x1234_5678, carry: None }
}
/// Seed the dither generator (dither is decoder-specific noise; tests compare two seeds).
@ -650,6 +688,7 @@ impl Decoder {
/// Forget the overlap (before decoding from another position).
pub fn reset(&mut self) {
self.delay = [[0.0; 256]; 6];
self.carry = None;
}
fn dither(&mut self) -> f32 {
@ -658,12 +697,20 @@ impl Decoder {
((self.dither >> 8) as f32 / (1u32 << 24) as f32 * 2.0 - 1.0) * 0.707
}
/// Zero-mean, unit-variance noise for spectral extension (§E3.6.4.2.4): uniform in ±√3.
fn noise(&mut self) -> f32 {
self.dither() * (3f32.sqrt() / 0.707)
}
/// Decode one syncframe.
pub fn decode(&mut self, frame: &[u8]) -> Result<Decoded> {
let h = parse_header(frame)?;
if frame.len() < h.frame_bytes {
return Err(Error::Invalid("truncated syncframe"));
}
if h.is_eac3() {
return self.decode_eac3(&frame[..h.frame_bytes], h);
}
let fscod = (frame[4] >> 6) as usize;
// Dither is reseeded per syncframe from its CRC words (plus the configured seed), so a
// frame decodes to the same samples however it was reached (seeking, render caches).
@ -708,39 +755,83 @@ impl Decoder {
let l = r.read(6) as usize;
r.skip((l + 1) * 8);
}
let nfchans = NFCHANS[acmod];
let mut f = Frame::new(nfchans, fscod);
let mut f = Frame::new(NFCHANS[acmod], fscod);
let pcm = self.blocks(&mut r, &mut f, acmod, lfeon, 6, None)?;
Ok(Decoded { header: h, channels: wav_order(pcm, acmod, lfeon) })
}
/// Decode one E-AC-3 syncframe (`frame` is exactly the syncframe).
fn decode_eac3(&mut self, frame: &[u8], h: Header) -> Result<Decoded> {
if !h.is_primary() {
return Ok(Decoded { header: h, channels: Vec::new() });
}
// E-AC-3 has a single CRC, at the end (§E2.2.6): reseed the dither from the last words
let n = frame.len();
let tail = frame.get(n.saturating_sub(4)..).and_then(|t| <[u8; 4]>::try_from(t).ok()).unwrap_or_default();
self.dither = self.seed ^ u32::from_be_bytes(tail).wrapping_mul(0x9E37_79B9);
let mut r = Bits::new(frame);
let bsi = eac3::read_bsi(&mut r)?;
let (acmod, lfeon) = (h.acmod as usize & 7, h.lfeon);
let nblocks = h.blocks as usize;
let mut e = eac3::FrameInfo::read(&mut r, &bsi, acmod, lfeon, nblocks, n / 2)?;
let mut f = match self.carry.take() {
Some((a, l, fs, f)) if (a, l, fs) == (acmod, lfeon, bsi.fscod) => f,
_ => Box::new(Frame::new(NFCHANS[acmod], bsi.fscod)),
};
let pcm = self.blocks(&mut r, &mut f, acmod, lfeon, nblocks, Some(&mut e))?;
self.carry = Some((acmod, lfeon, bsi.fscod, f));
Ok(Decoded { header: h, channels: wav_order(pcm, acmod, lfeon) })
}
/// Decode `nblocks` audio blocks and synthesize them (inverse transform, overlap-add), in coded
/// channel order with the LFE last.
fn blocks(&mut self, r: &mut Bits, f: &mut Frame, acmod: usize, lfeon: bool, nblocks: usize, mut x: Option<&mut eac3::FrameInfo>) -> Result<Vec<Vec<f32>>> {
let nfchans = f.nfchans;
let nout = nfchans + lfeon as usize;
let mut pcm: Vec<Vec<f32>> = vec![Vec::with_capacity(1536); nout];
for blk in 0..6 {
let mut pcm: Vec<Vec<f32>> = vec![Vec::with_capacity(256 * nblocks); nout];
for blk in 0..nblocks {
let mut coefs = [[0f32; 256]; 6];
self.audio_block(&mut r, &mut f, acmod, lfeon, blk, &mut coefs)?;
self.audio_block(r, f, acmod, lfeon, blk, &mut coefs, x.as_deref_mut())?;
if r.overrun() {
return Err(Error::Invalid("audio block runs past the frame"));
}
// inverse transform, overlap-add
let mut buf = [0f32; 512];
for ch in 0..nout {
for (ch, out) in pcm.iter_mut().enumerate() {
let src = if ch < nfchans { ch } else { LFE };
let short = ch < nfchans && f.blksw[ch];
imdct(&coefs[src], short, &mut buf);
let d = &mut self.delay[src];
for n in 0..256 {
pcm[ch].push(2.0 * (buf[n] + d[n]));
out.push(2.0 * (buf[n] + d[n]));
d[n] = buf[256 + n];
}
}
}
Ok(Decoded { header: h, channels: wav_order(pcm, acmod, lfeon) })
Ok(pcm)
}
fn audio_block(&mut self, r: &mut Bits, f: &mut Frame, acmod: usize, lfeon: bool, blk: usize, coefs: &mut [[f32; 256]; 6]) -> Result<()> {
#[allow(clippy::too_many_arguments)]
fn audio_block(
&mut self,
r: &mut Bits,
f: &mut Frame,
acmod: usize,
lfeon: bool,
blk: usize,
coefs: &mut [[f32; 256]; 6],
mut x: Option<&mut eac3::FrameInfo>,
) -> Result<()> {
let nf = f.nfchans;
for ch in 0..nf {
f.blksw[ch] = r.bit();
let eac3 = x.is_some();
match x.as_deref() {
Some(e) if !e.blkswe => f.blksw = [false; 5],
_ => (0..nf).for_each(|ch| f.blksw[ch] = r.bit()),
}
for ch in 0..nf {
f.dithflag[ch] = r.bit();
match x.as_deref() {
// dither on when the flags are not sent (§E2.2.4)
Some(e) if !e.dithflage => f.dithflag = [true; 5],
_ => (0..nf).for_each(|ch| f.dithflag[ch] = r.bit()),
}
for k in 0..if acmod == 0 { 2 } else { 1 } {
if r.bit() {
@ -749,26 +840,62 @@ impl Decoder {
f.dynrng[k] = 1.0;
}
}
// E-AC-3 spectral extension strategy and coordinates
if let Some(e) = x.as_deref_mut() {
e.spx.read(r, blk, acmod, nf)?;
}
let spx = x.as_deref().map(|e| &e.spx).filter(|s| s.inu);
let spxbegf = spx.map(|s| s.begf);
let spx_begin = spx.map(|s| s.begin_bin());
let spx_chans = spx.map(|s| s.chinspx).unwrap_or([false; 5]);
// coupling strategy
if r.bit() {
f.cplinu = r.bit();
let strategy = match x.as_deref() {
Some(e) => e.cplstre[blk].then_some(e.cplinu[blk]),
None => r.bit().then(|| r.bit()),
};
if let Some(inu) = strategy {
f.cplinu = inu;
if f.cplinu {
for ch in 0..nf {
f.chincpl[ch] = r.bit();
if eac3 && r.bit() {
return Err(Error::Unsupported("E-AC-3 enhanced coupling".into()));
}
if eac3 && acmod == 2 {
f.chincpl = [true, true, false, false, false];
} else {
for ch in 0..nf {
f.chincpl[ch] = r.bit();
}
}
if acmod == 2 {
f.phsflginu = r.bit();
}
f.cplbegf = r.read(4) as usize;
f.cplendf = r.read(4) as usize;
if f.cplendf + 3 <= f.cplbegf {
f.cplend = match spxbegf {
// the coupling region ends where spectral extension begins (§E2.2.4)
Some(b) if b < 6 => b + 1,
Some(b) => 2 * b - 4,
None => r.read(4) as usize + 3,
};
if f.cplend <= f.cplbegf {
return Err(Error::Invalid("coupling end below start"));
}
let nsub = 3 + f.cplendf - f.cplbegf;
// coupling band structure, by absolute sub-band
let mut strc = [false; 18];
match x.as_deref_mut() {
Some(e) => {
if r.bit() {
for s in f.cplbegf + 1..f.cplend {
e.cplbndstrc[s] = r.bit();
}
}
strc = e.cplbndstrc;
}
None => (f.cplbegf + 1..f.cplend).for_each(|s| strc[s] = r.bit()),
}
let mut band = 0;
f.sub_to_band[0] = 0;
for s in 1..nsub {
if !r.bit() {
for s in 1..f.cplend - f.cplbegf {
if !strc[f.cplbegf + s] {
band += 1;
}
f.sub_to_band[s] = band;
@ -776,15 +903,33 @@ impl Decoder {
f.ncplbnd = band + 1;
} else {
f.chincpl = [false; 5];
if let Some(e) = x.as_deref_mut() {
e.firstcplcos = [true; 5];
e.firstcplleak = true;
f.phsflginu = false;
}
}
} else if blk == 0 {
} else if blk == 0 && !eac3 {
return Err(Error::Invalid("no coupling strategy in block 0"));
}
// coupling coordinates
if f.cplinu {
let mut any = false;
for ch in 0..nf {
if f.chincpl[ch] && r.bit() {
if !f.chincpl[ch] {
if let Some(e) = x.as_deref_mut() {
e.firstcplcos[ch] = true;
}
continue;
}
let coe = match x.as_deref_mut() {
Some(e) if e.firstcplcos[ch] => {
e.firstcplcos[ch] = false;
true
}
_ => r.bit(),
};
if coe {
any = true;
let mstr = r.read(2) as i32;
for bnd in 0..f.ncplbnd {
@ -792,7 +937,7 @@ impl Decoder {
let m = r.read(4) as f32;
let v = if e == 15 { m / 16.0 } else { (m + 16.0) / 32.0 };
let co = v * 2f32.powi(-(e + 3 * mstr));
for s in 0..(3 + f.cplendf - f.cplbegf) {
for s in 0..(f.cplend - f.cplbegf) {
if f.sub_to_band[s] == bnd {
f.cplco[ch][s] = co;
}
@ -803,7 +948,7 @@ impl Decoder {
if acmod == 2 && f.phsflginu && any {
for bnd in 0..f.ncplbnd {
let flag = r.bit();
for s in 0..(3 + f.cplendf - f.cplbegf) {
for s in 0..(f.cplend - f.cplbegf) {
if f.sub_to_band[s] == bnd {
f.phsflg[s] = flag;
}
@ -811,34 +956,55 @@ impl Decoder {
}
}
}
// rematrixing
if acmod == 2 && r.bit() {
let n = if !f.cplinu || f.cplbegf > 2 {
4
} else if f.cplbegf > 0 {
3
// rematrixing (E-AC-3 always sends the flags in block 0)
if acmod == 2 && ((eac3 && blk == 0) || r.bit()) {
let n = if f.cplinu {
match f.cplbegf {
0 => 2,
1 | 2 => 3,
_ => 4,
}
} else {
2
match spxbegf {
Some(b) if b < 2 => 3,
_ => 4,
}
};
f.rematflg = [false; 4];
for k in 0..n {
f.rematflg[k] = r.bit();
}
}
// exponent strategies
if f.cplinu {
f.expstr[CPL] = r.read(2) as u8;
}
for ch in 0..nf {
f.expstr[ch] = r.read(2) as u8;
}
if lfeon {
f.expstr[LFE] = r.read(1) as u8;
// exponent strategies (E-AC-3: from the audio frame)
match x.as_deref() {
Some(e) => {
if f.cplinu {
f.expstr[CPL] = e.cplexpstr[blk];
}
f.expstr[..nf].copy_from_slice(&e.chexpstr[blk][..nf]);
if lfeon {
f.expstr[LFE] = e.lfeexpstr[blk];
}
}
None => {
if f.cplinu {
f.expstr[CPL] = r.read(2) as u8;
}
for ch in 0..nf {
f.expstr[ch] = r.read(2) as u8;
}
if lfeon {
f.expstr[LFE] = r.read(1) as u8;
}
}
}
for ch in 0..nf {
if f.expstr[ch] != 0 {
if f.chincpl[ch] && f.cplinu {
f.endmant[ch] = f.cplstrtmant();
} else if let (true, Some(begin)) = (spx_chans[ch], spx_begin) {
// §E3.3.3: the channel ends where spectral extension begins
f.endmant[ch] = begin;
} else {
let bw = r.read(6) as usize;
if bw > 60 {
@ -870,40 +1036,105 @@ impl Decoder {
decode_exponents(r, abs, 2, 1, &mut f.exps[LFE], 0, false)?;
}
// bit allocation parametric information
if r.bit() {
f.g.sdcycod = r.read(2) as usize;
f.g.fdcycod = r.read(2) as usize;
f.g.sgaincod = r.read(2) as usize;
f.g.dbpbcod = r.read(2) as usize;
f.g.floorcod = r.read(3) as usize;
} else if blk == 0 {
return Err(Error::Invalid("no bit allocation information in block 0"));
match x.as_deref() {
Some(e) if !e.bamode => {
// §E2.2.4 defaults
(f.g.sdcycod, f.g.fdcycod, f.g.sgaincod, f.g.dbpbcod, f.g.floorcod) = (2, 1, 1, 2, 7);
}
_ => {
if r.bit() {
f.g.sdcycod = r.read(2) as usize;
f.g.fdcycod = r.read(2) as usize;
f.g.sgaincod = r.read(2) as usize;
f.g.dbpbcod = r.read(2) as usize;
f.g.floorcod = r.read(3) as usize;
} else if blk == 0 && !eac3 {
return Err(Error::Invalid("no bit allocation information in block 0"));
}
}
}
if r.bit() {
f.csnroffst = r.read(6) as i32;
if f.cplinu {
f.fsnroffst[CPL] = r.read(4) as i32;
f.fgaincod[CPL] = r.read(3) as usize;
match x.as_deref() {
Some(e) => {
// SNR offset strategies (Table E2.9), fast gain codes
let sets = [CPL, 0, 1, 2, 3, 4, LFE];
match e.snroffststr {
0 => {
f.csnroffst = e.frmcsnroffst;
sets.iter().for_each(|&c| f.fsnroffst[c] = e.frmfsnroffst);
}
s if blk == 0 || r.bit() => {
f.csnroffst = r.read(6) as i32;
if s == 1 {
let v = r.read(4) as i32;
sets.iter().for_each(|&c| f.fsnroffst[c] = v);
} else {
if f.cplinu {
f.fsnroffst[CPL] = r.read(4) as i32;
}
for ch in 0..nf {
f.fsnroffst[ch] = r.read(4) as i32;
}
if lfeon {
f.fsnroffst[LFE] = r.read(4) as i32;
}
}
}
_ => {}
}
if e.frmfgaincode && r.bit() {
if f.cplinu {
f.fgaincod[CPL] = r.read(3) as usize;
}
for ch in 0..nf {
f.fgaincod[ch] = r.read(3) as usize;
}
if lfeon {
f.fgaincod[LFE] = r.read(3) as usize;
}
} else {
f.fgaincod = [4; 7];
}
if e.strmtyp == 0 && r.bit() {
r.skip(10); // convsnroffst
}
}
for ch in 0..nf {
f.fsnroffst[ch] = r.read(4) as i32;
f.fgaincod[ch] = r.read(3) as usize;
None => {
if r.bit() {
f.csnroffst = r.read(6) as i32;
if f.cplinu {
f.fsnroffst[CPL] = r.read(4) as i32;
f.fgaincod[CPL] = r.read(3) as usize;
}
for ch in 0..nf {
f.fsnroffst[ch] = r.read(4) as i32;
f.fgaincod[ch] = r.read(3) as usize;
}
if lfeon {
f.fsnroffst[LFE] = r.read(4) as i32;
f.fgaincod[LFE] = r.read(3) as usize;
}
} else if blk == 0 {
return Err(Error::Invalid("no SNR offsets in block 0"));
}
}
if lfeon {
f.fsnroffst[LFE] = r.read(4) as i32;
f.fgaincod[LFE] = r.read(3) as usize;
}
} else if blk == 0 {
return Err(Error::Invalid("no SNR offsets in block 0"));
}
if f.cplinu && r.bit() {
f.cplleak = (((r.read(3) as i32) << 8) + 768, ((r.read(3) as i32) << 8) + 768);
if f.cplinu {
let leak = match x.as_deref_mut() {
Some(e) if e.firstcplleak => {
e.firstcplleak = false;
true
}
_ => r.bit(),
};
if leak {
f.cplleak = (((r.read(3) as i32) << 8) + 768, ((r.read(3) as i32) << 8) + 768);
}
}
// delta bit allocation
if blk == 0 {
f.deltbae = [2; 7];
}
if r.bit() {
if x.as_deref().is_none_or(|e| e.dbaflde) && r.bit() {
if f.cplinu {
f.deltbae[CPL] = r.read(2) as u8;
}
@ -921,7 +1152,7 @@ impl Decoder {
}
}
// skip field
if r.bit() {
if x.as_deref().is_none_or(|e| e.skipflde) && r.bit() {
let l = r.read(9) as usize;
r.skip(l * 8);
}
@ -1014,6 +1245,16 @@ impl Decoder {
}
}
}
// spectral extension: synthesize the high band of each channel from its low band
if let Some(e) = x.as_deref()
&& e.spx.inu
{
for ch in 0..nf {
if e.spx.chinspx[ch] {
e.spx.synthesize(ch, e.spxatten[ch], &mut coefs[ch], || self.noise());
}
}
}
// dynamic range
for ch in 0..nf {
let g = if acmod == 0 && ch == 1 { f.dynrng[1] } else { f.dynrng[0] };

175
crates/ac3/src/tests.rs
View File

@ -56,7 +56,7 @@ fn headers_and_gains() {
let b = [0x0B, 0x77, 0, 0, 30, 8 << 3, 0b1110_0001, 0b0100_0000];
let h = parse_header(&b).unwrap();
assert_eq!((h.sample_rate, h.bitrate_kbps, h.frame_bytes, h.acmod, h.lfeon, h.channels()), (48_000, 448, 1792, 7, true, 6));
assert_eq!(parse_header(&[0x0B, 0x77, 0, 0, 0, 16 << 3, 0, 0]), Err(Error::Unsupported("bsid 16 (E-AC-3)".into())));
assert_eq!(parse_header(&[0x0B, 0x77, 0, 0, 0, 17 << 3, 0, 0]), Err(Error::Unsupported("bsid 17".into())));
assert_eq!(dynrng_gain(0), 1.0);
assert!((dynrng_gain(0b1110_0000) - 0.5).abs() < 1e-7);
assert!((dynrng_gain(0b0111_1111) - 16.0 * 63.0 / 64.0).abs() < 1e-5);
@ -96,3 +96,176 @@ fn garbage_never_panics() {
}
}
}
/// MSB-first bit writer for synthetic E-AC-3 syncframes.
#[derive(Default)]
struct W {
b: Vec<u8>,
n: usize,
}
impl W {
fn put(&mut self, bits: u32, v: u32) {
for i in (0..bits).rev() {
if self.n.is_multiple_of(8) {
self.b.push(0);
}
let bit = ((v >> i) & 1) as u8;
*self.b.last_mut().unwrap() |= bit << (7 - self.n % 8);
self.n += 1;
}
}
}
/// An E-AC-3 syncframe of `words` 16-bit words: syncinfo and a bsi with no optional metadata
/// (fscod < 3 with `numblkscod`), then `audio` writes the rest; zero padded.
fn eac3_frame(strmtyp: u32, substreamid: u32, words: u32, numblkscod: u32, acmod: u32, audio: impl FnOnce(&mut W)) -> Vec<u8> {
let mut w = W::default();
w.put(16, 0x0B77);
w.put(2, strmtyp);
w.put(3, substreamid);
w.put(11, words - 1);
w.put(2, 0); // 48 kHz
w.put(2, numblkscod);
w.put(3, acmod);
w.put(1, 0); // lfeon
w.put(5, 16); // bsid
w.put(5, 31); // dialnorm
w.put(1, 0); // compre
if strmtyp == 1 {
w.put(1, 0); // chanmape
}
w.put(2, 0); // mixmdate, infomdate
if strmtyp == 0 && numblkscod != 3 {
w.put(1, 0); // convsync
}
w.put(1, 0); // addbsie
audio(&mut w);
let mut b = w.b;
b.resize(words as usize * 2, 0);
b
}
#[test]
fn eac3_headers() {
let f = eac3_frame(0, 0, 384, 3, 2, |_| {});
let h = parse_header(&f).unwrap();
assert_eq!((h.sample_rate, h.blocks, h.samples(), h.frame_bytes, h.bitrate_kbps, h.channels()), (48_000, 6, 1536, 768, 192, 2));
assert!(h.is_eac3() && h.is_primary());
// one block per syncframe: 256 samples
let h = parse_header(&eac3_frame(0, 0, 100, 0, 7, |_| {})).unwrap();
assert_eq!((h.blocks, h.samples(), h.channels(), h.bitrate_kbps), (1, 256, 5, 300));
// reduced sample rate (fscod 3, fscod2 1): six blocks at 22.05 kHz
let mut f = eac3_frame(0, 0, 384, 0, 2, |_| {});
f[4] = 0b1101_0100;
let h = parse_header(&f).unwrap();
assert_eq!((h.sample_rate, h.blocks, h.acmod), (22_050, 6, 2));
f[4] = 0b1111_0100;
assert_eq!(parse_header(&f), Err(Error::Invalid("reserved fscod2")));
// a dependent substream and a second independent program are recognised and skipped
let mut d = Decoder::new();
for (strmtyp, id) in [(1, 0), (0, 1)] {
let f = eac3_frame(strmtyp, id, 64, 3, 2, |_| {});
let out = d.decode(&f).unwrap();
assert!(!out.header.is_primary() && out.channels.is_empty());
}
assert_eq!(parse_header(&eac3_frame(3, 0, 64, 3, 2, |_| {})), Err(Error::Unsupported("reserved E-AC-3 stream type 3".into())));
}
#[test]
fn eac3_tools_we_do_not_decode_are_refused() {
// mono, six blocks, AHT enabled and used: chexpstr D15 then reuse, so chahtinu is sent
let aht = eac3_frame(0, 0, 64, 3, 1, |w| {
w.put(1, 1); // expstre
w.put(1, 1); // ahte
w.put(2, 0); // snroffststr
w.put(8, 0); // transproce … spxattene
w.put(2, 1); // chexpstr[0] = D15
w.put(10, 0); // chexpstr[1..6] = reuse
w.put(5, 0); // convexpstr
w.put(1, 1); // chahtinu
});
assert_eq!(Decoder::new().decode(&aht).unwrap_err(), Error::Unsupported("E-AC-3 adaptive hybrid transform".into()));
// stereo with coupling in block 0 using enhanced coupling
let ecpl = eac3_frame(0, 0, 64, 3, 2, |w| {
w.put(1, 1); // expstre
w.put(1, 0); // ahte
w.put(2, 0); // snroffststr
w.put(8, 0);
w.put(1, 1); // cplinu[0]
w.put(5, 0); // cplstre[1..6]
w.put(6, 0b01_01_01); // block 0: cplexpstr, chexpstr × 2
w.put(30, 0); // blocks 1-5: reuse
w.put(10, 0); // convexpstr
w.put(10, 0); // frmcsnroffst, frmfsnroffst
w.put(1, 0); // blkstrtinfoe
w.put(1, 0); // dynrnge
w.put(1, 0); // spxinu
w.put(1, 1); // ecplinu
});
assert_eq!(Decoder::new().decode(&ecpl).unwrap_err(), Error::Unsupported("E-AC-3 enhanced coupling".into()));
}
#[test]
fn eac3_single_block_frames_reuse_exponents_across_syncframes() {
// mono, one block per syncframe; the first sends exponents (all 10, bandwidth code 0), the
// second reuses them. Zero SNR offsets allocate no bits: every coefficient is dither.
let frame = |new_exps: bool| {
eac3_frame(0, 0, 32, 0, 1, |w| {
// (expstre and ahte are not sent with fewer than six blocks)
w.put(2, 0); // snroffststr
w.put(8, 0); // transproce … spxattene
w.put(2, new_exps as u32); // chexpstr[0]: D15 or reuse
w.put(1, 0); // convexpstre
w.put(10, 0); // frmcsnroffst, frmfsnroffst
w.put(1, 0); // dynrnge
w.put(1, 0); // spxinu
if new_exps {
w.put(6, 0); // chbwcod → 73 coefficients
w.put(4, 10); // absolute exponent
for _ in 0..24 {
w.put(7, 62); // three zero deltas
}
w.put(2, 0); // gainrng
}
w.put(1, 0); // convsnroffste
})
};
let (first, second) = (frame(true), frame(false));
let mut d = Decoder::new();
let a = d.decode(&first).unwrap();
assert_eq!((a.header.samples(), a.channels.len(), a.channels[0].len()), (256, 1, 256));
let b = d.decode(&second).unwrap();
let peak = |c: &[f32]| c.iter().fold(0f32, |m, v| m.max(v.abs()));
assert!(peak(&b.channels[0]) > 1e-5 && peak(&b.channels[0]) < 0.1, "{}", peak(&b.channels[0]));
// after a reset there are no exponents to reuse: no coefficients, silence
d.reset();
let c = d.decode(&second).unwrap();
assert_eq!(peak(&c.channels[0]), 0.0);
}
#[test]
fn eac3_garbage_never_panics() {
let mut d = Decoder::new();
let mut x = 0x5EEDu32;
for len in [8usize, 40, 300, 1500, 4096] {
for _ in 0..300 {
let mut f: Vec<u8> = (0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
x as u8
})
.collect();
f[0] = 0x0B;
f[1] = 0x77;
// independent substream 0, frmsiz within the buffer, bsid 11-16
let words = (len / 2).clamp(4, 2048) - 1;
f[2] = (words >> 8) as u8 & 7;
f[3] = words as u8;
f[5] = (f[5] & 7) | ((11 + x % 6) as u8) << 3;
let _ = d.decode(&f);
}
}
}

132
crates/ac3/tests/oracle.rs
View File

@ -1,5 +1,5 @@
//! AC-3 against ffmpeg: ffmpeg's AC-3 encoder writes raw syncframes from generated tones and
//! noise; ffmpeg's decode of the same file is the reference. AC-3 output is not bit-exact across
//! AC-3 and E-AC-3 against ffmpeg: ffmpeg's AC-3 and E-AC-3 encoders write raw syncframes from
//! generated tones and noise; ffmpeg's decode of the same file is the reference. AC-3 output is not bit-exact across
//! decoders (zero-bit mantissas are filled with decoder-specific dither, §7.3.4; float vs fixed
//! arithmetic), so the criterion is SNR against ffmpeg's decode.
@ -8,6 +8,8 @@ use std::process::{Command, Stdio};
struct Spec {
name: &'static str,
/// ffmpeg encoder and raw muxer: "ac3" or "eac3".
codec: &'static str,
input: &'static str,
rate: u32,
channels: u32,
@ -19,6 +21,7 @@ struct Spec {
const SPECS: &[Spec] = &[
Spec {
name: "ac3_stereo_192k",
codec: "ac3",
input: "aevalsrc=exprs=0.3*sin(2*PI*441*t)+0.1*sin(2*PI*3100*t)|0.25*sin(2*PI*660*t)+0.1*sin(2*PI*(500+4000*t)*t):s=48000",
rate: 48_000,
channels: 2,
@ -28,16 +31,26 @@ const SPECS: &[Spec] = &[
// low bitrate stereo: coupling and rematrixing
Spec {
name: "ac3_stereo_96k_coupling",
codec: "ac3",
input: "aevalsrc=exprs=0.3*sin(2*PI*441*t)+0.05*sin(2*PI*9100*t)|0.3*sin(2*PI*441*t)+0.05*sin(2*PI*11000*t):s=44100",
rate: 44_100,
channels: 2,
enc: &["-b:a", "96k", "-channel_coupling", "1"],
min_snr: 40.0,
},
Spec { name: "ac3_mono_32k", input: "sine=frequency=1000:sample_rate=32000", rate: 32_000, channels: 1, enc: &["-b:a", "96k"], min_snr: 60.0 },
Spec {
name: "ac3_mono_32k",
codec: "ac3",
input: "sine=frequency=1000:sample_rate=32000",
rate: 32_000,
channels: 1,
enc: &["-b:a", "96k"],
min_snr: 60.0,
},
// 5.1 with LFE
Spec {
name: "ac3_51_448k",
codec: "ac3",
input: "aevalsrc=exprs=0.3*sin(2*PI*300*t)|0.3*sin(2*PI*400*t)|0.3*sin(2*PI*500*t)|0.3*sin(2*PI*60*t)|0.2*sin(2*PI*700*t)|0.2*sin(2*PI*800*t):s=48000:c=5.1",
rate: 48_000,
channels: 6,
@ -47,22 +60,86 @@ const SPECS: &[Spec] = &[
// noise (wideband, many zero-bit mantissas): the SNR is limited by dither
Spec {
name: "ac3_noise_stereo",
codec: "ac3",
input: "anoisesrc=color=pink:sample_rate=48000:amplitude=0.3:seed=3,pan=stereo|c0=c0|c1=-0.5*c0",
rate: 48_000,
channels: 2,
enc: &["-b:a", "128k"],
min_snr: 18.0,
},
// E-AC-3 (Annex E): ffmpeg writes six-block syncframes with frame-based exponent strategies
Spec {
name: "eac3_stereo_192k",
codec: "eac3",
input: "aevalsrc=exprs=0.3*sin(2*PI*441*t)+0.1*sin(2*PI*3100*t)|0.25*sin(2*PI*660*t)+0.1*sin(2*PI*(500+4000*t)*t):s=48000",
rate: 48_000,
channels: 2,
enc: &["-b:a", "192k"],
min_snr: 60.0,
},
// low bitrate stereo: coupling (with the default band structure) and rematrixing
Spec {
name: "eac3_stereo_64k_coupling",
codec: "eac3",
input: "aevalsrc=exprs=0.3*sin(2*PI*441*t)+0.05*sin(2*PI*9100*t)|0.3*sin(2*PI*441*t)+0.05*sin(2*PI*11000*t):s=44100",
rate: 44_100,
channels: 2,
enc: &["-b:a", "64k", "-channel_coupling", "1"],
min_snr: 35.0,
},
Spec {
name: "eac3_mono_32k",
codec: "eac3",
input: "sine=frequency=1000:sample_rate=32000",
rate: 32_000,
channels: 1,
enc: &["-b:a", "64k"],
min_snr: 60.0,
},
// 5.1 with LFE and mixing / informational metadata in the bsi
Spec {
name: "eac3_51_384k_metadata",
codec: "eac3",
input: "aevalsrc=exprs=0.3*sin(2*PI*300*t)|0.3*sin(2*PI*400*t)|0.3*sin(2*PI*500*t)|0.3*sin(2*PI*60*t)|0.2*sin(2*PI*700*t)|0.2*sin(2*PI*800*t):s=48000:c=5.1",
rate: 48_000,
channels: 6,
enc: &[
"-b:a",
"384k",
"-dmix_mode",
"loro",
"-ltrt_cmixlev",
"0.707",
"-loro_surmixlev",
"0.5",
"-dsurex_mode",
"on",
"-mixing_level",
"100",
"-room_type",
"small",
],
min_snr: 55.0,
},
Spec {
name: "eac3_noise_stereo",
codec: "eac3",
input: "anoisesrc=color=pink:sample_rate=48000:amplitude=0.3:seed=3,pan=stereo|c0=c0|c1=-0.5*c0",
rate: 48_000,
channels: 2,
enc: &["-b:a", "96k", "-dsur_mode", "on", "-dheadphone_mode", "off"],
min_snr: 18.0,
},
];
fn make(ff: &Path, s: &Spec) -> Option<PathBuf> {
let out = filmcraft_testkit::fixtures_dir("ac3").join(format!("{}.ac3", s.name));
let out = filmcraft_testkit::fixtures_dir("ac3").join(format!("{}.{}", s.name, s.codec));
filmcraft_testkit::fixtures::generate(&out, |tmp| {
let st = Command::new(ff)
.args(["-hide_banner", "-v", "error", "-y", "-f", "lavfi", "-i", s.input, "-t", "1"])
.args(["-c:a", "ac3"])
.args(["-c:a", s.codec])
.args(s.enc)
.args(["-f", "ac3"])
.args(["-f", s.codec])
.arg(tmp)
.stdin(Stdio::null())
.status();
@ -94,6 +171,7 @@ fn ffmpeg_encoded_streams_decode() {
for fr in frames(&data) {
let d = dec.decode(fr).unwrap();
assert_eq!((d.header.sample_rate, d.channels.len() as u32), (s.rate, s.channels), "{}", s.name);
assert_eq!(d.header.is_eac3(), s.codec == "eac3", "{}", s.name);
for (o, c) in ours.iter_mut().zip(d.channels) {
o.extend(c);
}
@ -136,12 +214,52 @@ fn ffmpeg_encoded_streams_decode() {
}
}
/// Damaged E-AC-3 syncframes (bit flips, truncation, corrupt sizes) give errors or noise, never
/// a panic.
#[test]
fn damaged_eac3_frames_never_panic() {
let ff = filmcraft_testkit::require_ffmpeg!();
let mut x = 0x00C0_FFEEu32;
let mut rnd = move || {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
x
};
for s in SPECS.iter().filter(|s| s.codec == "eac3") {
let f = make(&ff, s).unwrap_or_else(|| panic!("could not generate {}", s.name));
let data = std::fs::read(&f).unwrap();
let mut dec = filmcraft_ac3::Decoder::new();
for fr in frames(&data).into_iter().take(8) {
for k in 0..300 {
let mut m = fr.to_vec();
match k % 3 {
0 => {
for _ in 0..1 + rnd() % 8 {
let bit = rnd() as usize % (m.len() * 8);
m[bit / 8] ^= 0x80 >> (bit % 8);
}
}
1 => m.truncate(rnd() as usize % m.len()),
_ => {
// keep the sync word, scramble frmsiz / fscod / numblkscod / acmod
m[2] ^= rnd() as u8;
m[3] ^= rnd() as u8;
m[4] ^= rnd() as u8;
}
}
let _ = dec.decode(&m);
}
}
}
}
#[test]
#[ignore]
fn generate_fixtures() {
let Some(ff) = filmcraft_testkit::ffmpeg_or_skip("ac3 fixtures") else { return };
for s in SPECS {
let out = filmcraft_testkit::fixtures_dir("ac3").join(format!("{}.ac3", s.name));
let out = filmcraft_testkit::fixtures_dir("ac3").join(format!("{}.{}", s.name, s.codec));
filmcraft_testkit::fixtures::generate_and_report(&format!("ac3/{}", s.name), &[out], || make(&ff, s));
}
}

30
crates/automation/src/bridge.rs
View File

@ -47,14 +47,14 @@ impl BridgeClient {
}
let mut start = self.call("engine.execute", json!({"command": id, "params": params})).await?;
let Some(job) = start.get("job").and_then(Value::as_u64) else { return Ok(start) };
let result = loop {
let (result, offline) = loop {
tokio::time::sleep(JOB_POLL).await;
let jobs = self.call("engine.execute", json!({"command": "jobs.list", "params": {}})).await?;
let Some(j) = jobs.as_array().and_then(|a| a.iter().find(|j| j["id"].as_u64() == Some(job))).cloned() else {
break Value::Null;
break (Value::Null, None);
};
if j["finished"].as_bool() == Some(true) {
break j["result"].clone();
break (j["result"].clone(), j.get("offlineMedia").cloned());
}
};
if let Some(e) = result.get("error").and_then(Value::as_str) {
@ -62,6 +62,9 @@ impl BridgeClient {
}
if let Some(o) = start.as_object_mut() {
o.insert("result".into(), result);
if let Some(off) = offline {
o.insert("offlineMedia".into(), off);
}
}
Ok(start)
}
@ -121,6 +124,11 @@ mod tests {
/// A stand-in for the app's control server: the export starts job 7, which `jobs.list` reports
/// running twice and then finished with `result`. Every request is recorded.
async fn fake_app(result: Value) -> (String, Arc<Mutex<Vec<Value>>>) {
fake_app_with(result, None).await
}
/// [`fake_app`] whose finished job also lists `offline` as its `offlineMedia`.
async fn fake_app_with(result: Value, offline: Option<Value>) -> (String, Arc<Mutex<Vec<Value>>>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap().to_string();
let seen = Arc::new(Mutex::new(Vec::new()));
@ -138,7 +146,11 @@ mod tests {
Some("jobs.list") => {
polls += 1;
let finished = polls >= 3;
json!([{"id": 7, "finished": finished, "result": if finished { result.clone() } else { Value::Null }}])
let mut j = json!({"id": 7, "finished": finished, "result": if finished { result.clone() } else { Value::Null }});
if let Some(o) = offline.clone().filter(|_| finished) {
j["offlineMedia"] = o;
}
json!([j])
}
_ => json!(null),
};
@ -165,6 +177,16 @@ mod tests {
assert_eq!(seen.iter().filter(|q| q["params"]["command"] == "jobs.list").count(), 3);
}
/// The offline media a polled export drew comes back as in a blocking export's reply (#1054).
#[tokio::test]
async fn a_polled_export_reports_its_offline_media() {
let off = json!([{"item": 8, "status": "missing"}]);
let (addr, _) = fake_app_with(json!({"frames": 3}), Some(off.clone())).await;
let b = BridgeClient::new(addr).unwrap();
let r = b.execute("file.exportMedia", json!({"path": "/tmp/out.mov", "wait": true})).await.unwrap();
assert_eq!(r["offlineMedia"], off, "{r}");
}
#[tokio::test]
async fn a_failed_blocking_export_is_an_error() {
let (addr, _) = fake_app(json!({"error": "encode: disk full"})).await;

8
crates/automation/src/long_job.rs
View File

@ -72,6 +72,7 @@ impl FilmcraftMcp {
rep.report(done, total, j["status"].as_str().unwrap_or_default()).await;
}
if finished {
let offline = state.as_ref().and_then(|j| j.get("offlineMedia").cloned());
let result = state.map(|j| j["result"].clone()).unwrap_or(Value::Null);
if let Some(e) = result.get("error").and_then(Value::as_str) {
return CallToolResult::error(vec![Content::text(format!("export failed: {e}"))]);
@ -79,6 +80,9 @@ impl FilmcraftMcp {
let mut out = start;
if let Some(o) = out.as_object_mut() {
o.insert("result".into(), result);
if let Some(off) = offline {
o.insert("offlineMedia".into(), off);
}
}
return CallToolResult::success(vec![Content::text(serde_json::to_string_pretty(&out).unwrap_or_default())]);
}
@ -108,7 +112,7 @@ impl FilmcraftMcp {
/// Delete what an interrupted export wrote, and nothing else: the output file itself, the numbered
/// stills of an image sequence (`<stem><3+ digits>.<same extension>`, only when the output is a
/// PNG/TIFF/BMP sequence) and the caption sidecar
/// PNG/TIFF/BMP/JPEG/Targa/DPX sequence) and the caption sidecar
/// (`<stem>.srt` / `<stem>.vtt`), each only if it was written since `since`. Other files that
/// merely share the name's stem (`<stem>.aep`, `<stem>2.psd`, `<stem>1.mp4` beside a movie, …) are
/// never touched, even when they were saved during the export.
@ -152,7 +156,7 @@ fn is_export_output(name: &str, file: &str, stem: &str, ext: &str) -> bool {
/// Whether an export to a file with extension `ext` writes an image sequence (one still per frame).
fn is_sequence_extension(ext: &str) -> bool {
use filmcraft_engine::export::Format;
Format::from_name(ext).is_some_and(|f| matches!(f, Format::PngSequence | Format::TiffSequence | Format::BmpSequence))
Format::from_name(ext).is_some_and(Format::is_image_sequence)
}
#[cfg(test)]

9
crates/av1/README.md
View File

@ -70,12 +70,15 @@ with libdav1d: **bit-exact** on all 6 fixtures.
wasm32, or without the default `threads` feature, everything runs on the calling thread). Headers are parsed in order; each frame is
then decoded by a frame worker once the reference frames it reads are finished, so independent
frames (hierarchical GOP layers) decode concurrently; tiles of a frame decode in parallel into
private buffers that are merged afterwards. Results never depend on scheduling (the oracle tests
compare 1 thread, many threads and libdav1d). With frame threads, pictures may be returned by a
private buffers that are merged afterwards; the in-loop filters (deblocking, CDEF, loop
restoration) each run in parallel bands of 64 rows (`postfilter.rs` explains why the bands are
independent). Results never depend on scheduling (the oracle tests compare 1 thread, many threads
and libdav1d). With frame threads, pictures may be returned by a
later `decode_pts` call or by `flush`; each carries the `pts` of the temporal unit that showed it.
`tests/perf.rs` (ignored) prints single-threaded fps (thread CPU time), per-stage ms/frame
(`Decoder::stats()`) and all-core fps for 1080p fixtures. The hot loops (sub-pixel filters,
(`Decoder::stats()`) and all-core fps with its per-stage ms/frame for 1080p fixtures (key frames,
GOPs, tiles, low delay) and a single-tile 2160p GOP; `AV1_PERF_ONLY=a,b` picks fixtures. The hot loops (sub-pixel filters,
CDEF, inverse transforms, deblocking, entropy decoding) are written for auto-vectorisation.
Measured 2026-10-01 on a 14-core Apple Silicon Mac (release build, SVT-AV1 preset 8 1080p

53
crates/av1/src/cdef.rs
View File

@ -1,35 +1,45 @@
//! CDEF (7.15).
use crate::frame::{FrameBuf, Plane};
use crate::frame::{FrameBuf, RowBand};
use crate::par::Pool;
use crate::postfilter::bands;
use crate::spec_tables::*;
use crate::state::FrameState;
/// Run CDEF on `fs.cur`; returns the CdefFrame, or None when no block is filtered (CdefFrame
/// equals CurrFrame).
pub(crate) fn apply(fs: &FrameState) -> Option<FrameBuf> {
/// equals CurrFrame). Bands of 64 luma rows (whole 8x8 blocks) are filtered in parallel: every
/// block reads only CurrFrame and writes only its own samples.
pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option<FrameBuf> {
let fh = &fs.fh;
if fh.coded_lossless || fh.allow_intrabc || !fs.seq.enable_cdef {
return None;
}
let (mi_rows, mi_cols) = (fh.mi_rows as usize, fh.mi_cols as usize);
let mut out = fs.cur.clone();
let mut any = false;
let mut r = 0;
while r < mi_rows {
let mut c = 0;
while c < mi_cols {
let idx = fs.cdef_idx(r & !15, c & !15);
if idx >= 0 && cdef_block(fs, &mut out, r, c, idx as usize) {
any = true;
let ranges = bands(pool, mi_rows * 4);
let mut planes = out.row_bands(&ranges);
let any = pool.map_mut(&mut planes, |b, band| {
let (start, end) = ranges[b];
let mut any = false;
let mut r = start / 4;
while r < mi_rows.min(end / 4) {
let mut c = 0;
while c < mi_cols {
let idx = fs.cdef_idx(r & !15, c & !15);
if idx >= 0 && cdef_block(fs, band, r, c, idx as usize) {
any = true;
}
c += 2;
}
c += 2;
r += 2;
}
r += 2;
}
any.then_some(out)
any
});
drop(planes);
any.contains(&true).then_some(out)
}
fn cdef_block(fs: &FrameState, out: &mut FrameBuf, r: usize, c: usize, idx: usize) -> bool {
fn cdef_block(fs: &FrameState, out: &mut [RowBand; 3], r: usize, c: usize, idx: usize) -> bool {
let mi = &fs.mi;
let sk = |rr: usize, cc: usize| -> bool { if rr < mi.rows && cc < mi.cols { mi.skip[mi.idx(rr, cc)] } else { true } };
let skip = sk(r, c) && sk(r + 1, c) && sk(r, c + 1) && sk(r + 1, c + 1);
@ -125,7 +135,7 @@ fn constrain(diff: i32, threshold: i32, adj: u32) -> i32 {
}
#[allow(clippy::too_many_arguments)]
fn filter(fs: &FrameState, out: &mut FrameBuf, plane: usize, r: usize, c: usize, pri: i32, sec: i32, damping: i32, dir: usize) {
fn filter(fs: &FrameState, out: &mut [RowBand; 3], plane: usize, r: usize, c: usize, pri: i32, sec: i32, damping: i32, dir: usize) {
if pri == 0 && sec == 0 {
// Every tap contributes 0 and the clamp to [min, max] keeps x: CdefFrame = CurrFrame.
return;
@ -169,7 +179,7 @@ fn filter(fs: &FrameState, out: &mut FrameBuf, plane: usize, r: usize, c: usize,
let sec_taps = [CDEF_SEC_TAPS[pri_tap][0] as i32, CDEF_SEC_TAPS[pri_tap][1] as i32];
let adj = |t: i32| if t == 0 { 0 } else { (damping - (31 - (t as u32).leading_zeros()) as i32).max(0) as u32 };
let (pri_adj, sec_adj) = (adj(pri), adj(sec));
let dst = &mut out.planes[plane];
let dst = &mut out[plane];
if all_avail {
// Every tap is available: whole rows at a time (constrain() with threshold 0 is 0).
let mut taps = [(0isize, 0i32, 0i32, 0u32); 12];
@ -230,7 +240,12 @@ fn filter(fs: &FrameState, out: &mut FrameBuf, plane: usize, r: usize, c: usize,
/// CDEF of a block whose taps are all available, `W` samples per row. Samples (at most 12
/// bits), differences and the weighted tap sum all fit in i16, which doubles the SIMD width.
#[inline(always)]
fn filter_rows<const W: usize>(buf: &[i32; PAD_STRIDE * PAD_STRIDE], taps: &[(isize, i32, i32, u32); 12], dst: &mut Plane, (y0, x0, h): (usize, usize, usize)) {
fn filter_rows<const W: usize>(
buf: &[i32; PAD_STRIDE * PAD_STRIDE],
taps: &[(isize, i32, i32, u32); 12],
dst: &mut RowBand,
(y0, x0, h): (usize, usize, usize),
) {
let mut b16 = [0i16; PAD_STRIDE * PAD_STRIDE];
for (d, &s) in b16.iter_mut().zip(buf.iter()) {
*d = s as i16;

89
crates/av1/src/frame.rs
View File

@ -106,6 +106,48 @@ impl Plane {
dst.row_from_mut(self.oy + y, self.ox)[..self.stride].copy_from_slice(s);
}
}
/// Disjoint mutable bands of whole rows, for filtering in parallel: one per `[start, end)`
/// range of rows (relative to `oy`; ascending and not overlapping, else the later band is
/// cut to what is left). Ranges are clipped to the stored rows.
pub fn row_bands(&mut self, ranges: &[(usize, usize)]) -> Vec<RowBand<'_>> {
let (stride, oy) = (self.stride.max(1), self.oy);
let mut out = Vec::with_capacity(ranges.len());
let mut rest = self.data.as_mut_slice();
// First row of `rest`.
let mut y = 0usize;
for &(start, end) in ranges {
let skip = start.saturating_sub(y).saturating_mul(stride).min(rest.len());
rest = std::mem::take(&mut rest).split_at_mut(skip).1;
y = y.saturating_add(skip / stride);
let len = end.saturating_sub(y).saturating_mul(stride).min(rest.len());
let (band, tail) = std::mem::take(&mut rest).split_at_mut(len);
out.push(RowBand { data: band, stride, y0: oy.saturating_add(y) });
y = y.saturating_add(len / stride);
rest = tail;
}
out
}
}
/// Mutable whole rows [y0, y0 + data.len() / stride) of a plane, addressed with the plane's
/// coordinates (see [`Plane::row_bands`]).
pub struct RowBand<'a> {
pub data: &'a mut [u16],
pub stride: usize,
pub y0: usize,
}
impl RowBand<'_> {
#[inline(always)]
pub fn set(&mut self, x: usize, y: usize, v: u16) {
self.data[(y - self.y0) * self.stride + x] = v;
}
#[inline(always)]
pub fn row_mut(&mut self, y: usize) -> &mut [u16] {
let o = (y - self.y0) * self.stride;
&mut self.data[o..o + self.stride]
}
}
/// A frame's three planes, sized to the macroblock-aligned coded area plus margins.
@ -158,6 +200,16 @@ impl FrameBuf {
}
}
/// [`Plane::row_bands`] of all three planes for ranges of luma rows (chroma rows scaled
/// down): element `i` holds the bands of range `i`.
pub fn row_bands(&mut self, luma: &[(usize, usize)]) -> Vec<[RowBand<'_>; 3]> {
let ssy = self.subsampling_y;
let chroma: Vec<(usize, usize)> = luma.iter().map(|&(s, e)| (s >> ssy, e >> ssy)).collect();
let [p0, p1, p2] = &mut self.planes;
let (b1, b2) = (p1.row_bands(&chroma), p2.row_bands(&chroma));
p0.row_bands(luma).into_iter().zip(b1).zip(b2).map(|((y, u), v)| [y, u, v]).collect()
}
pub fn plane_width(&self, plane: usize) -> usize {
if plane == 0 { self.width } else { (self.width + self.subsampling_x) >> self.subsampling_x }
}
@ -285,3 +337,40 @@ impl MiInfo {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rows(b: &RowBand) -> (usize, usize) {
(b.y0, b.data.len() / b.stride)
}
#[test]
fn row_bands_split_whole_rows() {
let mut p = Plane::new(8, 20);
for (i, v) in p.data.iter_mut().enumerate() {
*v = (i / 8) as u16;
}
let bands = p.row_bands(&[(0, 8), (8, 16), (16, usize::MAX)]);
assert_eq!(bands.iter().map(rows).collect::<Vec<_>>(), [(0, 8), (8, 8), (16, 4)]);
// Each band's row y holds the samples of plane row y.
for mut b in bands {
let y = b.y0 + 1;
assert!(b.row_mut(y).iter().all(|&v| v as usize == y));
}
// Gaps between bands (the loop filter's even / odd turns).
let bands = p.row_bands(&[(0, 4), (10, 14), (18, 30)]);
assert_eq!(bands.iter().map(rows).collect::<Vec<_>>(), [(0, 4), (10, 4), (18, 2)]);
}
#[test]
fn row_bands_never_overlap_or_panic() {
let mut p = Plane::new(4, 10);
// Overlapping, descending and out-of-range ranges are cut to what is left.
let bands = p.row_bands(&[(2, 6), (4, 8), (1, 3), (9, 2), (50, 60), (usize::MAX, usize::MAX)]);
assert_eq!(bands.iter().map(rows).collect::<Vec<_>>(), [(2, 4), (6, 2), (8, 0), (9, 0), (10, 0), (10, 0)]);
let mut empty = Plane::default();
assert!(empty.row_bands(&[(0, 64), (64, usize::MAX)]).iter().all(|b| b.data.is_empty()));
}
}

165
crates/av1/src/postfilter.rs
View File

@ -1,20 +1,33 @@
//! In-loop post filters run by decode_frame_wrapup: loop filter (7.14), CDEF (7.15),
//! super-resolution upscaling (7.16) and loop restoration (7.17).
use crate::frame::RowBand;
use crate::spec_tables::*;
use crate::state::FrameState;
use crate::stats::{DecodeStats, Stage, Timer};
/// The post-filters run on the frame worker; `_pool` is for row-parallel filtering (not yet used).
/// `draft`: only super-resolution (which sets the output size) runs.
pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, _pool: &crate::par::Pool, draft: bool) {
/// Luma rows per band of the row-parallel filters: whole 64x64 CDEF filter blocks, and no
/// transform block crosses a band edge.
const BAND_ROWS: usize = 64;
/// The [start, end) luma rows of the bands covering `rows`: [`BAND_ROWS`] each (the last one
/// runs to the end of the plane), or one band for everything when `pool` is sequential.
pub(crate) fn bands(pool: &crate::par::Pool, rows: usize) -> Vec<(usize, usize)> {
let n = if pool.is_parallel() { rows.div_ceil(BAND_ROWS).max(1) } else { 1 };
(0..n).map(|b| (b * BAND_ROWS, if b + 1 == n { usize::MAX } else { (b + 1) * BAND_ROWS })).collect()
}
/// The post-filters run on the frame worker, each in row bands on `pool` (the result does not
/// depend on how the bands are scheduled). `draft`: only super-resolution (which sets the
/// output size) runs.
pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate::par::Pool, draft: bool) {
let mut t = Timer::start();
let lvl = fs.fh.lf.level;
if (lvl[0] != 0 || lvl[1] != 0) && !draft {
loop_filter(fs);
loop_filter(fs, pool);
}
stats.add(Stage::LoopFilter, t.lap());
let cdef = if draft { None } else { crate::cdef::apply(fs) };
let cdef = if draft { None } else { crate::cdef::apply(fs, pool) };
stats.add(Stage::Cdef, t.lap());
let (up_cur, up_cdef) = if fs.fh.use_superres {
let c = crate::restoration::upscale(fs, &fs.cur);
@ -25,7 +38,7 @@ pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, _pool: &crate:
};
stats.add(Stage::Superres, t.lap());
fs.cur = if fs.fh.lr.uses_lr && !draft {
crate::restoration::loop_restoration(fs, &up_cur, up_cdef.as_ref().unwrap_or(&up_cur))
crate::restoration::loop_restoration(fs, &up_cur, up_cdef.as_ref().unwrap_or(&up_cur), pool)
} else {
up_cdef.unwrap_or(up_cur)
};
@ -35,28 +48,102 @@ pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, _pool: &crate:
// ---------------------------------------------------------------------------------------------
// Loop filter (7.14)
fn loop_filter(fs: &mut FrameState) {
let (mi_rows, mi_cols) = (fs.fh.mi_rows as usize, fs.fh.mi_cols as usize);
for plane in 0..fs.num_planes {
if plane == 0 || fs.fh.lf.level[1 + plane] != 0 {
for pass in 0..2 {
let row_step = if plane == 0 { 1 } else { 1 << fs.ssy };
let col_step = if plane == 0 { 1 } else { 1 << fs.ssx };
let mut row = 0;
while row < mi_rows {
let mut col = 0;
while col < mi_cols {
edge(fs, plane, pass, row, col);
col += col_step;
}
row += row_step;
/// Luma rows above a band that its horizontal edges read or write (p6 .. p0 of the widest
/// filter, rounded up to whole 4x4 rows); chroma: `LF_MARGIN >> subsampling_y`.
const LF_MARGIN: usize = 8;
/// The loop filter in row bands, all of a frame's in one parallel pass (each join costs a
/// worker wake-up, which is slow on a busy machine).
///
/// Vertical edges (pass 0) only touch the 4 rows of their own edge. A horizontal edge (pass 1)
/// reads and writes at most `filter_size / 2` samples on each side, never more than the
/// transform blocks on either side are high, so no two edges touch the same sample and
/// any order gives the spec's raster-order result. Transform blocks never cross a band edge, so
/// the only edges reaching outside a band are those on its top row, whose p side lies in the
/// band above. Each band therefore also filters the vertical edges of a copy of the
/// `LF_MARGIN` rows above it (the same work as the band above does on them), runs its top
/// horizontal edges against that copy, and afterwards the samples those edges changed are
/// written back; the band above never touches them.
fn loop_filter(fs: &mut FrameState, pool: &crate::par::Pool) {
let luma = bands(pool, fs.fh.mi_rows as usize * 4);
let planes: Vec<usize> = (0..fs.num_planes).filter(|&p| p == 0 || fs.fh.lf.level[1 + p] != 0).collect();
let mut cur = std::mem::take(&mut fs.cur);
let f: &FrameState = fs;
// Unfiltered copies of the rows above each band but the first.
let margins: Vec<[Vec<u16>; 3]> = luma
.iter()
.map(|&(start, _)| {
std::array::from_fn(|p| {
let sub_y = if p == 0 { 0 } else { f.ssy };
let pl = &cur.planes[p];
let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y);
let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride);
if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() }
})
})
.collect();
let mut bands: Vec<([RowBand; 3], [Vec<u16>; 3])> = cur.row_bands(&luma).into_iter().zip(margins).collect();
let changed = pool.map_mut(&mut bands, |b, (band, margin)| {
let mut changed: [Vec<(usize, u16)>; 3] = Default::default();
for &p in &planes {
changed[p] = filter_band(f, p, luma[b], &mut band[p], &mut margin[p]);
}
changed
});
drop(bands);
for (&(start, _), ch) in luma.iter().zip(&changed) {
for (p, list) in ch.iter().enumerate() {
let sub_y = if p == 0 { 0 } else { f.ssy };
let pl = &mut cur.planes[p];
let first = (start.saturating_sub(LF_MARGIN) >> sub_y) * pl.stride;
for &(i, v) in list {
if let Some(d) = pl.data.get_mut(first + i) {
*d = v;
}
}
}
}
fs.cur = cur;
}
fn edge(fs: &mut FrameState, plane: usize, pass: usize, row: usize, col: usize) {
/// Both loop filter passes over the band of `plane` holding luma rows [start, end); `margin` is
/// an unfiltered copy of the rows above the band (empty for the first band). Returns the samples
/// of `margin` that the band's top horizontal edges changed, as (index, value).
fn filter_band(fs: &FrameState, plane: usize, (start, end): (usize, usize), band: &mut RowBand, margin: &mut [u16]) -> Vec<(usize, u16)> {
let mi_rows = fs.fh.mi_rows as usize;
let (r0, r1) = (start / 4, mi_rows.min(end / 4));
let sub_y = if plane == 0 { 0 } else { fs.ssy };
let edges = |band: &mut RowBand, mut above: Option<&mut RowBand>, pass: usize, rows: std::ops::Range<usize>| {
let row_step = if plane == 0 { 1 } else { 1 << fs.ssy };
let col_step = if plane == 0 { 1 } else { 1 << fs.ssx };
let mi_cols = fs.fh.mi_cols as usize;
let mut row = rows.start;
while row < rows.end {
let mut col = 0;
while col < mi_cols {
edge(fs, band, above.as_deref_mut(), plane, pass, row, col);
col += col_step;
}
row += row_step;
}
};
edges(band, None, 0, r0..r1);
if margin.is_empty() {
edges(band, None, 1, r0..r1);
return Vec::new();
}
let stride = band.stride;
let y0 = start.saturating_sub(LF_MARGIN) >> sub_y;
let mut above = RowBand { data: margin, stride, y0 };
edges(&mut above, None, 0, r0.saturating_sub(LF_MARGIN / 4)..r0);
let before = above.data.to_vec();
edges(band, Some(&mut above), 1, r0..r1);
above.data.iter().zip(&before).enumerate().filter(|(_, (a, b))| a != b).map(|(i, (&a, _))| (i, a)).collect()
}
/// One loop filter edge (7.14.2) at mode info position (row, col): 4 lines of samples across
/// it. `above` holds the rows above `band` for horizontal edges on the band's top row.
fn edge(fs: &FrameState, band: &mut RowBand, above: Option<&mut RowBand>, plane: usize, pass: usize, row: usize, col: usize) {
let (sub_x, sub_y) = if plane == 0 { (0, 0) } else { (fs.ssx, fs.ssy) };
let (dx, dy) = if pass == 0 { (1i32, 0i32) } else { (0, 1) };
let x = col * 4;
@ -102,16 +189,40 @@ fn edge(fs: &mut FrameState, plane: usize, pass: usize, row: usize, col: usize)
return;
}
let t = LfThresh::new(limit, blimit, thresh, fs.bit_depth);
let pl = &mut fs.cur.planes[plane];
let stride = pl.stride;
let (step, along) = if pass == 0 { (1, stride) } else { (stride, 1) };
let base = yp * stride + xp;
let chroma = plane > 0;
let stride = band.stride;
if let Some(above) = above.filter(|_| pass == 1 && yp == band.y0) {
// The p side is in the rows above the band: filter each line in a copy that spans both.
let (n_above, n_band) = (above.data.len() / stride.max(1), band.data.len() / stride.max(1));
for k in 0..4 {
let x = xp + k;
let mut line = [0u16; 2 * LF_LINE];
for j in 1..=n_above.min(LF_LINE) {
line[LF_LINE - j] = above.data[(n_above - j) * stride + x];
}
for j in 0..n_band.min(LF_LINE) {
line[LF_LINE + j] = band.data[j * stride + x];
}
sample_filter(&mut line, LF_LINE, 1, chroma, filter_size, t);
for j in 1..=n_above.min(LF_LINE) {
above.data[(n_above - j) * stride + x] = line[LF_LINE - j];
}
for j in 0..n_band.min(LF_LINE) {
band.data[j * stride + x] = line[LF_LINE + j];
}
}
return;
}
let (step, along) = if pass == 0 { (1, stride) } else { (stride, 1) };
let base = (yp - band.y0) * stride + xp;
for k in 0..4 {
sample_filter(&mut pl.data, base + k * along, step, chroma, filter_size, t);
sample_filter(band.data, base + k * along, step, chroma, filter_size, t);
}
}
/// Samples on each side of an edge that the sample filter can read (at most p6 .. q6).
const LF_LINE: usize = 8;
/// Adaptive filter strength (7.14.4): (lvl, limit, blimit, thresh).
fn strength(fs: &FrameState, row: usize, col: usize, plane: usize, pass: usize) -> (i32, i32, i32, i32) {
let mi = &fs.mi;

94
crates/av1/src/restoration.rs
View File

@ -1,6 +1,8 @@
//! Super-resolution upscaling (7.16) and loop restoration (7.17).
use crate::frame::{FrameBuf, Plane};
use crate::frame::{FrameBuf, Plane, RowBand};
use crate::par::Pool;
use crate::postfilter::bands;
use crate::spec_tables::*;
use crate::state::FrameState;
@ -70,56 +72,66 @@ impl LrCtx<'_> {
}
/// Loop restoration (7.17): returns LrFrame given UpscaledCurrFrame and UpscaledCdefFrame.
pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf) -> FrameBuf {
/// Bands of 64 luma rows are restored in parallel: every unit reads only the two input frames
/// and writes only its own samples.
pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, pool: &Pool) -> FrameBuf {
let fh = &fs.fh;
let mut lr = cdef.clone();
let up_w = fh.upscaled_width as i32;
let fr_h = fh.frame_height as i32;
let bd = fs.bit_depth;
for plane in 0..fs.num_planes {
let rtype_frame = fh.lr.frame_restoration_type[plane];
if rtype_frame == RESTORE_NONE as u8 {
continue;
}
let (sub_x, sub_y) = if plane > 0 { (fs.ssx as i32, fs.ssy as i32) } else { (0, 0) };
let unit_size = fh.lr.loop_restoration_size[plane] as i32;
let unit_rows = fs.lr_unit_rows[plane] as i32;
let unit_cols = fs.lr_unit_cols[plane] as i32;
let plane_end_x = ((up_w + sub_x) >> sub_x) - 1;
let plane_end_y = ((fr_h + sub_y) >> sub_y) - 1;
let mut row = 0;
while row * 4 < fr_h {
let luma_y = row * 4;
let stripe_num = (luma_y + 8) / 64;
let stripe_start_y = (-8 + stripe_num * 64) >> sub_y;
let stripe_end_y = stripe_start_y + (64 >> sub_y) - 1;
let ctx = LrCtx { cur: &cur.planes[plane], cdef: &cdef.planes[plane], plane_end_x, plane_end_y, stripe_start_y, stripe_end_y };
let unit_row = (unit_rows - 1).min(((row * 4 + 8) >> sub_y) / unit_size);
let mut col = 0;
while col * 4 < up_w {
let unit_col = (unit_cols - 1).min(((col * 4) >> sub_x) / unit_size);
let x = (col * 4) >> sub_x;
let y = (row * 4) >> sub_y;
let w = (4 >> sub_x).min(plane_end_x - x + 1);
let h = (4 >> sub_y).min(plane_end_y - y + 1);
if w > 0 && h > 0 {
let ui = (unit_row * unit_cols + unit_col) as usize;
match fs.lr.lr_type[plane][ui] {
t if t == RESTORE_WIENER as u8 => wiener(fs, &ctx, &mut lr.planes[plane], plane, ui, x, y, w, h, bd),
t if t == RESTORE_SGRPROJ as u8 => self_guided(fs, &ctx, &mut lr.planes[plane], plane, ui, x, y, w, h, bd),
_ => {}
}
}
col += 1;
let ranges = bands(pool, fh.frame_height as usize);
let mut planes = lr.row_bands(&ranges);
pool.map_mut(&mut planes, |b, band| {
// Rows of 4 luma samples in this band.
let (start, end) = ranges[b];
let (row0, row1) = ((start / 4) as i32, (end / 4).min(fh.frame_height as usize) as i32);
for (plane, out) in band.iter_mut().enumerate().take(fs.num_planes) {
let rtype_frame = fh.lr.frame_restoration_type[plane];
if rtype_frame == RESTORE_NONE as u8 {
continue;
}
let (sub_x, sub_y) = if plane > 0 { (fs.ssx as i32, fs.ssy as i32) } else { (0, 0) };
let unit_size = fh.lr.loop_restoration_size[plane] as i32;
let unit_rows = fs.lr_unit_rows[plane] as i32;
let unit_cols = fs.lr_unit_cols[plane] as i32;
let plane_end_x = ((up_w + sub_x) >> sub_x) - 1;
let plane_end_y = ((fr_h + sub_y) >> sub_y) - 1;
let mut row = row0;
while row < row1 && row * 4 < fr_h {
let luma_y = row * 4;
let stripe_num = (luma_y + 8) / 64;
let stripe_start_y = (-8 + stripe_num * 64) >> sub_y;
let stripe_end_y = stripe_start_y + (64 >> sub_y) - 1;
let ctx = LrCtx { cur: &cur.planes[plane], cdef: &cdef.planes[plane], plane_end_x, plane_end_y, stripe_start_y, stripe_end_y };
let unit_row = (unit_rows - 1).min(((row * 4 + 8) >> sub_y) / unit_size);
let mut col = 0;
while col * 4 < up_w {
let unit_col = (unit_cols - 1).min(((col * 4) >> sub_x) / unit_size);
let x = (col * 4) >> sub_x;
let y = (row * 4) >> sub_y;
let w = (4 >> sub_x).min(plane_end_x - x + 1);
let h = (4 >> sub_y).min(plane_end_y - y + 1);
if w > 0 && h > 0 {
let ui = (unit_row * unit_cols + unit_col) as usize;
match fs.lr.lr_type[plane][ui] {
t if t == RESTORE_WIENER as u8 => wiener(fs, &ctx, out, plane, ui, x, y, w, h, bd),
t if t == RESTORE_SGRPROJ as u8 => self_guided(fs, &ctx, out, plane, ui, x, y, w, h, bd),
_ => {}
}
}
col += 1;
}
row += 1;
}
row += 1;
}
}
});
drop(planes);
lr
}
#[allow(clippy::too_many_arguments)]
fn wiener(fs: &FrameState, ctx: &LrCtx, out: &mut Plane, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
fn wiener(fs: &FrameState, ctx: &LrCtx, out: &mut RowBand, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
let round0: u32 = if bd == 12 { 5 } else { 3 };
let round1: u32 = if bd == 12 { 9 } else { 11 };
let coef = &fs.lr.lr_wiener[plane][ui];
@ -163,7 +175,7 @@ fn wiener(fs: &FrameState, ctx: &LrCtx, out: &mut Plane, plane: usize, ui: usize
}
#[allow(clippy::too_many_arguments)]
fn self_guided(fs: &FrameState, ctx: &LrCtx, out: &mut Plane, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
fn self_guided(fs: &FrameState, ctx: &LrCtx, out: &mut RowBand, plane: usize, ui: usize, x: i32, y: i32, w: i32, h: i32, bd: u32) {
let set = fs.lr.lr_sgr_set[plane][ui] as usize;
let mut flt0 = [[0i32; 4]; 4];
let mut flt1 = [[0i32; 4]; 4];

18
crates/av1/tests/common/mod.rs
View File

@ -16,6 +16,11 @@ pub fn ffmpeg_dav1d() -> Option<PathBuf> {
ffmpeg_with(&[("-decoders", "libdav1d")])
}
/// ffmpeg with libaom (4:2:2 and 4:4:4 fixtures, which SVT-AV1 cannot make) and libdav1d.
pub fn ffmpeg_libaom() -> Option<PathBuf> {
ffmpeg_with(&[("-encoders", "libaom-av1"), ("-decoders", "libdav1d")])
}
/// Builds such as the Windows "essentials" ones lack these libraries; skip rather than fail.
fn ffmpeg_with(needs: &[(&str, &str)]) -> Option<PathBuf> {
let ff = filmcraft_testkit::ffmpeg_or_skip("av1 oracle")?;
@ -38,9 +43,11 @@ pub fn run(ff: &Path, args: &[&str]) {
assert!(out.status.success(), "ffmpeg {:?} failed: {}", args, String::from_utf8_lossy(&out.stderr));
}
/// An AV1 fixture: a lavfi source encoded by ffmpeg's libsvtav1 into IVF.
/// An AV1 fixture: a lavfi source encoded by ffmpeg (libsvtav1 unless [`Spec::encoder`] says
/// otherwise) into IVF.
pub struct Spec {
pub name: &'static str,
pub encoder: &'static str,
pub lavfi: String,
pub frames: u32,
pub pix_fmt: &'static str,
@ -50,7 +57,12 @@ pub struct Spec {
impl Spec {
pub fn new(name: &'static str, lavfi: impl Into<String>, frames: u32, pix_fmt: &'static str, enc: &[&str]) -> Spec {
Spec { name, lavfi: lavfi.into(), frames, pix_fmt, enc: enc.iter().map(|s| s.to_string()).collect() }
Spec { name, encoder: "libsvtav1", lavfi: lavfi.into(), frames, pix_fmt, enc: enc.iter().map(|s| s.to_string()).collect() }
}
pub fn encoder(mut self, encoder: &'static str) -> Spec {
self.encoder = encoder;
self
}
}
@ -60,7 +72,7 @@ pub fn make(ff: &Path, spec: &Spec) -> PathBuf {
let tmp = filmcraft_testkit::temp_path(&path);
let frames = spec.frames.to_string();
let t = tmp.to_str().unwrap().to_string();
let mut args: Vec<&str> = vec!["-f", "lavfi", "-i", spec.lavfi.as_str(), "-frames:v", frames.as_str(), "-pix_fmt", spec.pix_fmt, "-c:v", "libsvtav1"];
let mut args: Vec<&str> = vec!["-f", "lavfi", "-i", spec.lavfi.as_str(), "-frames:v", frames.as_str(), "-pix_fmt", spec.pix_fmt, "-c:v", spec.encoder];
args.extend(spec.enc.iter().map(String::as_str));
args.extend(["-f", "ivf", t.as_str()]);
run(ff, &args);

16
crates/av1/tests/oracle_inter.rs
View File

@ -101,3 +101,19 @@ fn inter_1080p() {
check_bit_exact(&ff, &s);
}
}
/// 4:2:2 and 4:4:4 (libaom; SVT-AV1 only makes 4:2:0): the post-filters run in row bands whose
/// chroma rows depend on the vertical subsampling, so these check the band edges there too.
#[test]
fn inter_chroma_444_422() {
let Some(ff) = ffmpeg_libaom() else { return };
let p = ["-cpu-used", "8", "-crf", "40", "-enable-restoration", "1"];
let src = "testsrc2=s=416x400:r=25,noise=alls=8:allf=t";
for s in [
Spec::new("inter_aom_444", src, 6, "yuv444p", &p).encoder("libaom-av1"),
Spec::new("inter_aom_422", src, 6, "yuv422p", &p).encoder("libaom-av1"),
Spec::new("inter_aom_444_10bit", src, 6, "yuv444p10le", &p).encoder("libaom-av1"),
] {
check_bit_exact(&ff, &s);
}
}

47
crates/av1/tests/perf.rs
View File

@ -51,8 +51,8 @@ fn run(frames: &[Vec<u8>], threads: usize) -> Run {
}
fn bench(spec: &Spec) {
// AV1_PERF_ONLY=<substring> limits the run to matching fixtures.
if std::env::var("AV1_PERF_ONLY").is_ok_and(|f| !spec.name.contains(f.as_str())) {
// AV1_PERF_ONLY=<substring>[,<substring>...] limits the run to matching fixtures.
if std::env::var("AV1_PERF_ONLY").is_ok_and(|f| !f.split(',').any(|f| spec.name.contains(f))) {
return;
}
let Some(ff) = ffmpeg() else { return };
@ -68,7 +68,6 @@ fn bench(spec: &Spec) {
}
}
let b = best.unwrap();
let n = b.stats.frames.max(1) as f64;
let cpu_fps = b.cpu.map(|c| format!("{:6.1}", b.shown as f64 / c)).unwrap_or_else(|| " ?".into());
println!(
"{:<26} {:>3} shown {:>3} decoded {:>3} tiles/frame | 1 thread: {} fps (thread CPU) {:6.1} fps (wall)",
@ -79,24 +78,35 @@ fn bench(spec: &Spec) {
cpu_fps,
b.shown as f64 / b.wall
);
let total = b.stats.total_secs();
let mut line = String::from(" ms/frame:");
println!("{}", stage_line(" ms/frame:", &b.stats));
// All cores: best of 3 by wall time.
let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
let mut best: Option<Run> = None;
for _ in 0..3 {
let r = run(&frames, threads);
assert_eq!(r.shown, b.shown);
if best.as_ref().is_none_or(|b| r.wall < b.wall) {
best = Some(r);
}
}
let m = best.unwrap();
println!(" {threads} threads: {:6.1} fps (wall, load average {})", b.shown as f64 / m.wall, load_average());
// Per-stage wall time on the frame workers (stages of different frames overlap).
println!("{}", stage_line(&format!(" {threads} threads, ms/frame:"), &m.stats));
}
/// Per-stage milliseconds per frame and share of the total.
fn stage_line(label: &str, stats: &DecodeStats) -> String {
let n = stats.frames.max(1) as f64;
let total = stats.total_secs();
let mut line = String::from(label);
for s in Stage::ALL {
let v = b.stats.secs(s);
let v = stats.secs(s);
if v > 0.0005 {
line += &format!(" {} {:.2} ({:.0}%)", s.name(), 1e3 * v / n, 100.0 * v / total);
}
}
println!("{line}");
// All cores: best of 3 by wall time.
let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
let mut best_wall = f64::INFINITY;
for _ in 0..3 {
let r = run(&frames, threads);
assert_eq!(r.shown, b.shown);
best_wall = best_wall.min(r.wall);
}
println!(" {threads} threads: {:6.1} fps (wall, load average {})", b.shown as f64 / best_wall, load_average());
line
}
#[test]
@ -110,4 +120,9 @@ fn decode_throughput() {
// Higher quality (more coefficients, finer partitions) and a multi-tile stream (4x2 tiles).
bench(&Spec::new("perf_1080p_gop_hq", src, 60, "yuv420p", &p("18", "keyint=60")));
bench(&Spec::new("perf_1080p_gop_tiles", src, 60, "yuv420p", &p("30", "keyint=60:tile-columns=2:tile-rows=1")));
// Low delay (each frame references the previous one, as screen and camera recorders encode):
// frames cannot overlap, so a frame's own parallelism decides the speed.
bench(&Spec::new("perf_1080p_lowdelay", src, 60, "yuv420p", &p("30", "keyint=60:pred-struct=1")));
// One tile at 2160p: the post-filters' row bands are most of the parallelism there.
bench(&Spec::new("perf_2160p_gop", "testsrc2=s=3840x2160:r=25,noise=alls=6:allf=t", 30, "yuv420p", &p("30", "keyint=30")));
}

15
crates/avi/Cargo.toml Normal file
View File

@ -0,0 +1,15 @@
[package]
name = "filmcraft-avi"
description = "Clean-room AVI (RIFF) and OpenDML demuxer"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[dependencies]
[dev-dependencies]
filmcraft-testkit = { workspace = true }

43
crates/avi/README.md Normal file
View File

@ -0,0 +1,43 @@
# filmcraft-avi
A clean-room AVI demuxer, written from Microsoft's published AVI RIFF file reference and the
OpenDML AVI File Format Extensions 1.02. Layer L0: no dependencies beyond `std`, no `unsafe`, builds
for `wasm32-unknown-unknown`. No FFmpeg, libavformat or other AVI implementation was read; ffmpeg
and ffprobe are used only as external **test oracles**.
## API
```rust
use filmcraft_avi::{open, StreamKind};
let file = open(&bytes)?; // any ByteSource: a slice, a file, a web Blob…
for (i, s) in file.streams.iter().enumerate() {
match s.kind {
StreamKind::Video => println!("{:?} {} fps", s.video.as_ref().map(|b| b.compression), s.rate as f64 / s.scale as f64),
StreamKind::Audio => println!("format 0x{:04x}", s.audio.as_ref().map_or(0, |w| w.format_tag)),
_ => {}
}
let first = file.read_chunk(&bytes, i, 0)?; // frame data, on demand
}
```
## What it reads
| Part | |
|---|---|
| Headers | `avih`; per stream `strh`, `strf` (`BITMAPINFOHEADER`, `WAVEFORMATEX` / `WAVEFORMATEXTENSIBLE`), `strn` |
| Index | OpenDML `indx` super index → `ix##` standard indexes (field indexes too), else `idx1` (offsets relative to `movi` or absolute), else a scan of the `movi` lists |
| Large files | `RIFF AVIX` extensions: through the OpenDML index, or scanned after `idx1`, which only covers the first RIFF |
| Key frames | from the index; a scan marks every chunk and says so (`keyframes_known`) |
| Damage | sizes clamped to the file (a cut-off recording opens with what is there), `rec ` lists nested at most 3 deep, tables bounded by what the file could hold |
`filmcraft-codecs` maps the streams onto decoders (`AviSource`): Motion JPEG, H.264, HEVC and
uncompressed RGB / YUV video; PCM, MP3, MP2 and AC-3 audio.
## Tests
`cargo test -p filmcraft-avi`: hand-built files for every index path (`idx1` relative and absolute,
`rec ` lists, OpenDML standard indexes, a scan), truncation, and every single-byte mutation and
truncation of them (never a panic). `tests/ffmpeg_oracle.rs` checks ffmpeg-made files (Motion JPEG,
H.264 with B-frames, BGR24, YUY2; PCM and MP3) against ffprobe; the ignored `opendml_file_past_1_gb`
checks a 1.16 GB OpenDML file with `AVIX` extensions.

542
crates/avi/src/lib.rs Normal file
View File

@ -0,0 +1,542 @@
//! Clean-room AVI demuxer, written from Microsoft's published AVI RIFF file reference and the
//! OpenDML AVI File Format Extensions 1.02.
//!
//! [`open`] reads the `hdrl` header list (`avih`, and per stream `strh` / `strf` / `strn` /
//! `indx`) of the first `RIFF AVI ` and builds a per-stream table of [`Chunk`]s from the best index
//! the file has: the OpenDML super index (`indx` → `ix##` standard indexes, which also cover the
//! `RIFF AVIX` extensions of files past 1 GB), else the AVI 1.0 `idx1` (plus a scan of any `AVIX`
//! extension it cannot see), else a scan of the `movi` lists. Frame data is read on demand with
//! [`AviFile::read_chunk`]. Truncated files (a recording cut off) open with what is there.
//!
//! Every size and count comes from the file and is treated as hostile: reads are bounded by the
//! file length, tables by the number of chunks the file could hold, and nesting by a fixed depth.
//!
//! Layer L0: no dependencies beyond `std`, no `unsafe`, builds for `wasm32-unknown-unknown`.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
mod source;
#[cfg(test)]
mod tests;
pub use source::ByteSource;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Error {
/// Not an AVI file (no `RIFF` … `AVI ` header).
NotAvi,
/// A read failed.
Io(String),
/// Structurally invalid data.
Invalid(String),
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Error::NotAvi => f.write_str("not an AVI file"),
Error::Io(e) => write!(f, "I/O error: {e}"),
Error::Invalid(e) => write!(f, "invalid AVI: {e}"),
}
}
}
impl std::error::Error for Error {}
pub type Result<T> = std::result::Result<T, Error>;
/// Whether `head` starts like an AVI file: `RIFF` <size> `AVI `.
pub fn sniff(head: &[u8]) -> bool {
head.get(..4) == Some(b"RIFF") && head.get(8..12) == Some(b"AVI ")
}
/// `avih`: the main header (frame period, flags, frame count, size).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct MainHeader {
pub micro_sec_per_frame: u32,
pub flags: u32,
pub total_frames: u32,
pub streams: u32,
pub width: u32,
pub height: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamKind {
Video,
Audio,
/// Text, MIDI or anything else (`fccType`).
Other([u8; 4]),
}
/// `strf` of a video stream: a `BITMAPINFOHEADER`.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct BitmapInfo {
pub width: i32,
/// Positive: rows stored bottom-up (uncompressed RGB); negative: top-down.
pub height: i32,
pub bit_count: u16,
/// `biCompression` as four bytes: a FourCC (`MJPG`, `H264`, `YUY2`…) or 0 / 3 (`BI_RGB`,
/// `BI_BITFIELDS`) in little-endian.
pub compression: [u8; 4],
pub size_image: u32,
/// Bytes after the 40-byte header (codec setup, palette).
pub extra: Vec<u8>,
}
impl BitmapInfo {
/// Uncompressed RGB (`BI_RGB` / `BI_BITFIELDS`).
pub fn is_rgb(&self) -> bool {
matches!(u32::from_le_bytes(self.compression), 0 | 3)
}
}
/// `strf` of an audio stream: a `WAVEFORMATEX` (`WAVEFORMATEXTENSIBLE` is resolved to its sub-format).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct WaveFormat {
/// 1 PCM, 3 IEEE float, 0x50 MPEG audio, 0x55 MP3, 0x2000 AC-3, 0xFF AAC…
pub format_tag: u16,
pub channels: u16,
pub sample_rate: u32,
pub avg_bytes_per_sec: u32,
pub block_align: u16,
pub bits_per_sample: u16,
/// `cbSize` bytes after the 18-byte header.
pub extra: Vec<u8>,
}
/// One stored frame or audio chunk.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Chunk {
/// Offset of the data (after the 8-byte chunk header).
pub offset: u64,
pub size: u32,
/// A key frame (always true for audio and when the index does not say).
pub key: bool,
}
/// Where the chunk tables came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum IndexSource {
/// OpenDML `indx` super index and `ix##` standard indexes.
OpenDml,
/// The AVI 1.0 `idx1` (plus a scan of the `AVIX` extensions).
Idx1,
/// No usable index: the `movi` lists were scanned (key frames unknown).
Scan,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Stream {
pub kind: StreamKind,
/// `fccHandler`.
pub handler: [u8; 4],
/// Time base: `rate / scale` units per second (frames per second for video).
pub scale: u32,
pub rate: u32,
/// Start time in `scale / rate` units.
pub start: u32,
/// Length in `scale / rate` units.
pub length: u32,
/// Bytes per sample for fixed-size samples (PCM block align), 0 when samples vary.
pub sample_size: u32,
pub video: Option<BitmapInfo>,
pub audio: Option<WaveFormat>,
/// `strn`.
pub name: Option<String>,
pub chunks: Vec<Chunk>,
/// Whether [`Chunk::key`] comes from an index (a scan cannot tell).
pub keyframes_known: bool,
/// The OpenDML super index entries (offsets of the `ix##` chunks).
super_index: Vec<u64>,
}
impl Stream {
/// Stream index of a `movi` chunk id (`00dc`, `01wb`…): its first two characters.
fn number(id: &[u8; 4]) -> Option<usize> {
let d = |b: u8| (b as char).to_digit(16).map(|v| v as usize);
Some(d(id[0])? * 16 + d(id[1])?)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AviFile {
pub header: MainHeader,
pub streams: Vec<Stream>,
pub index: IndexSource,
}
/// Options for [`open_with`] (tests reach the fallback indexes with them).
#[derive(Clone, Copy, Debug, Default)]
pub struct OpenOptions {
pub ignore_odml: bool,
pub ignore_idx1: bool,
}
/// Headers larger than this are refused (a real `hdrl` is a few kilobytes).
const MAX_HEADER: u64 = 16 << 20;
/// Streams beyond this are ignored.
const MAX_STREAMS: usize = 64;
/// `LIST rec ` inside `movi` nests once; deeper is refused.
const MAX_DEPTH: u32 = 3;
fn io(e: std::io::Error) -> Error {
Error::Io(e.to_string())
}
fn read<S: ByteSource + ?Sized>(src: &S, at: u64, n: u64) -> Result<Vec<u8>> {
let end = at.checked_add(n).filter(|e| *e <= src.len()).ok_or_else(|| Error::Invalid(format!("{n} bytes at {at} run past the end")))?;
let len = usize::try_from(end - at).map_err(|_| Error::Invalid("chunk too large".into()))?;
let mut buf = vec![0; len];
src.read_at(at, &mut buf).map_err(io)?;
Ok(buf)
}
fn fourcc(b: &[u8], at: usize) -> [u8; 4] {
let mut f = [0; 4];
if let Some(s) = b.get(at..at + 4) {
f.copy_from_slice(s);
}
f
}
fn le16(b: &[u8], at: usize) -> u16 {
b.get(at..at + 2).map_or(0, |s| u16::from_le_bytes([s[0], s[1]]))
}
fn le32(b: &[u8], at: usize) -> u32 {
b.get(at..at + 4).map_or(0, |s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]))
}
fn le64(b: &[u8], at: usize) -> u64 {
u64::from(le32(b, at)) | (u64::from(le32(b, at + 4)) << 32)
}
/// A chunk header at `at`: (id, size, data offset). The size is clamped to the file (a truncated
/// recording keeps what is there).
fn header_at<S: ByteSource + ?Sized>(src: &S, at: u64) -> Result<([u8; 4], u64, u64)> {
let h = read(src, at, 8)?;
let data = at + 8;
let size = u64::from(le32(&h, 4)).min(src.len().saturating_sub(data));
Ok((fourcc(&h, 0), size, data))
}
/// Iterate the chunks of `[from, to)`: (id, size, data offset). Stops at the first unreadable
/// header; every step moves forward, so it ends.
fn children<S: ByteSource + ?Sized>(src: &S, from: u64, to: u64) -> Vec<([u8; 4], u64, u64)> {
let mut out = Vec::new();
let mut at = from;
while at.checked_add(8).is_some_and(|e| e <= to) {
let Ok((id, size, data)) = header_at(src, at) else { break };
out.push((id, size, data));
// chunks are padded to an even size
let Some(next) = data.checked_add(size).and_then(|e| e.checked_add(size & 1)) else { break };
if next <= at {
break;
}
at = next;
}
out
}
/// Open an AVI file.
pub fn open<S: ByteSource + ?Sized>(src: &S) -> Result<AviFile> {
open_with(src, OpenOptions::default())
}
pub fn open_with<S: ByteSource + ?Sized>(src: &S, opts: OpenOptions) -> Result<AviFile> {
let head = read(src, 0, 12.min(src.len())).map_err(|_| Error::NotAvi)?;
if !sniff(&head) {
return Err(Error::NotAvi);
}
// the RIFF forms: the first `AVI `, then any `AVIX` extensions (OpenDML)
let mut hdrl = None;
let mut movis: Vec<(u64, u64, u64)> = Vec::new(); // (position of the `movi` id, data start, data end)
let mut idx1 = None;
for (id, size, data) in children(src, 0, src.len()) {
if id != *b"RIFF" || size < 4 {
continue;
}
let end = data + size;
let form = fourcc(&read(src, data, 4)?, 0);
if form != *b"AVI " && form != *b"AVIX" {
continue;
}
for (cid, csize, cdata) in children(src, data + 4, end) {
match &cid {
b"LIST" if csize >= 4 => {
let ltype = fourcc(&read(src, cdata, 4)?, 0);
if ltype == *b"hdrl" && hdrl.is_none() && form == *b"AVI " {
hdrl = Some((cdata + 4, cdata + csize));
} else if ltype == *b"movi" {
movis.push((cdata, cdata + 4, cdata + csize));
}
}
b"idx1" if form == *b"AVI " => idx1 = Some((cdata, csize)),
_ => {}
}
}
}
let (hfrom, hto) = hdrl.ok_or_else(|| Error::Invalid("no hdrl header list".into()))?;
if hto - hfrom > MAX_HEADER {
return Err(Error::Invalid("header list too large".into()));
}
let (header, mut streams) = parse_hdrl(src, hfrom, hto)?;
if streams.is_empty() {
return Err(Error::Invalid("no streams".into()));
}
// chunk tables from the best index
let odml = !opts.ignore_odml && streams.iter().any(|s| !s.super_index.is_empty());
let index = if odml && read_odml(src, &mut streams) {
IndexSource::OpenDml
} else if let (false, Some((at, size)), Some(first)) = (opts.ignore_idx1, idx1, movis.first()) {
clear(&mut streams);
read_idx1(src, &mut streams, at, size, first.0);
// idx1 only covers the first RIFF: the AVIX extensions are scanned
for &(_, from, to) in movis.iter().skip(1) {
scan(src, &mut streams, from, to, 0);
}
if streams.iter().all(|s| s.chunks.is_empty()) { scan_all(src, &mut streams, &movis) } else { IndexSource::Idx1 }
} else {
scan_all(src, &mut streams, &movis)
};
Ok(AviFile { header, streams, index })
}
fn clear(streams: &mut [Stream]) {
for s in streams.iter_mut() {
s.chunks.clear();
s.keyframes_known = true;
}
}
fn scan_all<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream], movis: &[(u64, u64, u64)]) -> IndexSource {
clear(streams);
for &(_, from, to) in movis {
scan(src, streams, from, to, 0);
}
for s in streams.iter_mut() {
s.keyframes_known = false;
}
IndexSource::Scan
}
fn parse_hdrl<S: ByteSource + ?Sized>(src: &S, from: u64, to: u64) -> Result<(MainHeader, Vec<Stream>)> {
let mut header = MainHeader::default();
let mut streams = Vec::new();
for (id, size, data) in children(src, from, to) {
match &id {
b"avih" => {
let b = read(src, data, size.min(64))?;
header = MainHeader {
micro_sec_per_frame: le32(&b, 0),
flags: le32(&b, 12),
total_frames: le32(&b, 16),
streams: le32(&b, 24),
width: le32(&b, 32),
height: le32(&b, 36),
};
}
b"LIST" if size >= 4 && streams.len() < MAX_STREAMS && fourcc(&read(src, data, 4)?, 0) == *b"strl" => {
streams.push(parse_strl(src, data + 4, data + size)?);
}
_ => {}
}
}
Ok((header, streams))
}
fn parse_strl<S: ByteSource + ?Sized>(src: &S, from: u64, to: u64) -> Result<Stream> {
let mut s = Stream {
kind: StreamKind::Other([0; 4]),
handler: [0; 4],
scale: 1,
rate: 1,
start: 0,
length: 0,
sample_size: 0,
video: None,
audio: None,
name: None,
chunks: Vec::new(),
keyframes_known: true,
super_index: Vec::new(),
};
let mut strf = None;
for (id, size, data) in children(src, from, to) {
match &id {
b"strh" => {
let b = read(src, data, size.min(64))?;
s.kind = match &fourcc(&b, 0) {
b"vids" => StreamKind::Video,
b"auds" => StreamKind::Audio,
other => StreamKind::Other(*other),
};
s.handler = fourcc(&b, 4);
s.scale = le32(&b, 20);
s.rate = le32(&b, 24);
s.start = le32(&b, 28);
s.length = le32(&b, 32);
s.sample_size = le32(&b, 44);
}
b"strf" => strf = Some(read(src, data, size.min(1 << 20))?),
b"strn" => {
let b = read(src, data, size.min(4096))?;
let name = String::from_utf8_lossy(b.split(|&c| c == 0).next().unwrap_or_default()).trim().to_string();
s.name = (!name.is_empty()).then_some(name);
}
b"indx" => s.super_index = super_index(&read(src, data, size.min(MAX_HEADER))?),
_ => {}
}
}
if let Some(b) = strf {
match s.kind {
StreamKind::Video => {
s.video = Some(BitmapInfo {
width: le32(&b, 4) as i32,
height: le32(&b, 8) as i32,
bit_count: le16(&b, 14),
compression: fourcc(&b, 16),
size_image: le32(&b, 20),
extra: b.get(40.min(b.len())..).unwrap_or_default().to_vec(),
});
}
StreamKind::Audio => {
let cb = usize::from(le16(&b, 16));
let extra = b.get(18.min(b.len())..(18 + cb).min(b.len())).unwrap_or_default().to_vec();
let mut tag = le16(&b, 0);
// WAVE_FORMAT_EXTENSIBLE: the sub-format GUID starts with the real tag
if tag == 0xFFFE && extra.len() >= 22 {
tag = le16(&extra, 6);
}
s.audio = Some(WaveFormat {
format_tag: tag,
channels: le16(&b, 2),
sample_rate: le32(&b, 4),
avg_bytes_per_sec: le32(&b, 8),
block_align: le16(&b, 12),
bits_per_sample: le16(&b, 14),
extra,
});
}
StreamKind::Other(_) => {}
}
}
if s.scale == 0 || s.rate == 0 {
(s.scale, s.rate) = (1, 1);
}
Ok(s)
}
/// The `ix##` chunk offsets of an `indx` super index (`AVI_INDEX_OF_INDEXES`, 4 longs per entry).
fn super_index(b: &[u8]) -> Vec<u64> {
let longs = usize::from(le16(b, 0));
let (subtype, itype, n) = (b.get(2).copied().unwrap_or(0), b.get(3).copied().unwrap_or(0xFF), le32(b, 4) as usize);
if itype != 0 || subtype != 0 || longs != 4 {
return Vec::new();
}
// header 24 bytes, entries of 16: qwOffset, dwSize, dwDuration
let n = n.min(b.len().saturating_sub(24) / 16);
(0..n).map(|k| le64(b, 24 + k * 16)).filter(|&o| o > 0).collect()
}
/// Fill the chunk tables from the OpenDML standard indexes; false (tables cleared) when a stream
/// that has data lacks one or none could be read.
fn read_odml<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream]) -> bool {
let limit = src.len() / 8;
let mut total = 0u64;
for s in streams.iter_mut() {
s.chunks.clear();
s.keyframes_known = true;
for &at in &s.super_index {
let Ok((id, size, data)) = header_at(src, at) else { continue };
if &id[..2] != b"ix" {
continue;
}
let Ok(b) = read(src, data, size.min(MAX_HEADER * 8)) else { continue };
let (longs, subtype, itype) = (usize::from(le16(&b, 0)), b.get(2).copied().unwrap_or(0), b.get(3).copied().unwrap_or(0));
// AVI_INDEX_OF_CHUNKS; 2 longs per entry, or 3 for the field index (AVI_INDEX_2FIELD)
if itype != 1 || !(longs == 2 && subtype == 0 || longs == 3 && subtype == 1) {
continue;
}
let base = le64(&b, 12);
let step = longs * 4;
let n = (le32(&b, 4) as usize).min(b.len().saturating_sub(24) / step);
for k in 0..n {
let e = 24 + k * step;
let (off, raw) = (u64::from(le32(&b, e)), le32(&b, e + 4));
let size = raw & 0x7FFF_FFFF;
let Some(offset) = base.checked_add(off).filter(|o| o.checked_add(u64::from(size)).is_some_and(|end| end <= src.len())) else { continue };
total += 1;
if total > limit {
return false;
}
s.chunks.push(Chunk { offset, size, key: raw & 0x8000_0000 == 0 });
}
}
}
let ok = streams.iter().any(|s| !s.chunks.is_empty()) && streams.iter().all(|s| !s.super_index.is_empty() || matches!(s.kind, StreamKind::Other(_)));
if !ok {
clear(streams);
}
ok
}
/// Fill the chunk tables from `idx1` (16-byte entries: id, flags, offset, size). Offsets are
/// relative to the `movi` list's type id, or (some writers) absolute: the first entry decides.
fn read_idx1<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream], at: u64, size: u64, movi: u64) {
let Ok(b) = read(src, at, size.min(src.len() / 2)) else { return };
let entries: Vec<&[u8]> = b.as_chunks::<16>().0.iter().map(|e| e.as_slice()).collect();
let id_at = |pos: u64| read(src, pos, 4).ok().map(|v| fourcc(&v, 0));
let first = entries.iter().find(|e| le32(e, 4) & 1 == 0);
let relative = match first {
Some(e) => {
let (id, off) = (fourcc(e, 0), u64::from(le32(e, 8)));
id_at(movi + off) == Some(id) || id_at(off) != Some(id)
}
None => true,
};
let base = if relative { movi } else { 0 };
for e in entries {
let (id, flags, off, len) = (fourcc(e, 0), le32(e, 4), u64::from(le32(e, 8)), le32(e, 12));
// AVIIF_LIST entries (`rec ` lists) and palette changes carry no samples
if flags & 1 != 0 || &id[2..] == b"pc" {
continue;
}
let Some(s) = Stream::number(&id).and_then(|n| streams.get_mut(n)) else { continue };
let Some(offset) = base.checked_add(off).and_then(|o| o.checked_add(8)).filter(|o| o.checked_add(u64::from(len)).is_some_and(|end| end <= src.len()))
else {
continue;
};
s.chunks.push(Chunk { offset, size: len, key: flags & 0x10 != 0 || s.kind != StreamKind::Video });
}
}
/// Append the data chunks of a `movi` list (`[from, to)`) to the streams' tables, in file order.
fn scan<S: ByteSource + ?Sized>(src: &S, streams: &mut [Stream], from: u64, to: u64, depth: u32) {
if depth >= MAX_DEPTH {
return;
}
for (id, size, data) in children(src, from, to) {
if id == *b"LIST" {
if size >= 4 {
scan(src, streams, data + 4, data + size, depth + 1);
}
continue;
}
if &id[..2] == b"ix" || &id[2..] == b"pc" {
continue;
}
if let Some(s) = Stream::number(&id).and_then(|n| streams.get_mut(n)) {
s.chunks.push(Chunk { offset: data, size: u32::try_from(size).unwrap_or(u32::MAX), key: true });
}
}
}
impl AviFile {
/// The bytes of chunk `i` of stream `stream`.
pub fn read_chunk<S: ByteSource + ?Sized>(&self, src: &S, stream: usize, i: usize) -> Result<Vec<u8>> {
let c = self.streams.get(stream).and_then(|s| s.chunks.get(i)).ok_or_else(|| Error::Invalid(format!("no chunk {i} in stream {stream}")))?;
read(src, c.offset, u64::from(c.size))
}
}

59
crates/avi/src/source.rs Normal file
View File

@ -0,0 +1,59 @@
//! Random-access byte sources for the demuxer (same shape as the other container crates', so one
//! adapter serves them all).
use std::io;
/// Random-access, read-only byte source (file, in-memory buffer, web Blob…).
pub trait ByteSource {
/// Total length in bytes.
fn len(&self) -> u64;
/// Fill `buf` entirely with bytes starting at `offset`; fail with `UnexpectedEof` if past the end.
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>;
/// True if the source holds no bytes.
fn is_empty(&self) -> bool {
self.len() == 0
}
}
fn eof() -> io::Error {
io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
}
impl ByteSource for [u8] {
fn len(&self) -> u64 {
<[u8]>::len(self) as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
let start = usize::try_from(offset).map_err(|_| eof())?;
let end = start.checked_add(buf.len()).ok_or_else(eof)?;
buf.copy_from_slice(self.get(start..end).ok_or_else(eof)?);
Ok(())
}
}
impl ByteSource for Vec<u8> {
fn len(&self) -> u64 {
self.as_slice().len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.as_slice().read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for &T {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}
impl<T: ByteSource + ?Sized> ByteSource for std::sync::Arc<T> {
fn len(&self) -> u64 {
(**self).len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
(**self).read_at(offset, buf)
}
}

269
crates/avi/src/tests.rs Normal file
View File

@ -0,0 +1,269 @@
use super::*;
fn chunk(id: &[u8; 4], data: &[u8]) -> Vec<u8> {
let mut v = id.to_vec();
v.extend_from_slice(&(data.len() as u32).to_le_bytes());
v.extend_from_slice(data);
if data.len() % 2 == 1 {
v.push(0);
}
v
}
fn list(ltype: &[u8; 4], body: &[u8]) -> Vec<u8> {
let mut d = ltype.to_vec();
d.extend_from_slice(body);
chunk(b"LIST", &d)
}
fn strl(video: bool) -> Vec<u8> {
let mut strh = vec![0u8; 56];
strh[..4].copy_from_slice(if video { b"vids" } else { b"auds" });
strh[4..8].copy_from_slice(if video { b"MJPG" } else { &[0; 4] });
let (scale, rate, ss) = if video { (1u32, 25u32, 0u32) } else { (4, 192_000, 4) };
strh[20..24].copy_from_slice(&scale.to_le_bytes());
strh[24..28].copy_from_slice(&rate.to_le_bytes());
strh[44..48].copy_from_slice(&ss.to_le_bytes());
let strf = if video {
let mut b = vec![0u8; 40];
b[..4].copy_from_slice(&40u32.to_le_bytes());
b[4..8].copy_from_slice(&64i32.to_le_bytes());
b[8..12].copy_from_slice(&48i32.to_le_bytes());
b[14..16].copy_from_slice(&24u16.to_le_bytes());
b[16..20].copy_from_slice(b"MJPG");
b
} else {
let mut b = vec![0u8; 18];
b[..2].copy_from_slice(&1u16.to_le_bytes());
b[2..4].copy_from_slice(&2u16.to_le_bytes());
b[4..8].copy_from_slice(&48_000u32.to_le_bytes());
b[8..12].copy_from_slice(&192_000u32.to_le_bytes());
b[12..14].copy_from_slice(&4u16.to_le_bytes());
b[14..16].copy_from_slice(&16u16.to_le_bytes());
b
};
let mut body = chunk(b"strh", &strh);
body.extend(chunk(b"strf", &strf));
body.extend(chunk(b"strn", if video { b"Camera\0" } else { b"Mic\0" }));
list(b"strl", &body)
}
/// A small AVI: 3 video frames (the second a delta, the third odd-sized) and 2 audio chunks,
/// optionally inside a `rec ` list, with an `idx1` whose offsets are relative to `movi` or absolute.
fn avi(idx1: bool, absolute: bool, rec: bool) -> Vec<u8> {
let mut avih = vec![0u8; 56];
avih[0..4].copy_from_slice(&40_000u32.to_le_bytes());
avih[12..16].copy_from_slice(&0x10u32.to_le_bytes());
avih[16..20].copy_from_slice(&3u32.to_le_bytes());
avih[24..28].copy_from_slice(&2u32.to_le_bytes());
avih[32..36].copy_from_slice(&64u32.to_le_bytes());
avih[36..40].copy_from_slice(&48u32.to_le_bytes());
let mut hdrl = chunk(b"avih", &avih);
hdrl.extend(strl(true));
hdrl.extend(strl(false));
let hdrl = list(b"hdrl", &hdrl);
let samples: [(&[u8; 4], Vec<u8>, bool); 5] =
[(b"00dc", vec![1; 10], true), (b"01wb", vec![2; 16], true), (b"00dc", vec![3; 6], false), (b"01wb", vec![4; 8], true), (b"00dc", vec![5; 7], true)];
let mut body = Vec::new();
let mut entries = Vec::new();
for (id, data, key) in &samples {
entries.push((**id, body.len() as u32 + 4, data.len() as u32, *key));
body.extend(chunk(id, data));
}
let movi_body = if rec {
let r = list(b"rec ", &body);
// the samples moved 12 bytes in (the rec list's header and type)
for e in &mut entries {
e.1 += 12;
}
r
} else {
body
};
let movi = list(b"movi", &movi_body);
let mut riff = b"AVI ".to_vec();
riff.extend(hdrl);
// position of the `movi` type id: after `RIFF` <size>, what `riff` holds so far, `LIST` <size>
let movi_at = (8 + riff.len() + 8) as u32;
riff.extend(movi);
if idx1 {
let mut ix = Vec::new();
for (id, off, len, key) in entries {
ix.extend_from_slice(&id);
ix.extend_from_slice(&(if key { 0x10u32 } else { 0 }).to_le_bytes());
ix.extend_from_slice(&(if absolute { off + movi_at } else { off }).to_le_bytes());
ix.extend_from_slice(&len.to_le_bytes());
}
riff.extend(chunk(b"idx1", &ix));
}
chunk(b"RIFF", &riff)
}
fn table(f: &AviFile, s: usize) -> Vec<(u32, bool)> {
f.streams[s].chunks.iter().map(|c| (c.size, c.key)).collect()
}
#[test]
fn headers_and_every_index_path_agree() {
for (idx1, absolute, rec, want) in [
(true, false, false, IndexSource::Idx1),
(true, true, false, IndexSource::Idx1),
(true, false, true, IndexSource::Idx1),
(false, false, false, IndexSource::Scan),
(false, false, true, IndexSource::Scan),
] {
let file = avi(idx1, absolute, rec);
let f = open(&file).unwrap();
let what = format!("idx1 {idx1} absolute {absolute} rec {rec}");
assert_eq!(f.index, want, "{what}");
assert_eq!((f.header.width, f.header.height, f.header.total_frames, f.header.micro_sec_per_frame), (64, 48, 3, 40_000));
let (v, a) = (&f.streams[0], &f.streams[1]);
assert_eq!((v.kind, v.handler, v.scale, v.rate, v.name.as_deref()), (StreamKind::Video, *b"MJPG", 1, 25, Some("Camera")));
assert_eq!(v.video.as_ref().map(|b| (b.width, b.height, b.bit_count, b.compression)), Some((64, 48, 24, *b"MJPG")));
let w = a.audio.as_ref().unwrap();
assert_eq!((w.format_tag, w.channels, w.sample_rate, w.block_align, w.bits_per_sample, a.sample_size), (1, 2, 48_000, 4, 16, 4));
// the index knows the delta frame; a scan cannot
let keys = if idx1 { [true, false, true] } else { [true; 3] };
assert_eq!(table(&f, 0), vec![(10, keys[0]), (6, keys[1]), (7, keys[2])], "{what}");
assert_eq!(f.streams[0].keyframes_known, idx1);
assert_eq!(table(&f, 1), vec![(16, true), (8, true)], "{what}");
assert_eq!(f.read_chunk(&file, 0, 1).unwrap(), vec![3; 6], "{what}");
assert_eq!(f.read_chunk(&file, 0, 2).unwrap(), vec![5; 7], "{what}");
assert_eq!(f.read_chunk(&file, 1, 1).unwrap(), vec![4; 8], "{what}");
assert!(f.read_chunk(&file, 0, 3).is_err() && f.read_chunk(&file, 5, 0).is_err());
}
// the idx1 path can be skipped
let f = open_with(&avi(true, false, false), OpenOptions { ignore_idx1: true, ..Default::default() }).unwrap();
assert_eq!(f.index, IndexSource::Scan);
}
#[test]
fn not_avi_and_truncated_recordings() {
assert_eq!(open(&b"RIFF\0\0\0\0WAVEfmt ".to_vec()), Err(Error::NotAvi));
assert_eq!(open(&Vec::new()), Err(Error::NotAvi));
assert!(sniff(&avi(true, false, false)) && !sniff(b"RIFF"));
// a recording cut off in its last frame: the headers and the whole frames are there
let full = avi(false, false, false);
let cut = full[..full.len() - 4].to_vec();
let f = open(&cut).unwrap();
assert_eq!(f.streams[0].chunks.len(), 3);
assert!(f.read_chunk(&cut, 0, 0).is_ok());
}
/// Every single-byte change and every truncation of a valid file opens or fails, never panics, and
/// whatever opens reads its chunks without panicking.
#[test]
fn mutations_never_panic() {
for base in [avi(true, false, true), avi(false, false, false), avi(true, true, false)] {
for cut in 0..base.len() {
let b = base[..cut].to_vec();
if let Ok(f) = open(&b) {
for (s, st) in f.streams.iter().enumerate() {
for i in 0..st.chunks.len() {
let _ = f.read_chunk(&b, s, i);
}
}
}
}
for at in 0..base.len() {
for v in [0x00u8, 0xFF, 0x7F, 0x80] {
let mut b = base.clone();
b[at] = v;
if let Ok(f) = open(&b) {
for (s, st) in f.streams.iter().enumerate() {
for i in 0..st.chunks.len() {
let _ = f.read_chunk(&b, s, i);
}
}
}
}
}
}
}
/// An OpenDML file: each stream's `indx` super index points at an `ix##` standard index (base
/// offset + offsets to the data, bit 31 of the size marking delta frames). Built twice: the second
/// pass knows where the `ix##` chunks landed.
fn avi_odml(ix_at: [u64; 2]) -> (Vec<u8>, [u64; 2]) {
let indx = |stream: u8, at: u64| {
let mut b = Vec::new();
b.extend_from_slice(&4u16.to_le_bytes()); // wLongsPerEntry
b.extend_from_slice(&[0, 0]); // subtype, AVI_INDEX_OF_INDEXES
b.extend_from_slice(&1u32.to_le_bytes());
b.extend_from_slice(&[b'0', b'0' + stream, if stream == 0 { b'd' } else { b'w' }, if stream == 0 { b'c' } else { b'b' }]);
b.extend_from_slice(&[0; 12]);
b.extend_from_slice(&at.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
chunk(b"indx", &b)
};
let mut hdrl = chunk(b"avih", &[0; 56]);
for (s, at) in ix_at.iter().enumerate() {
let l = strl(s == 0);
// strl = LIST <size> strl …: append indx inside it
let mut body = l[12..].to_vec();
body.extend(indx(s as u8, *at));
hdrl.extend(list(b"strl", &body));
}
let mut riff = b"AVI ".to_vec();
riff.extend(list(b"hdrl", &hdrl));
// movi: video 10 (key) 6 (delta) 7 (key), audio 16 8; then ix00, ix01
let movi_data_at = 8 + riff.len() as u64 + 12; // RIFF hdr, so far, LIST hdr + `movi`
let mut body = Vec::new();
let mut at: [Vec<(u64, u32, bool)>; 2] = [Vec::new(), Vec::new()];
for (s, data, key) in [(0usize, vec![1u8; 10], true), (1, vec![2; 16], true), (0, vec![3; 6], false), (1, vec![4; 8], true), (0, vec![5; 7], true)] {
let id = if s == 0 { b"00dc" } else { b"01wb" };
at[s].push((movi_data_at + body.len() as u64 + 8, data.len() as u32, key));
body.extend(chunk(id, &data));
}
let mut where_ix = [0u64; 2];
for (s, entries) in at.iter().enumerate() {
let base = 100u64; // offsets are relative to qwBaseOffset
let mut b = Vec::new();
b.extend_from_slice(&2u16.to_le_bytes());
b.extend_from_slice(&[0, 1]); // subtype 0, AVI_INDEX_OF_CHUNKS
b.extend_from_slice(&(entries.len() as u32).to_le_bytes());
b.extend_from_slice(if s == 0 { b"00dc" } else { b"01wb" });
b.extend_from_slice(&base.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
for &(off, size, key) in entries {
b.extend_from_slice(&((off - base) as u32).to_le_bytes());
b.extend_from_slice(&(size | if key { 0 } else { 0x8000_0000 }).to_le_bytes());
}
where_ix[s] = movi_data_at + body.len() as u64;
body.extend(chunk(if s == 0 { b"ix00" } else { b"ix01" }, &b));
}
riff.extend(list(b"movi", &body));
(chunk(b"RIFF", &riff), where_ix)
}
#[test]
fn opendml_standard_indexes() {
let (_, at) = avi_odml([0, 0]);
let (file, again) = avi_odml(at);
assert_eq!(at, again, "the layout does not depend on the offsets");
let f = open(&file).unwrap();
assert_eq!(f.index, IndexSource::OpenDml);
assert_eq!(table(&f, 0), vec![(10, true), (6, false), (7, true)]);
assert_eq!(table(&f, 1), vec![(16, true), (8, true)]);
assert_eq!(f.read_chunk(&file, 0, 2).unwrap(), vec![5; 7]);
// ignoring the OpenDML index, the scan finds the same chunks (and skips the ix## chunks)
let s = open_with(&file, OpenOptions { ignore_odml: true, ..Default::default() }).unwrap();
assert_eq!(s.index, IndexSource::Scan);
assert_eq!(s.streams[0].chunks.iter().map(|c| c.offset).collect::<Vec<_>>(), f.streams[0].chunks.iter().map(|c| c.offset).collect::<Vec<_>>());
// a broken super index (pointing nowhere) falls back to a scan
let (bad, _) = avi_odml([3, 5]);
assert_eq!(open(&bad).unwrap().index, IndexSource::Scan);
// and mutations of the OpenDML file never panic
for i in 0..file.len() {
let mut b = file.clone();
b[i] ^= 0xFF;
if let Ok(f) = open(&b) {
for (s, st) in f.streams.iter().enumerate() {
for k in 0..st.chunks.len() {
let _ = f.read_chunk(&b, s, k);
}
}
}
}
}

151
crates/avi/tests/ffmpeg_oracle.rs Normal file
View File

@ -0,0 +1,151 @@
//! ffmpeg-made AVI files (skipped without ffmpeg): the chunk tables match ffprobe's packet counts,
//! and every index the file has (`idx1`; OpenDML once a file passes 1 GB) and a `movi` scan describe
//! the same chunks.
use std::path::{Path, PathBuf};
use std::process::Command;
use filmcraft_avi::{IndexSource, OpenOptions, StreamKind, open, open_with};
fn fixture(name: &str, args: &[&str]) -> Option<PathBuf> {
let ffmpeg = filmcraft_testkit::ffmpeg_or_skip("avi oracle")?;
let out = filmcraft_testkit::fixtures_dir("avi").join(name);
filmcraft_testkit::fixtures::generate(&out, |tmp: &Path| {
let mut c = Command::new(&ffmpeg);
c.args(["-v", "error", "-y"]).args(args).args(["-f", "avi"]).arg(tmp);
c.status().is_ok_and(|s| s.success())
})
}
fn packets(path: &Path, stream: usize) -> usize {
let ffprobe = filmcraft_testkit::ffprobe().expect("ffprobe next to ffmpeg");
let out = Command::new(ffprobe)
.args(["-v", "error", "-select_streams", &stream.to_string(), "-count_packets", "-show_entries", "stream=nb_read_packets", "-of", "csv=p=0"])
.arg(path)
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).trim().parse().unwrap_or(0)
}
fn packet_bytes(path: &Path, stream: usize) -> u64 {
let ffprobe = filmcraft_testkit::ffprobe().expect("ffprobe next to ffmpeg");
let out = Command::new(ffprobe)
.args(["-v", "error", "-select_streams", &stream.to_string(), "-show_entries", "packet=size", "-of", "csv=p=0"])
.arg(path)
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).lines().filter_map(|l| l.trim().parse::<u64>().ok()).sum()
}
const TONE: [&str; 4] = ["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=2"];
fn cases() -> Vec<(&'static str, Vec<&'static str>, [u8; 4])> {
let pic = vec!["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2"];
let with = |v: &[&'static str], a: &[&'static str]| -> Vec<&'static str> {
let mut x = pic.clone();
x.extend_from_slice(&TONE);
x.extend_from_slice(v);
x.extend_from_slice(a);
x
};
vec![
("mjpeg_pcm.avi", with(&["-c:v", "mjpeg", "-q:v", "4", "-pix_fmt", "yuvj420p"], &["-c:a", "pcm_s16le"]), *b"MJPG"),
("h264_mp3.avi", with(&["-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p"], &["-c:a", "libmp3lame", "-b:a", "128k"]), *b"H264"),
("bgr24_pcm.avi", with(&["-c:v", "rawvideo", "-pix_fmt", "bgr24"], &["-c:a", "pcm_s16le"]), [0; 4]),
("yuyv_pcm.avi", with(&["-c:v", "rawvideo", "-pix_fmt", "yuyv422"], &["-c:a", "pcm_s16le"]), *b"YUY2"),
]
}
#[test]
fn ffmpeg_files_match_ffprobe_on_every_index_path() {
for (name, args, fourcc) in cases() {
let Some(path) = fixture(name, &args) else { return };
let bytes = std::fs::read(&path).unwrap();
let f = open(&bytes).unwrap();
// ffmpeg reserves room for the OpenDML index but writes only idx1 under 1 GB
assert_eq!(f.index, IndexSource::Idx1, "{name}");
assert_eq!(f.streams.len(), 2, "{name}");
let (v, a) = (&f.streams[0], &f.streams[1]);
assert_eq!((v.kind, a.kind), (StreamKind::Video, StreamKind::Audio), "{name}");
let bmp = v.video.as_ref().unwrap();
assert_eq!((bmp.width, bmp.height.abs(), bmp.compression), (160, 120, fourcc), "{name}");
assert_eq!((v.rate as f64 / v.scale as f64).round(), 25.0, "{name}");
assert_eq!(a.audio.as_ref().map(|w| (w.sample_rate, w.channels)), Some((48_000, 1)), "{name}");
// the tables hold what ffprobe reads
assert_eq!(v.chunks.len(), packets(&path, 0), "{name}: video");
// PCM chunks are packets; ffprobe re-splits MP3 into its frames, which AVI chunks need not
// follow, so compressed audio compares by bytes
if a.audio.as_ref().is_some_and(|w| w.format_tag == 1) {
assert_eq!(a.chunks.len(), packets(&path, 1), "{name}: audio");
} else {
let ours: u64 = a.chunks.iter().map(|c| u64::from(c.size)).sum();
assert_eq!(ours, packet_bytes(&path, 1), "{name}: audio bytes");
}
assert!(v.chunks[0].key, "{name}: the first frame is a key frame");
if fourcc == *b"H264" {
assert!(v.chunks.iter().any(|c| !c.key), "{name}: the index marks the delta frames");
}
// a scan finds the same chunks (it cannot know key frames)
let scanned = open_with(&bytes, OpenOptions { ignore_odml: true, ignore_idx1: true }).unwrap();
assert_eq!(scanned.index, IndexSource::Scan, "{name}");
for s in 0..2 {
assert_eq!(sizes(&scanned, s), sizes(&f, s), "{name}: scan stream {s}");
}
}
}
fn sizes(f: &filmcraft_avi::AviFile, s: usize) -> Vec<(u64, u32)> {
f.streams[s].chunks.iter().map(|c| (c.offset, c.size)).collect()
}
/// A file past 1 GB, as long recordings are: ffmpeg writes `RIFF AVIX` extensions and the
/// OpenDML index. Needs ffmpeg and 1.2 GB of disk: `cargo test -p filmcraft-avi --test
/// ffmpeg_oracle -- --ignored`.
#[test]
#[ignore]
fn opendml_file_past_1_gb() {
let args = [
"-f",
"lavfi",
"-i",
"testsrc2=s=640x480:r=25:d=50",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=48000:d=50",
"-c:v",
"rawvideo",
"-pix_fmt",
"bgr24",
"-c:a",
"pcm_s16le",
];
let Some(path) = fixture("large_opendml.avi", &args) else { return };
let file = std::fs::File::open(&path).unwrap();
let src = FileSource(file);
let f = open(&src).unwrap();
assert_eq!(f.index, IndexSource::OpenDml);
assert_eq!(f.streams[0].chunks.len(), packets(&path, 0));
assert_eq!(f.streams[1].chunks.len(), packets(&path, 1));
assert_eq!(f.streams[0].chunks.len(), 1250);
assert!(f.streams[0].chunks.last().unwrap().offset > 1 << 30, "the last frames are in an AVIX extension");
// idx1 covers the first RIFF only; the AVIX extensions are scanned: the same chunks
let idx1 = open_with(&src, OpenOptions { ignore_odml: true, ..Default::default() }).unwrap();
assert_eq!(idx1.index, IndexSource::Idx1);
for s in 0..2 {
assert_eq!(sizes(&idx1, s), sizes(&f, s), "stream {s}");
}
assert_eq!(f.read_chunk(&src, 0, 1249).unwrap().len(), 640 * 480 * 3);
}
/// A file on disk as a byte source, read where asked.
struct FileSource(std::fs::File);
impl filmcraft_avi::ByteSource for FileSource {
fn len(&self) -> u64 {
self.0.metadata().map(|m| m.len()).unwrap_or(0)
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> {
std::os::unix::fs::FileExt::read_exact_at(&self.0, buf, offset)
}
}

2
crates/captions/README.md
View File

@ -16,7 +16,7 @@ and writes caption files, converts them to and from caption tracks, and draws ca
| Scenarist SCC `.scc` | yes | yes | CEA-608 at 29.97 fps, drop-frame or not. Reads pop-on, roll-up and paint-on (channel 1); writes pop-on. |
| MacCaption MCC `.mcc` | yes | yes | One SMPTE ST 334-2 caption distribution packet (CDP) per frame in MacCaption's hex text with letter abbreviations. Writes 29.97 fps (`30DF` / `30`) with CEA-608 field 1 (the SCC schedule) **and** CEA-708 service 1 (two-window pop-on emulation). Reads CEA-608 field 1 when present, else CEA-708 service 1; time code rates 30DF, 30, 60DF, 60, 25, 24, 50. |
| EBU STL `.stl` | yes | yes | EBU Tech 3264-E: GSI block + 128-byte TTI blocks, `STL25.01` (25 fps) or `STL30.01` (read/written as 29.97 fps, non-drop labels), ISO/IEC 6937 Latin text (accents as diacritic + letter), italics / underline codes as `<i>` / `<u>`, extension blocks for long subtitles, start-of-programme (TCP) subtracted. |
| TTML `.ttml` (IMSC1) | yes | yes | Writes the IMSC1 Text profile (TTML2 namespaces): `<p>` per cue, `<br/>`, styled `<span>`s for `<i>` / `<b>` / `<u>`, `ttm:agent` speakers, exact frame times (`123f`) at the sequence rate. Reads any TTML (TTML1, TTML2, the 2006 `ttaf1` drafts): clock / frame / offset / tick times, `dur`, nested time containers, timed spans, referenced styles. |
| TTML `.ttml` (IMSC1) | yes | yes | Writes the IMSC1 Text profile (TTML2 namespaces): `<p>` per cue, `<br/>`, styled `<span>`s for `<i>` / `<b>` / `<u>`, `ttm:agent` speakers, exact frame times (`123f`) at the sequence rate. Reads any TTML (TTML1, TTML2, the 2006 `ttaf1` drafts): clock / frame / offset / tick times, `dur`, nested time containers (`body` / `div` begins offset and ends clip their contents; `seq` containers are read as `par` with a warning), timed spans, referenced styles. |
| DFXP `.dfxp` | yes | yes | TTML1 with the DFXP presentation profile; clock times in milliseconds (frame-exact after `snap_to_frames`). |
All cue times are exact `Tick`s. Milliseconds convert exactly (1 ms = 254 016 000 ticks); SCC

89
crates/captions/src/ttml.rs
View File

@ -10,8 +10,8 @@
//!
//! **Reading** is namespace-agnostic (TTML1 2010, the 2006 `ttaf1` drafts and TTML2 all read):
//! clock times (`HH:MM:SS`, `.fraction`, `:FF` frames and `.subframes`), offset times (`h`, `m`,
//! `s`, `ms`, `f`, `t` with `ttp:tickRate`), `begin` / `end` / `dur`, nested time containers
//! (`body` / `div` begins add up), timed `<span>`s inside an untimed `<p>`, `<br/>`, whitespace
//! `s`, `ms`, `f`, `t` with `ttp:tickRate`), `begin` / `end` / `dur`, nested `par` time
//! containers (`body` / `div` begins add up and their ends clip what is inside), timed `<span>`s inside an untimed `<p>`, `<br/>`, whitespace
//! collapsing, styles referenced by id (italic / bold / underline) and `ttm:agent` / `xml:id`.
use filmcraft_time::{FrameRate, TICKS_PER_SECOND, Tick};
@ -423,44 +423,84 @@ pub fn parse(text: &str) -> Result<Document> {
let Some(body) = root.children().find(|n| n.is_element() && n.tag_name().name() == "body") else {
return Ok(doc);
};
walk(body, Tick::ZERO, &tm, &styles, Fmt::default(), &mut doc);
if root.descendants().any(|n| attr(n, "timeContainer").is_some_and(|v| v.trim() == "seq")) {
doc.warnings.push("timeContainer=\"seq\" is not supported; its children are timed as in a par container".into());
}
// the body is a time container like a div: its begin offsets, and its end clips, everything inside
let t = times(body, Scope { begin: Tick::ZERO, end: None }, tm, &mut doc);
walk(body, Scope { begin: t.begin.unwrap_or(Tick::ZERO), end: t.end }, &tm, &styles, Fmt::default(), &mut doc);
doc.cues.sort_by_key(|c| (c.start, c.end));
Ok(doc)
}
/// Begin and end of a timed element relative to `parent` (None when it has no timing).
fn times(n: roxmltree::Node, parent: Tick, tm: &Timing, doc: &mut Document) -> (Option<Tick>, Option<Tick>) {
let p = |k: &str, doc: &mut Document| -> Option<Tick> {
/// The active interval of the enclosing time containers (`par`): child times are offsets from
/// `begin`, and nothing inside outlasts `end` (None when no ancestor sets an end).
#[derive(Clone, Copy)]
struct Scope {
begin: Tick,
end: Option<Tick>,
}
/// Whether `n` has timing attributes of its own.
fn has_timing(n: roxmltree::Node) -> bool {
["begin", "end", "dur"].into_iter().any(|k| attr(n, k).is_some())
}
/// Timing of an element in its parent container.
struct Times {
/// The begin, when the element sets one.
begin: Option<Tick>,
/// Its own end (`end`, or `begin` + `dur`, the earlier of the two) clipped to the parent's
/// end, else the parent's end.
end: Option<Tick>,
/// A timing attribute did not parse (and was warned about).
bad: bool,
}
fn times(n: roxmltree::Node, parent: Scope, tm: Timing, doc: &mut Document) -> Times {
let mut bad = false;
let mut p = |k: &str, doc: &mut Document| -> Option<Tick> {
let v = attr(n, k)?;
let t = parse_time(v, *tm);
let t = parse_time(v, tm);
if t.is_none() {
bad = true;
doc.warnings.push(format!("bad time expression \"{v}\""));
}
t
};
let begin = p("begin", doc).map(|b| parent + b);
let end = match (p("end", doc), p("dur", doc)) {
(Some(e), _) => Some(parent + e),
(None, Some(d)) => Some(begin.unwrap_or(parent) + d),
_ => None,
let begin = p("begin", doc).map(|b| parent.begin + b);
let own_end = match (p("end", doc).map(|e| parent.begin + e), p("dur", doc).map(|d| begin.unwrap_or(parent.begin) + d)) {
(Some(e), Some(d)) => Some(e.min(d)),
(e, d) => e.or(d),
};
(begin, end)
let end = match (own_end, parent.end) {
(Some(e), Some(pe)) => Some(e.min(pe)),
(e, pe) => e.or(pe),
};
Times { begin, end, bad }
}
fn walk(n: roxmltree::Node, parent: Tick, tm: &Timing, styles: &std::collections::HashMap<String, Fmt>, fmt: Fmt, doc: &mut Document) {
fn walk(n: roxmltree::Node, parent: Scope, tm: &Timing, styles: &std::collections::HashMap<String, Fmt>, fmt: Fmt, doc: &mut Document) {
for c in n.children().filter(|c| c.is_element()) {
let f = style_of(c, styles, fmt);
match c.tag_name().name() {
"div" => {
let (b, _) = times(c, parent, tm, doc);
walk(c, b.unwrap_or(parent), tm, styles, f, doc);
let t = times(c, parent, *tm, doc);
walk(c, Scope { begin: t.begin.unwrap_or(parent.begin), end: t.end }, tm, styles, f, doc);
}
"p" => {
let (b, e) = times(c, parent, tm, doc);
let t = times(c, parent, *tm, doc);
if t.bad {
continue;
}
let (b, e) = (t.begin.unwrap_or(parent.begin), t.end);
let speaker = attr(c, "agent").map(|a| a.trim().trim_start_matches('_').replace('_', " "));
let id = c.attributes().find(|a| a.name() == "id").map(|a| a.value().to_string());
match (b, e) {
(Some(b), Some(e)) => {
let spans: Vec<roxmltree::Node> = c.children().filter(|x| x.is_element() && x.tag_name().name() == "span" && has_timing(*x)).collect();
// a timed paragraph, or one without timed spans in a container with an end, is one cue
let whole = if has_timing(c) || spans.is_empty() { e } else { None };
match whole {
Some(e) => {
let mut o = Out { s: String::new(), pending: false };
let tags: Vec<&str> = [(f.italic, "i"), (f.bold, "b"), (f.underline, "u")].into_iter().filter(|x| x.0).map(|x| x.1).collect();
for t in &tags {
@ -475,12 +515,13 @@ fn walk(n: roxmltree::Node, parent: Tick, tm: &Timing, styles: &std::collections
doc.cues.push(Cue { start: b, end: e, text, speaker, id, ..Default::default() });
}
}
_ => {
None => {
// untimed paragraph: its timed spans are the cues
let base = b.unwrap_or(parent);
for sp in c.children().filter(|x| x.is_element() && x.tag_name().name() == "span") {
let (sb, se) = times(sp, base, tm, doc);
if let (Some(sb), Some(se)) = (sb, se) {
let scope = Scope { begin: b, end: e };
for sp in spans {
let st = times(sp, scope, *tm, doc);
let sb = st.begin.unwrap_or(scope.begin);
if let (false, Some(se)) = (st.bad, st.end) {
let mut o = Out { s: String::new(), pending: false };
collect(sp, styles, style_of(sp, styles, f), &mut o);
let text = clean(&o.s);

92
crates/captions/tests/ttml_body_timing.rs Normal file
View File

@ -0,0 +1,92 @@
//! TTML / DFXP inherited timing: `begin`, `end` and `dur` on `body` and `div` time containers
//! offset and clip the paragraphs inside them (#895).
use filmcraft_captions::{Format, WriteOptions, parse, write};
/// (start, end, text) of every cue, in seconds; panics on a parse error.
fn cues(xml: &str, format: Format) -> Vec<(f64, f64, String)> {
let doc = parse(xml.as_bytes(), format).unwrap();
assert!(doc.warnings.is_empty(), "{:?}", doc.warnings);
doc.cues.iter().map(|c| (c.start.seconds(), c.end.seconds(), c.text.clone())).collect()
}
fn both(xml: &str, want: &[(f64, f64, &str)]) {
let want: Vec<(f64, f64, String)> = want.iter().map(|(s, e, t)| (*s, *e, t.to_string())).collect();
for format in [Format::Ttml, Format::Dfxp] {
assert_eq!(cues(xml, format), want, "{format:?}: {xml}");
}
}
#[test]
fn body_begin_is_inherited() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div><p begin="1s" end="2s">x</p></div></body></tt>"#, &[(6.0, 7.0, "x")]);
}
#[test]
fn div_begin_is_inherited() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div begin="5s"><p begin="1s" end="2s">x</p></div></body></tt>"#, &[(6.0, 7.0, "x")]);
}
#[test]
fn untimed_containers_leave_times_unchanged() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div><p begin="1s" end="2s">x</p></div></body></tt>"#, &[(1.0, 2.0, "x")]);
}
#[test]
fn nested_begins_add_up() {
both(
r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div begin="00:00:10.000"><div begin="1s"><p begin="1s" end="2s">deep</p></div><p begin="0s" dur="1s">mid</p></div><p begin="1s" end="2s">top</p></body></tt>"#,
&[(6.0, 7.0, "top"), (15.0, 16.0, "mid"), (17.0, 18.0, "deep")],
);
}
#[test]
fn body_begin_offsets_timed_spans() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div><p><span begin="1s" end="2s">x</span></p></div></body></tt>"#, &[(6.0, 7.0, "x")]);
}
#[test]
fn container_ends_clip_descendants() {
// div active 11 s to 13 s: a later paragraph is dropped, a longer one is cut at 13 s
both(
r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="1s"><div begin="10s" end="12s"><p begin="1s" end="5s">cut</p><p begin="3s" end="4s">gone</p></div></body></tt>"#,
&[(12.0, 13.0, "cut")],
);
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body dur="3s"><div><p begin="1s" end="5s">x</p></div></body></tt>"#, &[(1.0, 3.0, "x")]);
}
#[test]
fn paragraphs_without_an_end_last_until_their_container_ends() {
both(
r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div begin="2s" end="6s"><p begin="1s">open</p></div><div begin="10s" dur="2s"><p>whole</p></div></body></tt>"#,
&[(3.0, 6.0, "open"), (10.0, 12.0, "whole")],
);
}
#[test]
fn end_and_dur_take_the_earlier() {
both(r#"<tt xmlns="http://www.w3.org/ns/ttml"><body><div><p begin="1s" end="5s" dur="2s">x</p></div></body></tt>"#, &[(1.0, 3.0, "x")]);
}
#[test]
fn seq_containers_warn() {
let xml = r#"<tt xmlns="http://www.w3.org/ns/ttml" xmlns:ttp="http://www.w3.org/ns/ttml#parameter"><body><div timeContainer="seq"><p begin="1s" end="2s">x</p></div></body></tt>"#;
for format in [Format::Ttml, Format::Dfxp] {
let doc = parse(xml.as_bytes(), format).unwrap();
assert_eq!(doc.cues.len(), 1);
assert!(doc.warnings.iter().any(|w| w.contains("seq")), "{:?}", doc.warnings);
}
}
#[test]
fn inherited_times_survive_export() {
// the writer uses untimed body / div and absolute paragraph times, so the resolved times stay
let xml = r#"<tt xmlns="http://www.w3.org/ns/ttml"><body begin="5s"><div begin="1s" end="4s"><p begin="1s" end="9s">x</p></div></body></tt>"#;
for format in [Format::Ttml, Format::Dfxp] {
let doc = parse(xml.as_bytes(), format).unwrap();
let back = parse(&write(&doc, format, WriteOptions::default()), format).unwrap();
let times = |d: &filmcraft_captions::Document| d.cues.iter().map(|c| (c.start, c.end, c.text.clone())).collect::<Vec<_>>();
assert_eq!(times(&back), times(&doc), "{format:?}");
assert_eq!((back.cues[0].start.seconds(), back.cues[0].end.seconds()), (7.0, 9.0), "{format:?}");
}
}

4
crates/codecs/Cargo.toml
View File

@ -14,16 +14,19 @@ filmcraft-time = { workspace = true }
filmcraft-color = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-media = { workspace = true }
filmcraft-avi = { workspace = true }
filmcraft-isobmff = { workspace = true }
filmcraft-matroska = { workspace = true }
filmcraft-mxf = { workspace = true }
filmcraft-mpegts = { workspace = true }
filmcraft-mpeg2v = { workspace = true }
filmcraft-ac3 = { workspace = true }
filmcraft-mpa = { workspace = true }
filmcraft-ogg = { workspace = true }
filmcraft-bitstream = { workspace = true }
filmcraft-h264 = { workspace = true }
filmcraft-hevc = { workspace = true }
filmcraft-vp8 = { workspace = true }
filmcraft-vp9 = { workspace = true }
filmcraft-prores = { workspace = true }
filmcraft-dnx = { workspace = true }
@ -31,6 +34,7 @@ filmcraft-av1 = { workspace = true }
filmcraft-apv = { workspace = true }
filmcraft-aac = { workspace = true }
filmcraft-opus = { workspace = true }
rayon = { workspace = true }
thiserror = { workspace = true }
log = { workspace = true }
# Decode timing counters (`gop_stats`); std's Instant panics on wasm32.

276
crates/codecs/src/audio.rs
View File

@ -20,13 +20,18 @@ enum Inner {
Aac { dec: Box<filmcraft_aac::Decoder>, asc: Vec<u8>, up: Option<Box<Upsample2x>> },
/// AAC whose configuration arrives with the first frame (ADTS / LATM).
AacPending,
/// MPEG audio at this rate whose layer arrives with the first frame (MP4 / QuickTime name the
/// codec, not the layer; layers I / II go to our decoder, layer III to symphonia's).
MpegPending(u32),
/// Our own Opus decoder (always 48 kHz output; pre-skip is left to container timestamps).
/// `order` maps output channel → decoded channel (Vorbis → WAV/SMPTE order for surround).
Opus { dec: Box<filmcraft_opus::Decoder>, order: Option<&'static [usize]> },
/// Bootstrap decoders (MP3, ALAC, FLAC, HE-AAC…) via symphonia.
Symphonia(Box<dyn Decoder>),
/// Our AC-3 decoder (ATSC A/52).
/// Our AC-3 and E-AC-3 decoder (ATSC A/52, Annex E).
Ac3(Box<filmcraft_ac3::Decoder>),
/// Our MPEG-1/2 audio Layer I and II decoder (ISO/IEC 11172-3, 13818-3).
Mpa(Box<filmcraft_mpa::Decoder>),
}
/// Opus always decodes at 48 kHz (RFC 7845 §5.1: the input rate in the header is informational).
@ -56,13 +61,16 @@ fn vorbis_to_wav_order(channels: usize) -> Option<&'static [usize]> {
/// Channel order for AAC surround (ISO/IEC 14496-3 Table 1.19, our decoder's element order: C, L R,
/// surrounds, LFE): output channel → decoded channel, giving the WAV/SMPTE order (L R C LFE Ls Rs)
/// used elsewhere, as the AC-3 decoder does and the AAC encoder's caller undoes on export. Only the
/// unambiguous channel configurations 3–6; 1, 2, 7 and PCE layouts keep the decoded order.
/// unambiguous channel configurations 3–6, 11 (6.1: … LFE, back centre, sides) and 12 (7.1: … LFE,
/// backs, sides); 1, 2, 7 and PCE layouts keep the decoded order.
fn aac_to_wav_order(channel_config: u8) -> Option<&'static [usize]> {
Some(match channel_config {
3 => &[1, 2, 0],
4 => &[1, 2, 0, 3],
5 => &[1, 2, 0, 3, 4],
6 => &[1, 2, 0, 5, 3, 4],
11 => &[1, 2, 0, 6, 5, 3, 4],
12 => &[1, 2, 0, 7, 5, 6, 3, 4],
_ => return None,
})
}
@ -124,15 +132,22 @@ impl PacketDecoder {
let order = if dec.head().mapping_family == 1 { vorbis_to_wav_order(channels) } else { None };
Ok(Self { inner: Inner::Opus { dec: Box::new(dec), order }, channels })
}
/// MPEG-1/2 audio layer I, II or III (symphonia).
/// MPEG-1/2 audio layer I, II (our decoder: symphonia rejects the bit rate / mode
/// combinations ISO/IEC 11172-3 does not allow, such as ffmpeg's default 384 kbit/s mono MP2)
/// or III (symphonia).
pub fn mpeg_audio(layer: u8, sample_rate: u32) -> Result<Self> {
let codec = match layer {
1 => symphonia::core::codecs::CODEC_TYPE_MP1,
2 => symphonia::core::codecs::CODEC_TYPE_MP2,
1 | 2 => return Ok(Self { inner: Inner::Mpa(Box::default()), channels: 0 }),
_ => CODEC_TYPE_MP3,
};
Self::new(codec, sample_rate, None)
}
/// MPEG-1/2 audio of a layer the frame headers name (see [`mpeg_audio_layer`]): MP4's
/// `esds` object types 0x69 / 0x6B and QuickTime's `.mp3` / `.mp2` cover layers I–III alike
/// (ffmpeg stores MP2 as `mp4a` + 0x6B).
pub fn lazy_mpeg_audio(sample_rate: u32) -> Self {
Self { inner: Inner::MpegPending(sample_rate), channels: 0 }
}
/// AAC configured by [`Self::ensure_aac`] from in-band headers (ADTS, LATM).
pub fn lazy_aac() -> Self {
Self { inner: Inner::AacPending, channels: 0 }
@ -148,7 +163,7 @@ impl PacketDecoder {
self.inner = Inner::Aac { dec: Box::new(dec), asc: asc.to_vec(), up: None };
Ok(())
}
/// AC-3 (ATSC A/52).
/// AC-3 and E-AC-3 (ATSC A/52, Annex E).
pub fn ac3() -> Result<Self> {
Ok(Self { inner: Inner::Ac3(Box::new(filmcraft_ac3::Decoder::new())), channels: 0 })
}
@ -160,16 +175,21 @@ impl PacketDecoder {
use filmcraft_isobmff::CodecConfig as C;
match c {
C::Aac(a) => Self::aac(&a.asc, if rate > 0 { rate } else { a.sample_rate }),
C::Mp3 => Self::new(CODEC_TYPE_MP3, rate, None),
C::Ac3 { .. } => Self::ac3(),
C::Mp3 => Ok(Self::lazy_mpeg_audio(rate)),
C::Ac3 { .. } | C::Eac3 { .. } => Self::ac3(),
C::Alac { cookie } => Self::new(CODEC_TYPE_ALAC, rate, Some(cookie.clone())),
C::Flac(_) => Self::new(CODEC_TYPE_FLAC, rate, None),
// the decoder needs the STREAMINFO body: `dfLa` holds it after the 4-byte block header
C::Flac(f) => Self::new(CODEC_TYPE_FLAC, rate, f.metadata_blocks.get(4..38).map(<[u8]>::to_vec)),
C::Opus(o) => Self::opus(filmcraft_opus::OpusHead::from_dops(&o.to_bytes()).map_err(|e| CodecError::Unsupported(format!("Opus: {e}")))?),
other => Err(CodecError::Unsupported(format!("{} audio", other.name()))),
}
}
/// Decode one packet into planar channels.
pub fn decode(&mut self, data: &[u8], ts: u64) -> Result<Vec<Vec<f32>>> {
if let Inner::MpegPending(rate) = self.inner {
let layer = mpeg_audio_layer(data).ok_or_else(|| CodecError::Decode("MPEG audio: no frame header".into()))?;
self.inner = Self::mpeg_audio(layer, rate)?.inner;
}
let dec = match &mut self.inner {
Inner::Aac { dec, up, .. } => {
let mut out = dec.decode(data).map_err(|e| CodecError::Decode(e.to_string()))?;
@ -192,13 +212,18 @@ impl PacketDecoder {
}
Inner::Symphonia(d) => d,
Inner::AacPending => return Err(CodecError::Decode("AAC: no configuration yet".into())),
Inner::MpegPending(_) => return Err(CodecError::Decode("MPEG audio: no frame header".into())),
Inner::Ac3(d) => {
// a packet may hold several syncframes (MP4 / Matroska)
// a packet may hold several syncframes (MP4 / Matroska); E-AC-3 dependent
// substreams and further programs decode to no channels and are skipped
let mut out: Vec<Vec<f32>> = Vec::new();
let mut p = 0;
while p < data.len() {
let f = d.decode(&data[p..]).map_err(|e| CodecError::Decode(e.to_string()))?;
p += f.header.frame_bytes;
if f.channels.is_empty() {
continue;
}
if out.is_empty() {
out = f.channels;
} else {
@ -210,6 +235,31 @@ impl PacketDecoder {
self.channels = out.len();
return Ok(out);
}
Inner::Mpa(d) => {
// a packet holds one frame or several; a damaged first frame is an error, damage
// after a good frame ends the packet
let mut out: Vec<Vec<f32>> = Vec::new();
let mut p = 0;
while let Some(rest) = data.get(p..).filter(|r| !r.is_empty()) {
let f = match d.decode(rest) {
Ok(f) => f,
Err(e) if out.is_empty() => return Err(CodecError::Decode(e.to_string())),
Err(_) => break,
};
p += f.header.frame_bytes;
if out.is_empty() {
out = f.channels;
} else if out.len() == f.channels.len() {
for (o, c) in out.iter_mut().zip(f.channels) {
o.extend(c);
}
} else {
break;
}
}
self.channels = out.len();
return Ok(out);
}
};
let pkt = Packet::new_from_slice(0, ts, 0, data);
let buf = dec.decode(&pkt).map_err(|e| CodecError::Decode(e.to_string()))?;
@ -238,14 +288,25 @@ impl PacketDecoder {
up.reset();
}
}
Inner::Mpa(d) => d.reset(),
Inner::Opus { dec, .. } => dec.reset(),
Inner::Symphonia(d) => d.reset(),
Inner::Ac3(d) => d.reset(),
Inner::AacPending => {}
Inner::AacPending | Inner::MpegPending(_) => {}
}
}
}
/// The layer (1, 2 or 3) named by the MPEG-1/2 audio frame header that starts `frame`: the 12-bit
/// sync word (11 bits for MPEG-2.5), then `ID` and the 2-bit `layer` field, 3 for layer I down to 1
/// for layer III (ISO/IEC 11172-3 §2.4.2.3). `None` without a sync word or for the reserved layer 0.
pub fn mpeg_audio_layer(frame: &[u8]) -> Option<u8> {
match *frame {
[0xFF, b1, ..] if b1 & 0xE0 == 0xE0 && (b1 >> 1) & 3 != 0 => Some(4 - ((b1 >> 1) & 3)),
_ => None,
}
}
/// The output rate of an AAC stream: twice the core rate for HE-AAC (ISO/IEC 14496-3 §1.6.5).
/// Explicit signalling names the rate in the `AudioSpecificConfig`; with implicit signalling (an
/// AAC-LC config at 24 kHz or less) SBR is only visible as fill-element data in the access units,
@ -279,21 +340,39 @@ pub enum FixedFrames<'a> {
Aac(&'a [u8]),
/// MPEG-1/2 audio, layer I, II or III.
MpegAudio,
/// AC-3 (and E-AC-3).
/// AC-3 (and E-AC-3 in a stream of single syncframes, MPEG-TS).
Ac3,
/// E-AC-3 packets of whole syncframes (MP4, Matroska): an access unit may hold several
/// syncframes and dependent substreams.
Eac3,
}
/// Samples each packet decodes to at the stream's output `rate`, for codecs where that is the same
/// for every packet: AAC 1024 (2048 when [`PacketDecoder::aac`] upsamples an HE-AAC core); MPEG
/// audio and AC-3 from the frame header of `first`, the stream's first packet. `None` when it is
/// not known: an AAC configuration our decoder does not read, a header that does not parse, or a
/// header whose rate is not `rate`.
/// audio and AC-3 from the frame header of `first`, the stream's first packet; E-AC-3 from the
/// syncframes of the program we decode in `first`. `None` when it is not known: an AAC
/// configuration our decoder does not read, a header that does not parse, or a header whose rate
/// is not `rate`.
pub fn fixed_packet_samples(codec: FixedFrames, first: &[u8], rate: u32) -> Option<i64> {
let mpegts = match codec {
FixedFrames::Aac(asc) => {
let core = filmcraft_aac::Decoder::new(asc).ok()?.sample_rate();
return Some(if upsamples_core(core, rate) { 2048 } else { 1024 });
}
FixedFrames::Eac3 => {
let mut samples = 0usize;
let mut p = 0;
while let Some(h) = first.get(p..).and_then(|b| filmcraft_ac3::parse_header(b).ok()) {
if h.is_primary() {
if h.sample_rate != rate {
return None;
}
samples += h.samples();
}
p += h.frame_bytes;
}
return i64::try_from(samples).ok().filter(|&n| n > 0);
}
FixedFrames::MpegAudio => filmcraft_mpegts::Codec::MpegAudio,
FixedFrames::Ac3 => filmcraft_mpegts::Codec::Ac3,
};
@ -405,7 +484,7 @@ impl Upsample2x {
}
}
/// Whether AC-3 audio decodes (our ATSC A/52 decoder).
/// Whether AC-3 and E-AC-3 audio decode (our ATSC A/52 decoder).
pub const AC3_DECODER: bool = true;
/// An AudioSpecificConfig for AAC with these parameters (ISO/IEC 14496-3 §1.6.2.1, GA specific
@ -719,6 +798,22 @@ pub struct AudioFileSource {
impl AudioFileSource {
pub fn decode(name: &str, bytes: Arc<[u8]>) -> Result<Self> {
let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
if matches!(ext.as_str(), "mp2" | "mp3")
&& let Some((layer, rate, samples)) = decode_mpeg_layer12(&bytes)
{
let frames = samples.first().map_or(0, Vec::len);
let info = MediaInfo {
name: name.to_string(),
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: samples.len() as u32, codec: format!("MP{layer}"), bits_per_sample: None }],
container: ext.to_uppercase(),
start_timecode: None,
file_size: Some(bytes.len() as u64),
};
return Ok(Self { info, rate, samples });
}
let mss = MediaSourceStream::new(Box::new(std::io::Cursor::new(bytes.clone())), Default::default());
let mut hint = Hint::new();
hint.with_extension(&ext);
@ -774,6 +869,49 @@ impl AudioFileSource {
}
}
/// A raw MPEG audio Layer I / II stream (`.mp2`) decoded with our decoder: (layer, sample rate,
/// planar samples). Frames follow each other from the first frame header (after an ID3v2 tag);
/// damaged data is skipped up to the next frame header of the same layer and rate, and a frame
/// with another channel count keeps its duration as silence. `None` when the stream does not start
/// with a Layer I / II frame followed by another frame (or the end), e.g. MP3.
fn decode_mpeg_layer12(bytes: &[u8]) -> Option<(u8, u32, Vec<Vec<f32>>)> {
let mut p = match bytes {
// ID3v2: 10-byte header, a syncsafe size, and a 10-byte footer when flagged
[b'I', b'D', b'3', _, _, flags, s0, s1, s2, s3, ..] => {
let size = [*s0, *s1, *s2, *s3].iter().fold(0usize, |a, &b| (a << 7) | (b & 0x7F) as usize);
size.saturating_add(if flags & 0x10 != 0 { 20 } else { 10 })
}
_ => 0,
};
let frame_at = |p: usize| bytes.get(p..).and_then(|b| filmcraft_mpa::parse_header(b).ok()).filter(|h| h.frame_bytes > 0);
let first = frame_at(p)?;
let next = p.saturating_add(first.frame_bytes);
if first.layer == 3 || (next < bytes.len() && frame_at(next).is_none_or(|h| h.layer != first.layer || h.sample_rate != first.sample_rate)) {
return None;
}
let mut dec = filmcraft_mpa::Decoder::new();
let mut out: Vec<Vec<f32>> = vec![Vec::new(); first.channels()];
while let Some(rest) = bytes.get(p..).filter(|r| r.len() >= 4) {
match dec.decode(rest) {
Ok(f) if f.header.layer == first.layer && f.header.sample_rate == first.sample_rate => {
let n = f.header.samples();
if f.channels.len() == out.len() {
for (o, c) in out.iter_mut().zip(f.channels) {
o.extend(c);
}
} else {
for o in out.iter_mut() {
o.resize(o.len() + n, 0.0);
}
}
p += f.header.frame_bytes;
}
_ => p += 1,
}
}
out.first().is_some_and(|c| !c.is_empty()).then_some((first.layer, first.sample_rate, out))
}
impl MediaSource for AudioFileSource {
fn info(&self) -> &MediaInfo {
&self.info
@ -794,6 +932,57 @@ pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<std::result::Result<Shared
Some(AudioFileSource::decode(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
#[cfg(test)]
mod mpeg_layer12_tests {
use super::*;
/// A silent 48 kHz MPEG-1 Layer II frame (all-zero allocations) of `bitrate_index` and `mode`.
fn silent_frame(bitrate_index: u8, mode: u8) -> Vec<u8> {
let h = [0xFF, 0xFD, bitrate_index << 4 | 0x04, mode << 6];
let mut f = h.to_vec();
f.resize(filmcraft_mpa::parse_header(&h).unwrap().frame_bytes, 0);
f
}
/// #809: mono MP2 at 384 kbit/s (ffmpeg's default for mono) is outside ISO/IEC 11172-3's
/// allowed combinations, and symphonia rejected every frame, so the track played as silence.
#[test]
fn mono_mp2_at_384k_decodes() {
let mut d = PacketDecoder::mpeg_audio(2, 48_000).unwrap();
let out = d.decode(&silent_frame(14, 3), 0).unwrap();
assert_eq!((out.len(), out[0].len(), d.channels), (1, 1152, 1));
// a packet of two stereo frames decodes to both
let two = [silent_frame(10, 0), silent_frame(10, 0)].concat();
let out = d.decode(&two, 1152).unwrap();
assert_eq!((out.len(), out[0].len()), (2, 2304));
// damage: an error for the first frame, the good frames before it otherwise
assert!(d.decode(&[0u8; 8], 0).is_err());
assert!(d.decode(&silent_frame(10, 0)[..100], 0).is_err());
let partial = [silent_frame(10, 0), vec![0xFF, 0xFD, 0xA4]].concat();
assert_eq!(d.decode(&partial, 0).unwrap()[0].len(), 1152);
d.reset();
}
/// Raw `.mp2` files: after an ID3v2 tag, frame after frame, skipping damage.
#[test]
fn raw_streams_resync_after_damage() {
let mono = silent_frame(14, 3);
let file = [&b"ID3\x04\0\0\0\0\0\x05abcde"[..], &mono, &mono, &[0xFF, 0xFD, 0x13, 0x37, 0x42], &mono].concat();
let (layer, rate, samples) = decode_mpeg_layer12(&file).unwrap();
assert_eq!((layer, rate, samples.len(), samples[0].len()), (2, 48_000, 1, 3 * 1152));
let src = AudioFileSource::decode("x.mp2", file.clone().into()).unwrap();
assert_eq!(src.info().audio().map(|a| (a.channels, a.codec.as_str())), Some((1, "MP2")));
// MP3, junk and truncated data are not ours (symphonia gets them)
assert!(decode_mpeg_layer12(&[0xFF, 0xFB, 0x90, 0x44, 0, 0]).is_none());
for n in 0..40 {
assert!(decode_mpeg_layer12(&file[..n]).is_none(), "{n}");
}
assert!(decode_mpeg_layer12(&[&mono[..], &[0x12; 50]].concat()).is_none());
// an ID3v2 tag claiming the largest size (with a footer) points past the end
assert!(decode_mpeg_layer12(&[&b"ID3\x04\0\x10\x7F\x7F\x7F\x7F"[..], &mono].concat()).is_none());
}
}
#[cfg(test)]
mod he_aac_tests {
use super::*;
@ -967,5 +1156,60 @@ mod packet_time_tests {
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xFB, 0x90, 0x44], 48_000), None);
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF], 44_100), None);
assert_eq!(fixed_packet_samples(FixedFrames::Ac3, &[0x0B, 0x77], 48_000), None);
// E-AC-3 access units: 8-byte syncframes (frmsiz 3) at 48 kHz, stereo. A six-block
// independent frame with a dependent substream (7.1) is 1536 samples; two one-block
// frames are 512
let frame = |strmtyp: u8, numblkscod: u8| [0x0B, 0x77, strmtyp << 6, 3, (numblkscod << 4) | (2 << 1), 16 << 3, 0, 0];
let au = |frames: &[[u8; 8]]| frames.concat();
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &au(&[frame(0, 3), frame(1, 3)]), 48_000), Some(1536));
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &au(&[frame(0, 0), frame(0, 0)]), 48_000), Some(512));
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &au(&[frame(0, 3)]), 44_100), None);
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &[0x0B, 0x77], 48_000), None);
}
}
#[cfg(test)]
mod mpeg_audio_tests {
use super::*;
/// A silent mono 48 kHz MPEG-1 frame of `layer` (2: 64 kbps, 192 bytes; 3: 56 kbps, 168
/// bytes): header, then all-zero bit allocation / side information.
fn silent_frame(layer: u8) -> Vec<u8> {
let (b1, len) = if layer == 2 { (0xFD, 192) } else { (0xFB, 168) };
let mut f = vec![0xFF, b1, 0x44, 0xC0];
f.resize(len, 0);
f
}
#[test]
fn layer_comes_from_the_frame_header() {
assert_eq!(mpeg_audio_layer(&[0xFF, 0xFF, 0x10, 0xC0]), Some(1));
assert_eq!(mpeg_audio_layer(&silent_frame(2)), Some(2));
assert_eq!(mpeg_audio_layer(&silent_frame(3)), Some(3));
// MPEG-2 layer III, MPEG-2.5 layer II
assert_eq!(mpeg_audio_layer(&[0xFF, 0xF3, 0x90, 0x44]), Some(3));
assert_eq!(mpeg_audio_layer(&[0xFF, 0xE5, 0x90, 0x44]), Some(2));
// reserved layer, no sync word, too short
for bad in [&[0xFF, 0xF9, 0x44, 0xC0][..], &[0xFF, 0x1D], &[0x00, 0xFD], &[0xFF], &[]] {
assert_eq!(mpeg_audio_layer(bad), None, "{bad:?}");
}
}
/// #800: ffmpeg stores MP2 in MP4 as `mp4a` + object type 0x6B, which names no layer. Handing
/// it to the layer III decoder rejected every frame and the track played as silence.
#[test]
fn mp4_mpeg_audio_decodes_the_layer_its_frames_name() {
for layer in [2u8, 3] {
let mut d = PacketDecoder::for_isobmff(&filmcraft_isobmff::CodecConfig::Mp3, 48_000).unwrap();
// a packet without a frame header is an error, not a crash, and configures nothing
assert!(d.decode(&[0u8; 16], 0).is_err());
d.reset();
for k in 0..3u64 {
let out = d.decode(&silent_frame(layer), k * 1152).unwrap_or_else(|e| panic!("layer {layer}: {e}"));
assert_eq!(out.len(), 1, "layer {layer}");
assert_eq!(out[0].len(), 1152, "layer {layer}");
assert!(out[0].iter().all(|&s| s == 0.0), "layer {layer}");
}
}
}
}

29
crates/codecs/src/audio_timing_tests.rs
View File

@ -176,3 +176,32 @@ fn playing_at_another_rate_leaves_no_holes() {
let tail: Vec<Vec<f32>> = got.iter().map(|c| c[2048..].to_vec()).collect();
assert_eq!(holes(&tail), 0);
}
/// #790: the encoder priming stored as `CodecDelay` (1024 samples: 21.333 ms at 48 kHz, 23.220 ms
/// at 44.1 kHz) is trimmed exactly. The demuxer takes it off the millisecond timestamps rounded to
/// whole milliseconds; starting the first packet there played the track 16 (48 kHz) or 10 (44.1 kHz)
/// samples late.
#[test]
fn matroska_codec_delay_is_trimmed_to_the_sample() {
for rate in [48_000u32, 44_100] {
let (asc, units) = aac(rate);
let want = continuous(&asc, rate, &units);
let delay_ns = (1024 * 1_000_000_000 + rate as i64 / 2) / rate as i64;
let mut spec = TrackSpec::new(TrackKind::Audio, "A_AAC");
spec.codec_private = asc.clone();
spec.audio = Some((rate as f64, 2, None));
spec.codec_delay_ns = delay_ns as u64;
let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![spec], MuxOptions::default()).unwrap();
for (k, u) in units.iter().enumerate() {
let start_ns = (k as i64 * 1024 * 1_000_000_000 + rate as i64 / 2) / rate as i64;
w.write_frame(0, start_ns - delay_ns, true, u, None).unwrap();
}
let bytes: Arc<[u8]> = w.finish().unwrap().into_inner().into();
let src = MkvSource::open("primed.mkv", bytes).unwrap();
let frames = units.len().saturating_sub(2) * 1024;
let got = played(&src, rate, frames);
let trimmed: Vec<Vec<f32>> = want.iter().map(|c| c[1024..].to_vec()).collect();
let (n, first) = mismatches(&got, &trimmed, 0);
assert_eq!(n, 0, "{rate} Hz: {n} samples differ from the decode with the priming trimmed, first (channel, sample) {first:?}");
}
}

585
crates/codecs/src/avi.rs Normal file
View File

@ -0,0 +1,585 @@
//! AVI media source (`filmcraft-avi` demux, GOP-aware video via [`crate::gop`]).
//!
//! Video: Motion JPEG (frames without their own Huffman tables get the standard ones), H.264 and
//! HEVC (Annex B byte streams, our decoders), and uncompressed RGB / YUV ([`RawFormat`]). A frame
//! slot with an empty chunk (a "drop frame") repeats the picture before it. MPEG-4 Part 2
//! (DivX / Xvid) and DV have no decoder yet: their streams are listed and report the codec.
//!
//! Audio: PCM (integer and float) straight from the chunks; MPEG audio (MP3, MP2) and AC-3 frames,
//! which AVI chunks need not follow, are found again across chunk boundaries
//! ([`filmcraft_mpegts::frame_bytes`]) and decoded as packets.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use filmcraft_avi::{AviFile, BitmapInfo, StreamKind, WaveFormat};
use filmcraft_frame::{AudioBuffer, VideoFrame};
use filmcraft_isobmff::{PcmConfig, SampleEntry};
use filmcraft_media::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, SharedSource, VideoStreamInfo};
use filmcraft_time::{FrameRate, Tick};
use crate::audio::{PacketDecoder, decode_pcm};
use crate::gop::{GopCache, VideoSamples};
use crate::video::{RawFormat, RawVideoDecoder, VideoDecoder};
use crate::{CodecError, make_video_decoder};
pub fn sniff(b: &[u8]) -> bool {
filmcraft_avi::sniff(b)
}
#[derive(Clone, Debug, PartialEq)]
enum VCodec {
Jpeg,
H264,
Hevc,
Raw(RawFormat),
/// No decoder: the codec's name.
Unsupported(String),
}
fn fourcc_text(f: [u8; 4]) -> String {
let t: String = f.iter().map(|&c| if c.is_ascii_graphic() { c as char } else { '?' }).collect();
t.trim_end_matches('?').to_string()
}
/// The decoder for a video stream and its label.
fn video_codec(b: &BitmapInfo) -> (VCodec, String) {
if let Some(raw) = RawFormat::from_avi(b.compression, b.bit_count) {
return (VCodec::Raw(raw), raw.label().into());
}
let mut f = b.compression;
f.make_ascii_uppercase();
match &f {
b"MJPG" | b"AVRN" | b"LJPG" | b"JPGL" | b"DMB1" | b"JPEG" | b"MJPA" => (VCodec::Jpeg, "Motion JPEG".into()),
b"H264" | b"X264" | b"AVC1" | b"DAVC" | b"VSSH" => (VCodec::H264, "H.264".into()),
b"HEVC" | b"H265" | b"HEV1" | b"HVC1" | b"X265" => (VCodec::Hevc, "HEVC".into()),
b"DIVX" | b"DX50" | b"XVID" | b"FMP4" | b"MP4V" | b"M4S2" | b"3IV2" | b"DIV3" | b"MP43" | b"MP42" => {
(VCodec::Unsupported("MPEG-4 Part 2 (DivX / Xvid)".into()), "MPEG-4 Part 2 (DivX / Xvid)".into())
}
b"DVSD" | b"DV25" | b"DV50" | b"DVHD" | b"DVSL" | b"CDVC" | b"DVCP" | b"DVPP" => (VCodec::Unsupported("DV".into()), "DV".into()),
_ => {
let name = fourcc_text(b.compression);
(VCodec::Unsupported(format!("\"{name}\"")), name)
}
}
}
/// One picture of the video stream: its frame slot (in `scale / rate` units) and chunk.
#[derive(Clone, Copy, Debug)]
struct VFrame {
pts: i64,
chunk: usize,
key: bool,
}
#[derive(Clone, Debug)]
enum ACodec {
Pcm(PcmConfig),
/// MPEG audio (layer 1-3) or AC-3 frames, found again across chunks.
Framed {
ts: filmcraft_mpegts::Codec,
layer: u8,
},
Unsupported(String),
}
fn audio_codec(w: &WaveFormat) -> (ACodec, String) {
let pcm = |float: bool| {
let bits = w.bits_per_sample;
let cfg = PcmConfig { bits, float, big_endian: false, signed: bits > 8, channels: u32::from(w.channels.max(1)), sample_rate: f64::from(w.sample_rate) };
(ACodec::Pcm(cfg), format!("PCM {bits}-bit{}", if float { " float" } else { "" }))
};
match w.format_tag {
1 if matches!(w.bits_per_sample, 8 | 16 | 24 | 32) => pcm(false),
3 if matches!(w.bits_per_sample, 32 | 64) => pcm(true),
0x55 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 3 }, "MP3".into()),
0x50 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::MpegAudio, layer: 2 }, "MPEG Audio".into()),
0x2000 => (ACodec::Framed { ts: filmcraft_mpegts::Codec::Ac3, layer: 0 }, "AC-3".into()),
tag => {
let name = format!("audio format 0x{tag:04X}");
(ACodec::Unsupported(name.clone()), name)
}
}
}
/// A slice of a chunk: (chunk, offset in it, length).
type Piece = (usize, usize, usize);
struct AudioState {
decoder: Option<PacketDecoder>,
packets: HashMap<usize, Arc<Vec<Vec<f32>>>>,
order: Vec<usize>,
last_decoded: Option<usize>,
}
struct AviAudio {
stream: usize,
codec: ACodec,
/// The pieces of packet `k` are `pieces[first[k]..first[k + 1]]`.
pieces: Vec<Piece>,
first: Vec<usize>,
/// Packet start positions in sample frames.
starts: Vec<i64>,
state: Mutex<AudioState>,
}
pub struct AviSource {
info: MediaInfo,
bytes: crate::Src,
file: AviFile,
vstream: Option<usize>,
vcodec: VCodec,
/// Pictures in stored (decode) order.
vframes: Vec<VFrame>,
/// `vframes` indices in presentation order (H.264 with B-frames is stored in decode order),
/// and their presentation times.
vorder: Vec<usize>,
vorder_pts: Vec<i64>,
/// Frame slots of the video stream (where the last picture ends).
vslots: i64,
rate: FrameRate,
bottom_up: bool,
video: GopCache,
audios: Vec<AviAudio>,
}
/// Whether an Annex B access unit holds an H.264 IDR or HEVC IRAP picture.
fn random_access(data: &[u8], hevc: bool) -> bool {
filmcraft_bitstream::annexb_nals(data).iter().any(|n| match n.first() {
Some(h) if hevc => (16..=23).contains(&((h >> 1) & 0x3F)),
Some(h) => h & 0x1F == 5,
None => false,
})
}
/// The packets of a framed audio stream: every frame header found in the stream of chunks, and
/// the samples each frame decodes to. Bytes that start no frame are skipped (resynchronisation).
fn split_frames(
file: &AviFile,
bytes: &crate::Src,
s: usize,
ts: &filmcraft_mpegts::Codec,
) -> (Vec<Piece>, Vec<usize>, Vec<u32>, Option<filmcraft_mpegts::FrameInfo>) {
let Some(st) = file.streams.get(s) else { return Default::default() };
// chunk start positions in the concatenated stream
let mut at = Vec::with_capacity(st.chunks.len());
let mut total = 0usize;
for c in &st.chunks {
at.push(total);
total = total.saturating_add(c.size as usize);
}
let locate = |pos: usize| -> Option<(usize, usize)> {
let k = at.partition_point(|&a| a <= pos).checked_sub(1)?;
Some((k, pos - at.get(k)?))
};
let (mut pieces, mut first, mut samples) = (Vec::new(), Vec::new(), Vec::new());
let mut info = None;
// a sliding window over the chunks: `win` holds the stream from `base`
let (mut win, mut base, mut next) = (Vec::<u8>::new(), 0usize, 0usize);
let mut pos = 0usize;
while pos < total {
// keep at least a header's worth (and the frame, when we know its length) in the window
while win.len() < (pos - base) + 4096 && next < st.chunks.len() {
win.extend(file.read_chunk(&bytes, s, next).unwrap_or_default());
next += 1;
}
let Some(here) = win.get(pos - base..) else { break };
match filmcraft_mpegts::frame_bytes(ts, here) {
Some(n) if pos + n <= total => {
if info.is_none() {
info = filmcraft_mpegts::frame_info(ts, here);
}
let k = filmcraft_mpegts::frame_info(ts, here).map_or(0, |f| f.samples);
first.push(pieces.len());
let mut p = pos;
while p < pos + n {
let Some((c, off)) = locate(p) else { break };
let len = (st.chunks.get(c).map_or(0, |ch| ch.size as usize) - off).min(pos + n - p);
if len == 0 {
break;
}
pieces.push((c, off, len));
p += len;
}
samples.push(k);
pos += n;
}
_ if here.len() < 8 && next >= st.chunks.len() => break,
_ => pos += 1,
}
// drop what is behind
if pos - base > 1 << 16 {
let cut = pos - base;
win.drain(..cut.min(win.len()));
base += cut;
}
}
(pieces, first, samples, info)
}
impl AviSource {
pub fn open(name: &str, bytes: Arc<[u8]>) -> crate::Result<Self> {
Self::open_reader(name, Arc::new(filmcraft_media::reader::MemReader(bytes)))
}
/// Open from a random-access reader: the index now, frames and audio on demand.
pub fn open_reader(name: &str, reader: filmcraft_media::SharedReader) -> crate::Result<Self> {
let bytes = crate::Src(reader);
let file = filmcraft_avi::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?;
let vstream = file.streams.iter().position(|s| s.kind == StreamKind::Video && s.video.is_some() && !s.chunks.is_empty());
let astreams: Vec<usize> = file
.streams
.iter()
.enumerate()
.filter(|(_, s)| s.kind == StreamKind::Audio && s.audio.is_some() && !s.chunks.is_empty())
.map(|(i, _)| i)
.take(filmcraft_media::MAX_AUDIO_STREAMS)
.collect();
if vstream.is_none() && astreams.is_empty() {
return Err(CodecError::Unsupported("no playable streams".into()));
}
// video: frame slots, pictures and key frames
let (mut vcodec, mut vframes, mut vslots, mut rate, mut bottom_up, mut video) =
(VCodec::Unsupported(String::new()), Vec::new(), 0i64, FrameRate::new(25, 1), false, None);
if let Some(vs) = vstream.and_then(|i| file.streams.get(i)) {
let b = vs.video.clone().unwrap_or_default();
let (codec, label) = video_codec(&b);
rate = FrameRate::new(i64::from(vs.rate), i64::from(vs.scale));
bottom_up = b.height > 0;
let (w, h) = (b.width.unsigned_abs(), b.height.unsigned_abs());
for (k, c) in vs.chunks.iter().enumerate() {
if c.size == 0 {
continue;
}
let key = match &codec {
VCodec::H264 | VCodec::Hevc if !vs.keyframes_known => {
file.read_chunk(&bytes, vstream.unwrap_or(0), k).is_ok_and(|d| random_access(&d, codec == VCodec::Hevc))
}
VCodec::H264 | VCodec::Hevc => c.key,
_ => true,
};
vframes.push(VFrame { pts: i64::from(vs.start) + k as i64, chunk: k, key });
}
vslots = i64::from(vs.start) + vs.chunks.len() as i64;
// AVI has no timestamps: H.264 with B-frames is stored in decode order, and the
// presentation order comes from the pictures' order counts. A first look at the start
// of the stream decides whether it reorders at all (reading every frame's header is the
// cost of a reordering stream only).
if codec == VCodec::H264
&& let Some(si) = vstream
{
let heads = |n: usize| -> Vec<Vec<u8>> {
vframes
.iter()
.take(n)
.map(|f| {
file.streams
.get(si)
.and_then(|s| s.chunks.get(f.chunk))
.and_then(|c| filmcraft_media::reader::read_range(&*bytes.0, c.offset, (c.size as usize).min(8 << 10)).ok())
.unwrap_or_default()
})
.collect()
};
let reorders = |p: &[i64]| p.windows(2).any(|w| w[1] < w[0]);
if crate::mxf::avc_presentation_order(&heads(300)).is_some_and(|p| reorders(&p))
&& let Some(ranks) = crate::mxf::avc_presentation_order(&heads(vframes.len()))
{
// the pictures keep the slots they occupy (empty drop-frame chunks between them
// stay), shown in their presentation order
let mut slots: Vec<i64> = vframes.iter().map(|f| f.pts).collect();
slots.sort_unstable();
for (f, r) in vframes.iter_mut().zip(ranks) {
if let Some(&p) = usize::try_from(r).ok().and_then(|r| slots.get(r)) {
f.pts = p;
}
}
}
}
let secs = vslots as f64 / rate.as_f64().max(1e-9);
// The rate the pictures play at: a writer that kept a fine time base (ffmpeg copying an
// MP4's 1/600) fills the slots between pictures with empty chunks, every 20th slot
// holding a picture at 30 fps. A regular spacing makes the rate; slots stay the units.
let shown = if vframes.len() > 1 {
let mut gaps: Vec<i64> = vframes.windows(2).map(|w| w[1].chunk as i64 - w[0].chunk as i64).filter(|g| *g > 0).take(1000).collect();
gaps.sort_unstable();
let gap = gaps.get(gaps.len() / 2).copied().unwrap_or(1).max(1);
let regular = gaps.iter().filter(|g| **g == gap).count() * 10 >= gaps.len() * 9;
if gap > 1 && regular { FrameRate::new(i64::from(vs.rate), i64::from(vs.scale).saturating_mul(gap)) } else { rate }
} else {
rate
};
let bitrate = (secs > 0.0).then(|| (vs.chunks.iter().map(|c| u64::from(c.size)).sum::<u64>() as f64 * 8.0 / secs) as u64);
video = Some(VideoStreamInfo {
width: w,
height: h,
frame_rate: shown,
par: (1, 1),
codec: label,
pixel_format: String::new(),
color: crate::stream_color::resolve(w, h, &[]),
has_alpha: false,
bitrate,
hdr: None,
});
vcodec = codec;
}
// audio: packets and their start positions
let mut audios = Vec::new();
let mut audio_streams = Vec::new();
for &s in &astreams {
let Some(st) = file.streams.get(s) else { continue };
let w = st.audio.clone().unwrap_or_default();
let (codec, label) = audio_codec(&w);
let mut sample_rate = w.sample_rate.max(1);
let mut channels = u32::from(w.channels.max(1));
let (pieces, first, starts) = match &codec {
ACodec::Pcm(cfg) => {
let block = usize::from(w.block_align.max(1)).max((usize::from(cfg.bits).div_ceil(8)) * channels as usize);
let mut t = 0i64;
let mut starts = Vec::with_capacity(st.chunks.len());
for c in &st.chunks {
starts.push(t);
t = t.saturating_add((c.size as usize / block) as i64);
}
let pieces: Vec<Piece> = st.chunks.iter().enumerate().map(|(k, c)| (k, 0, c.size as usize)).collect();
let first = (0..pieces.len()).collect();
(pieces, first, starts)
}
ACodec::Framed { ts, .. } => {
let (pieces, first, samples, info) = split_frames(&file, &bytes, s, ts);
if let Some(i) = info {
sample_rate = i.sample_rate.max(1);
channels = i.channels.max(1);
}
let mut t = 0i64;
let starts = samples
.iter()
.map(|&k| {
let s = t;
t = t.saturating_add(i64::from(k));
s
})
.collect();
(pieces, first, starts)
}
ACodec::Unsupported(_) => (Vec::new(), Vec::new(), Vec::new()),
};
// a stream that starts late (dwStart, in scale / rate units)
let delay = (i128::from(st.start) * i128::from(st.scale) * i128::from(sample_rate) / i128::from(st.rate.max(1))) as i64;
let starts: Vec<i64> = starts.into_iter().map(|s: i64| s.saturating_add(delay)).collect();
audio_streams.push(AudioStreamInfo {
sample_rate,
channels,
codec: label,
bits_per_sample: (w.bits_per_sample > 0).then_some(u32::from(w.bits_per_sample)),
});
let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None });
audios.push(AviAudio { stream: s, codec, pieces, first, starts, state });
}
// the longer of the picture and the sound
let vdur = if vslots > 0 { rate.tick_of(vslots) } else { Tick::ZERO };
let adur = audios
.iter()
.zip(&audio_streams)
.filter_map(|(a, i)| {
let last = a.starts.last()?;
let tail = a.starts.len().checked_sub(2).and_then(|k| a.starts.get(k)).map_or(0, |p| last - p);
Some(Tick::from_rational(last + tail, 1, i64::from(i.sample_rate)))
})
.max()
.unwrap_or_default();
let info = MediaInfo {
name: name.to_string(),
kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly },
duration: vdur.max(adur),
video,
audio_streams,
container: "AVI".into(),
start_timecode: None,
file_size: Some(bytes.0.len()),
};
let mut vorder: Vec<usize> = (0..vframes.len()).collect();
vorder.sort_by_key(|&i| vframes.get(i).map_or(0, |f| f.pts));
let vorder_pts = vorder.iter().map(|&i| vframes.get(i).map_or(0, |f| f.pts)).collect();
Ok(Self { info, bytes, file, vstream, vcodec, vframes, vorder, vorder_pts, vslots, rate, bottom_up, video: GopCache::new(None), audios })
}
fn read(&self, stream: usize, chunk: usize) -> crate::Result<Vec<u8>> {
self.file.read_chunk(&self.bytes, stream, chunk).map_err(|e| CodecError::Container(e.to_string()))
}
/// The bytes of audio packet `k`.
fn packet(&self, a: &AviAudio, k: usize) -> crate::Result<Vec<u8>> {
let (from, to) = (
a.first.get(k).copied().ok_or_else(|| CodecError::Container("audio packet out of range".into()))?,
a.first.get(k + 1).copied().unwrap_or(a.pieces.len()),
);
let mut out = Vec::new();
for &(c, off, len) in a.pieces.get(from..to).unwrap_or_default() {
let chunk = self.read(a.stream, c)?;
out.extend_from_slice(chunk.get(off..off + len).ok_or_else(|| CodecError::Container("audio chunk shorter than its index".into()))?);
}
Ok(out)
}
fn audio_packet(&self, a: &AviAudio, st: &mut AudioState, ainfo: &AudioStreamInfo, i: usize) -> crate::Result<Arc<Vec<Vec<f32>>>> {
if let Some(p) = st.packets.get(&i) {
return Ok(p.clone());
}
let data = self.packet(a, i)?;
let decoded = match &a.codec {
ACodec::Pcm(cfg) => decode_pcm(&data, cfg),
ACodec::Unsupported(name) => return Err(CodecError::Unsupported(name.clone())),
ACodec::Framed { ts, layer } => {
if st.decoder.is_none() {
st.decoder = Some(match ts {
filmcraft_mpegts::Codec::Ac3 => PacketDecoder::ac3()?,
_ => PacketDecoder::mpeg_audio(*layer, ainfo.sample_rate)?,
});
}
let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?;
// non-sequential access: reset and prime with the packet before (the bit reservoir)
if st.last_decoded.is_none_or(|l| l + 1 != i) {
d.reset();
if let Some(prev) = i.checked_sub(1).and_then(|j| self.packet(a, j).ok()) {
let _ = d.decode(&prev, 0);
}
}
let start = a.starts.get(i).copied().unwrap_or(0).max(0) as u64;
let r = d.decode(&data, start);
st.last_decoded = Some(i);
r.unwrap_or_default()
}
};
let p = Arc::new(decoded);
st.packets.insert(i, p.clone());
st.order.push(i);
if st.order.len() > 4096 {
let old = st.order.remove(0);
st.packets.remove(&old);
}
Ok(p)
}
}
/// The video stream as a [`VideoSamples`] table.
struct AviVideo<'a> {
src: &'a AviSource,
stream: usize,
}
impl VideoSamples for AviVideo<'_> {
fn count(&self) -> usize {
self.src.vframes.len()
}
fn pts(&self, i: usize) -> i64 {
self.src.vframes.get(i).map_or(0, |f| f.pts)
}
fn sync_before(&self, i: usize) -> usize {
let i = i.min(self.src.vframes.len().saturating_sub(1));
self.src.vframes.get(..=i).and_then(|f| f.iter().rposition(|f| f.key)).unwrap_or(0)
}
fn sample_at(&self, t: i64) -> Option<usize> {
// the picture shown in slot `t`: the last one at or before it (empty chunks repeat it)
if t >= self.src.vslots {
return None;
}
let k = self.src.vorder_pts.partition_point(|&p| p <= t).checked_sub(1)?;
self.src.vorder.get(k).copied()
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
let f = self.src.vframes.get(i).ok_or_else(|| CodecError::Container("frame out of range".into()))?;
self.src.read(self.stream, f.chunk)
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
let v = self.src.info.video.as_ref().ok_or_else(|| CodecError::Unsupported("no video stream".into()))?;
let (w, h) = (v.width.min(u32::from(u16::MAX)) as u16, v.height.min(u32::from(u16::MAX)) as u16);
match &self.src.vcodec {
VCodec::Jpeg => make_video_decoder(&SampleEntry::jpeg(w, h)),
VCodec::H264 => Ok(Box::new(crate::video::H264Decoder::annexb())),
VCodec::Hevc => Ok(Box::new(crate::video::HevcDecoder::annexb())),
VCodec::Raw(f) => Ok(Box::new(RawVideoDecoder::new(*f, v.width, v.height, self.src.bottom_up)?)),
VCodec::Unsupported(name) => Err(CodecError::Unsupported(format!("no decoder for {name} video"))),
}
}
}
impl MediaSource for AviSource {
fn info(&self) -> &MediaInfo {
&self.info
}
fn video_frame(&self, req: FrameRequest) -> Result<Arc<VideoFrame>, MediaError> {
let s = self.vstream.ok_or(MediaError::NoStream("video"))?;
let t = req.time.max(Tick::ZERO);
let slot = |t: Tick| self.rate.frame_at(t);
let late = filmcraft_media::cancel::catch_up().map(|m| slot(t - m));
Ok(self.video.frame_late(&AviVideo { src: self, stream: s }, slot(t), late)?)
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
self.audio_stream(0, start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?;
if let ACodec::Unsupported(name) = &a.codec {
return Err(MediaError::Unsupported(format!("{name} audio")));
}
let ch = ainfo.channels.max(1) as usize;
let ratio = f64::from(ainfo.sample_rate) / f64::from(sample_rate.max(1));
let s0 = (start as f64 * ratio).floor() as i64;
let need = (frames as f64 * ratio).ceil() as i64 + 2;
let mut src: Vec<Vec<f32>> = vec![vec![0.0; need.max(0) as usize]; ch];
let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner());
let mut i = a.starts.partition_point(|&x| x <= s0).saturating_sub(1);
while let Some(&pk_start) = a.starts.get(i) {
if pk_start >= s0 + need {
break;
}
let pk = self.audio_packet(a, &mut st, ainfo, i)?;
for (c, dst) in src.iter_mut().enumerate() {
let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue };
for (k, v) in chan.iter().enumerate() {
let pos = pk_start + k as i64 - s0;
if let Some(d) = usize::try_from(pos).ok().and_then(|p| dst.get_mut(p)) {
*d = *v;
}
}
}
i += 1;
}
drop(st);
let frac = s0 as f64 - start as f64 * ratio;
let mut out = AudioBuffer::silence(sample_rate, ch, frames);
for (c, o) in out.channels.iter_mut().enumerate() {
let Some(s) = src.get(c) else { continue };
for (k, d) in o.iter_mut().enumerate() {
let pos = k as f64 * ratio - frac;
let i0 = pos.floor().max(0.0) as usize;
let f = (pos - i0 as f64) as f32;
let a = s.get(i0).copied().unwrap_or(0.0);
let b = s.get(i0 + 1).copied().unwrap_or(a);
*d = a + (b - a) * f;
}
}
Ok(out)
}
}
pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource, MediaError>> {
if !sniff(&bytes) {
return None;
}
Some(AviSource::open(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
/// [`filmcraft_media::ReaderOpener`] for AVI.
pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedReader) -> Option<Result<SharedSource, MediaError>> {
if !sniff(head) {
return None;
}
Some(AviSource::open_reader(name, reader.clone()).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}

232
crates/codecs/src/avi_tests.rs Normal file
View File

@ -0,0 +1,232 @@
//! AVI files made by ffmpeg (skipped without it), checked against ffmpeg's decode of the same file:
//! bit-exact where the decoder is (H.264, uncompressed video, PCM), within decoder tolerance where
//! it differs (JPEG, MP3 / MP2 / AC-3).
use std::path::Path;
use std::process::Command;
use std::sync::Arc;
use filmcraft_media::FrameRequest;
use crate::tests::{ffmpeg, fixture_path, reference_yuv, yuv_bytes};
const PIC: [&str; 4] = ["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2"];
const TONE: [&str; 4] = ["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=2"];
fn avi(name: &str, video: &[&str], audio: &[&str]) -> Option<(std::path::PathBuf, filmcraft_media::SharedSource)> {
let mut args: Vec<&str> = PIC.to_vec();
args.extend_from_slice(&TONE);
args.extend_from_slice(video);
args.extend_from_slice(audio);
let path = fixture_path(name, &args)?;
let bytes: Arc<[u8]> = std::fs::read(&path).ok()?.into();
Some((path.clone(), crate::open_bytes(name, bytes).unwrap_or_else(|e| panic!("{name}: {e}"))))
}
/// ffmpeg's decode of the first audio stream as interleaved f32 at its own rate.
fn reference_audio(path: &Path) -> Vec<f32> {
let out = Command::new(ffmpeg().unwrap())
.args(["-v", "error", "-i"])
.arg(path)
.args(["-map", "0:a:0", "-f", "f32le", "-c:a", "pcm_f32le", "-"])
.output()
.unwrap();
out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
}
/// The first `n` mono samples of stream 0 as FilmCraft decodes them.
fn ours(src: &filmcraft_media::SharedSource, n: usize) -> Vec<f32> {
src.audio(0, n, 48_000).unwrap().channels[0].clone()
}
fn check_info(name: &str, src: &filmcraft_media::SharedSource, vcodec: &str, acodec: &str) {
let info = src.info();
assert_eq!(info.container, "AVI", "{name}");
let v = info.video.as_ref().unwrap();
assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, vcodec), "{name}");
assert_eq!(v.frame_rate.as_f64(), 25.0, "{name}");
let a = &info.audio_streams[0];
assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (48_000, 1, acodec), "{name}");
let secs = info.duration.seconds();
assert!((secs - 2.0).abs() < 0.1, "{name}: duration {secs}");
}
#[test]
fn h264_frames_are_bit_exact_with_random_access_and_mp3_plays() {
let Some((path, src)) =
avi("avi_h264_mp3.avi", &["-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p"], &["-c:a", "libmp3lame", "-b:a", "128k"])
else {
return;
};
check_info("h264", &src, "H.264", "MP3");
let reference = reference_yuv(&path, "yuv420p").unwrap();
let frame = reference.len() / 50;
let rate = src.info().frame_rate();
for k in [37, 3, 24, 49, 0, 13, 12, 11, 30, 31, 32] {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "H.264 frame {k} differs from ffmpeg");
}
// MP3 (a different decoder): the tone, in time with ffmpeg's decode
let r = reference_audio(&path);
let a = ours(&src, 48_000);
let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200);
// ffmpeg's test tone is at 1/8 of full scale
assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "MP3: tone {rms_ref}, difference {rms_diff}");
// random access into the middle matches sequential decoding
let mid = src.audio(24_000, 4_800, 48_000).unwrap().channels[0].clone();
let worst = mid.iter().zip(&a[24_000..28_800]).map(|(x, y)| (x - y).abs()).fold(0.0, f32::max);
assert!(worst < 0.01, "MP3 seek: {worst}");
}
fn rms_and_diff(reference: &[f32], ours: &[f32], range: std::ops::Range<usize>) -> (f32, f32) {
let n = range.len() as f32;
let rms = (range.clone().map(|i| reference[i].powi(2)).sum::<f32>() / n).sqrt();
let diff = (range.map(|i| (reference[i] - ours[i]).powi(2)).sum::<f32>() / n).sqrt();
(rms, diff)
}
#[test]
fn mjpeg_and_pcm() {
let Some((path, src)) = avi("avi_mjpeg_pcm.avi", &["-c:v", "mjpeg", "-q:v", "3", "-pix_fmt", "yuvj420p"], &["-c:a", "pcm_s16le"]) else { return };
check_info("mjpeg", &src, "Motion JPEG", "PCM 16-bit");
let reference = reference_yuv(&path, "rgba").unwrap();
let frame = reference.len() / 50;
let rate = src.info().frame_rate();
for k in [10, 0, 49] {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap().to_rgba8();
let r = &reference[k * frame..(k + 1) * frame];
let mean = f.iter().zip(r).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::<u32>() as f64 / f.len() as f64;
// another JPEG decoder and chroma upsampling than ffmpeg's: close, not identical
assert!(mean < 6.0, "JPEG frame {k}: mean difference {mean}");
}
// PCM: exactly ffmpeg's samples
let r = reference_audio(&path);
assert_eq!(ours(&src, 48_000), r[..48_000].to_vec());
}
#[test]
fn uncompressed_video_is_bit_exact() {
for (name, pix, codec, reference_fmt) in [
("avi_bgr24.avi", "bgr24", "Uncompressed RGB 24-bit", "rgba"),
("avi_yuyv.avi", "yuyv422", "Uncompressed YUV 4:2:2", "yuv422p"),
("avi_i420.avi", "yuv420p", "Uncompressed YUV 4:2:0", "yuv420p"),
("avi_nv12.avi", "nv12", "Uncompressed YUV 4:2:0", "yuv420p"),
] {
let Some((path, src)) = avi(name, &["-c:v", "rawvideo", "-pix_fmt", pix], &["-c:a", "pcm_s16le"]) else { return };
check_info(name, &src, codec, "PCM 16-bit");
let reference = reference_yuv(&path, reference_fmt).unwrap();
let frame = reference.len() / 50;
let rate = src.info().frame_rate();
for k in [7, 0, 49] {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
let got = if reference_fmt == "rgba" { f.to_rgba8() } else { yuv_bytes(&f) };
assert!(got == reference[k * frame..(k + 1) * frame], "{name}: frame {k} differs from ffmpeg");
}
}
}
#[test]
fn mp2_and_ac3_audio() {
for (name, codec, label) in [("avi_mp2.avi", "mp2", "MPEG Audio"), ("avi_ac3.avi", "ac3", "AC-3")] {
let Some((path, src)) = avi(name, &["-c:v", "mjpeg"], &["-c:a", codec, "-b:a", "192k"]) else { return };
assert_eq!(src.info().audio_streams[0].codec, label, "{name}");
let r = reference_audio(&path);
let a = ours(&src, 48_000);
let (rms_ref, rms_diff) = rms_and_diff(&r, &a, 4_800..43_200);
assert!(rms_ref > 0.05 && rms_diff < rms_ref * 0.05, "{name}: tone {rms_ref}, difference {rms_diff}");
}
}
/// Dropped frames are stored as empty chunks: their slot shows the picture before.
#[test]
fn drop_frames_hold_the_previous_picture() {
let args = [
"-f",
"lavfi",
"-i",
"testsrc2=s=160x120:r=25:d=2",
"-vf",
"select='not(eq(n\\,10))'",
"-fps_mode",
"passthrough",
"-c:v",
"rawvideo",
"-pix_fmt",
"bgr24",
];
let Some(path) = fixture_path("avi_drop.avi", &args) else { return };
let src = crate::open_bytes("avi_drop.avi", std::fs::read(&path).unwrap().into()).unwrap();
let rate = src.info().frame_rate();
let at = |k: i64| src.video_frame(FrameRequest::full(rate.tick_of(k))).unwrap().to_rgba8();
assert!(at(10) == at(9), "the dropped slot holds frame 9");
assert!(at(11) != at(9));
}
/// DivX / Xvid has no decoder yet: the file opens, the codec is named, the picture reports it, and
/// the sound plays.
#[test]
fn mpeg4_part2_is_named_and_its_audio_plays() {
let Some((_, src)) = avi("avi_xvid.avi", &["-c:v", "mpeg4", "-vtag", "XVID"], &["-c:a", "pcm_s16le"]) else { return };
assert_eq!(src.info().video.as_ref().unwrap().codec, "MPEG-4 Part 2 (DivX / Xvid)");
let e = src.video_frame(FrameRequest::full(filmcraft_time::Tick::ZERO)).unwrap_err().to_string();
assert!(e.contains("MPEG-4 Part 2"), "{e}");
assert!(ours(&src, 4_800).iter().any(|s| s.abs() > 0.1));
}
/// A Motion JPEG frame without Huffman tables (the AVI1 convention) decodes with the standard ones:
/// exactly as the same frame with them.
#[test]
fn jpeg_without_huffman_tables_uses_the_standard_ones() {
let Some(path) = fixture_path(
"default_huffman.jpg",
&["-f", "lavfi", "-i", "testsrc2=s=160x120", "-frames:v", "1", "-c:v", "mjpeg", "-huffman", "default", "-q:v", "3"],
) else {
return;
};
let jpeg = std::fs::read(&path).unwrap();
// remove every DHT segment
let mut stripped = jpeg[..2].to_vec();
let mut at = 2;
while at + 4 <= jpeg.len() && jpeg[at] == 0xFF && jpeg[at + 1] != 0xDA {
let len = u16::from_be_bytes([jpeg[at + 2], jpeg[at + 3]]) as usize;
if jpeg[at + 1] != 0xC4 {
stripped.extend_from_slice(&jpeg[at..at + 2 + len]);
}
at += 2 + len;
}
stripped.extend_from_slice(&jpeg[at..]);
assert!(stripped.len() < jpeg.len() && !stripped.windows(2).any(|w| w == [0xFF, 0xC4]));
use crate::video::VideoDecoder;
let mut d = crate::video::MjpegDecoder;
let a = d.decode(&jpeg, 0).unwrap().remove(0).frame.to_rgba8();
let b = d.decode(&stripped, 0).unwrap().remove(0).frame.to_rgba8();
assert!(a == b);
// the inserted segment is 418 bytes and well-formed: 4 tables of 12, 12, 162, 162 codes
let with = crate::video::with_default_huffman(&stripped);
assert_eq!(with.len(), stripped.len() + 420);
}
/// H.264 with B-frames copied from an MP4 whose time base is 1/600 (ffmpeg keeps it): 600 frame
/// slots a second, a picture in every 24th at 25 fps, empty drop chunks between. The pictures stay
/// at their times in presentation order, and the stream reports the rate they play at.
#[test]
fn copied_h264_keeps_its_timing_with_a_fine_time_base() {
let mp4 =
["-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=2", "-c:v", "libx264", "-bf", "2", "-g", "12", "-pix_fmt", "yuv420p", "-video_track_timescale", "600"];
let Some(src_mp4) = fixture_path("avi_copy_source.mp4", &mp4) else { return };
let src_path = src_mp4.to_string_lossy().into_owned();
let Some(path) = fixture_path("avi_copy_600.avi", &["-i", &src_path, "-c:v", "copy", "-bsf:v", "h264_mp4toannexb", "-f", "avi"]) else { return };
let src = crate::open_bytes("avi_copy_600.avi", std::fs::read(&path).unwrap().into()).unwrap();
let v = src.info().video.clone().unwrap();
assert_eq!(v.frame_rate.as_f64(), 25.0, "the rate the pictures play at, not the 600 slots");
let reference = reference_yuv(&path, "yuv420p").unwrap();
let frame = reference.len() / 50;
for k in [37usize, 3, 24, 49, 0, 13, 12, 11, 30] {
let at = filmcraft_time::Tick::from_rational(k as i64, 1, 25);
let f = src.video_frame(FrameRequest::full(at)).unwrap();
assert!(yuv_bytes(&f) == reference[k * frame..(k + 1) * frame], "picture {k} at {:.2} s differs from ffmpeg", k as f64 / 25.0);
}
// between pictures, the one before holds
let mid = src.video_frame(FrameRequest::full(filmcraft_time::Tick::from_rational(1, 1, 50))).unwrap();
assert!(yuv_bytes(&mid) == reference[0..frame]);
}

414
crates/codecs/src/gop.rs
View File

@ -109,8 +109,9 @@ impl std::ops::Sub for GopStats {
}
}
/// A container's video sample table, in decode (file) order.
pub trait VideoSamples {
/// A container's video sample table, in decode (file) order. `Sync`: a cache decodes on its own
/// threads ([`isolated`]).
pub trait VideoSamples: Sync {
fn count(&self) -> usize;
/// Presentation timestamp of sample `i` (track units).
fn pts(&self, i: usize) -> i64;
@ -122,6 +123,18 @@ pub trait VideoSamples {
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>>;
}
/// The sync sample to start decoding at for sample `i`: its sync sample, or an earlier one when
/// `i` is a leading picture of an open GOP (stored after its random-access picture, shown before
/// it, like the RASL pictures after every HEVC CRA of iPhone video). Those are predicted from the
/// GOP before, so a decoder starting at their random-access picture leaves them out.
fn decode_start(s: &dyn VideoSamples, i: usize) -> usize {
let mut key = s.sync_before(i);
while key > 0 && s.pts(i) < s.pts(key) {
key = s.sync_before(key - 1).min(key - 1);
}
key
}
struct State {
decoder: Option<Box<dyn VideoDecoder>>,
/// The decoder only produces intra pictures: frames decode independently and in parallel.
@ -140,6 +153,10 @@ struct State {
bytes: usize,
/// When a frame was last asked of this cache.
last_used: Instant,
/// Presentation pts of the frame the last request wanted: eviction keeps a window around it
/// (not around the newest decoded picture). Walking a long GOP dumps pictures in decode
/// order; a window around those would throw away the playhead frame and force another walk.
pin: i64,
/// Only background requests (thumbnails, [`filmcraft_media::cancel::with_background`]) have
/// used the decoder since it was made: it is given up as soon as another background request
/// is done, whatever the caches' recency.
@ -425,6 +442,57 @@ const MIN_FRAMES: usize = 64;
/// rather than restarting at the sync sample.
const CONTINUE_THROUGH: usize = 48;
/// Threads that run the caches' decoding (see [`isolated`]); `None` where none can be made.
#[cfg(not(target_arch = "wasm32"))]
fn decode_pool() -> Option<&'static rayon::ThreadPool> {
static POOL: std::sync::OnceLock<Option<rayon::ThreadPool>> = std::sync::OnceLock::new();
POOL.get_or_init(|| {
let n = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
rayon::ThreadPoolBuilder::new().num_threads(n).thread_name(|i| format!("decode-{i}")).build().ok()
})
.as_ref()
}
/// Run a cache's decoding, its lock held by the caller, on [`decode_pool`], the caller waiting
/// without doing other rayon work meanwhile (#480).
///
/// That decoding starts rayon work (a decoder's slices, cropping and rotating the frames). On the
/// pool of the render jobs that ask the cache for frames, a worker that took part of it could,
/// waiting inside it, pick up another of those jobs and block on the lock: the lock holder then
/// waits for its work on the blocked worker's stack, and that worker for the lock, for good (an
/// export of rotated iPhone clips stopped making frames). On the decode pool that work only ever
/// runs beside other caches' decoding, which never waits for a cache's lock. A caller that is a
/// rayon worker would also run other jobs of its pool while it waits for the decode pool, some
/// of them asking for a cache's lock while it holds this one: it waits on a plain thread instead.
fn isolated<T: Send>(f: impl FnOnce() -> crate::Result<T> + Send) -> crate::Result<T> {
let ctx = filmcraft_media::cancel::Context::current();
let job = move || ctx.run(f);
#[cfg(not(target_arch = "wasm32"))]
if let Some(pool) = decode_pool() {
if rayon::current_thread_index().is_none() {
return pool.install(job);
}
let slot = Mutex::new(Some(job));
let waited = std::thread::scope(|scope| {
let spawned = std::thread::Builder::new().name("decode-wait".into()).spawn_scoped(scope, || {
let job = slot.lock().unwrap_or_else(PoisonError::into_inner).take();
job.map(|job| pool.install(job))
});
spawned.map(|h| h.join())
});
return match waited {
Ok(Ok(Some(r))) => r,
Ok(Err(_)) => Err(CodecError::Decode("internal error while decoding".into())),
// no thread to wait on: decode here, as before
Ok(Ok(None)) | Err(_) => match slot.into_inner().unwrap_or_else(PoisonError::into_inner) {
Some(job) => job(),
None => Err(CodecError::Decode("internal error while decoding".into())),
},
};
}
job()
}
impl GopCache {
pub fn new(explicit_color: Option<ColorInfo>) -> Self {
// unit tests count decoder restarts: each of their caches gets a pool of its own
@ -445,6 +513,7 @@ impl GopCache {
drafts: Default::default(),
bytes: 0,
last_used: Instant::now(),
pin: i64::MIN,
background: false,
}),
pool: pool.clone(),
@ -482,7 +551,11 @@ impl GopCache {
}
fn store(&self, st: &mut State, pts: i64, f: VideoFrame, draft: bool) {
let f = self.finish(f);
self.store_finished(st, pts, self.finish(f), draft);
}
/// Cache a frame [`Self::finish`] has been applied to.
fn store_finished(&self, st: &mut State, pts: i64, f: VideoFrame, draft: bool) {
st.last_used = Instant::now();
if draft {
st.drafts.insert(pts);
@ -497,10 +570,19 @@ impl GopCache {
st.bytes -= old.byte_size();
filmcraft_frame::pool::recycle(old);
}
// evict frames far from the most recent (keep a window around the working position)
// Evict frames far from the request (pin), not the newest decode. A frame-threaded
// decoder can emit a burst past the playhead; pivoting on those pts used to drop the
// wanted picture and walk the GOP again.
let pivot = if st.pin != i64::MIN { st.pin } else { pts };
while st.bytes > budget && st.frames.len() > 2 {
let (Some(&first), Some(&last)) = (st.frames.keys().next(), st.frames.keys().next_back()) else { break };
let victim = if pts - first > last - pts { first } else { last };
let mut victim = if pivot - first > last - pivot { first } else { last };
if victim == pivot || victim == pts {
victim = if victim == first { last } else { first };
if victim == pivot || victim == pts {
break;
}
}
if let Some(v) = st.frames.remove(&victim) {
st.drafts.remove(&victim);
st.bytes -= v.byte_size();
@ -571,6 +653,7 @@ impl GopCache {
let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner());
let _decoding = DecodingGuard::enter(me);
st.last_used = Instant::now();
st.pin = want_pts;
let background = filmcraft_media::cancel::background();
if let Some(f) = st.cached(want_pts, draft).cloned() {
HITS.fetch_add(1, Ordering::Relaxed);
@ -604,12 +687,29 @@ impl GopCache {
if st.intra {
return self.intra_frame(st, s, i, want_pts);
}
let mut key = s.sync_before(i);
// with the lock held: see `isolated`
let st = &mut *st;
isolated(|| self.decode_until(st, s, i, want_pts, late_before, draft))
}
/// Decode (the lock held) until the frame with `want_pts` (sample `i`) is cached; return it.
fn decode_until(
&self,
st: &mut State,
s: &dyn VideoSamples,
i: usize,
want_pts: i64,
late_before: Option<i64>,
draft: bool,
) -> crate::Result<Arc<VideoFrame>> {
let n = s.count();
let mut key = decode_start(s, i);
// Continue the running decoder when it has passed the wanted sample's sync sample and
// either has not reached the sample yet or has been fed it without outputting it yet
// (a frame-threaded decoder holds many pictures in flight). Otherwise the frame was
// evicted or lies in another GOP: restart at the sync sample.
let running = st.next != usize::MAX && st.next <= n;
// evicted or lies in another GOP: restart at the sync sample. A run that started after
// `key` (at the random-access picture of a wanted leading picture) left the picture out.
let running = st.next != usize::MAX && st.next <= n && st.start <= key;
// Decoding on through a short stretch into the next GOP is cheaper than a restart, and
// playback wants those frames anyway.
let near = st.next <= key && key - st.next <= CONTINUE_THROUGH;
@ -656,14 +756,14 @@ impl GopCache {
let out = timed(|| decoder.decode(&data, s.pts(k)))?;
st.next += 1;
DECODED.fetch_add(1, Ordering::Relaxed);
self.store_output(&mut st, out);
self.store_output(st, out);
if st.cached(want_pts, draft).is_some() {
break;
}
}
if st.cached(want_pts, draft).is_none() {
let out = st.decoder.as_mut().map(|d| timed(|| d.flush())).unwrap_or_default();
self.store_output(&mut st, out);
self.store_output(st, out);
st.next = usize::MAX;
}
// nearest decoded frame at or before the wanted pts (robust to decoder pts quirks)
@ -694,7 +794,7 @@ impl GopCache {
want_pts: i64,
n: usize,
) -> crate::Result<Vec<crate::video::DecodedFrame>> {
let mut key = s.sync_before(i);
let mut key = decode_start(s, i);
// Continue the running decoder when the wanted sample is ahead within this GOP run.
let continuing = *next != usize::MAX && *next > key && *next <= i + 16 && *next <= n;
if !continuing {
@ -748,14 +848,17 @@ impl GopCache {
DECODED.fetch_add(1, Ordering::Relaxed);
timed(|| dec.decode(&data, want_pts))
});
// crop and rotate before taking the lock: that work runs on rayon (see `isolated`)
let res = res.map(|out| {
count_frames(&out);
out.into_iter().map(|d| (d.pts, self.finish(d.frame), d.draft)).collect::<Vec<_>>()
});
let mut st = self.shared.state.lock().unwrap_or_else(|e| e.into_inner());
if st.spare.len() < 16 {
st.spare.push(dec);
}
let res = res?;
count_frames(&res);
for d in res {
self.store(&mut st, d.pts, d.frame, d.draft);
for (pts, frame, draft) in res? {
self.store_finished(&mut st, pts, frame, draft);
}
st.at_or_before(want_pts, true).ok_or_else(|| CodecError::Decode("frame not produced".into()))
}
@ -887,6 +990,77 @@ mod tests {
assert!(s.decodes.load(Ordering::Relaxed) <= 60 + 24 + 1);
}
/// A decoder that, like a slice-threaded one, splits each picture's work over rayon.
struct ParDec(Box<dyn VideoDecoder>);
impl VideoDecoder for ParDec {
fn decode(&mut self, sample: &[u8], pts: i64) -> crate::Result<Vec<DecodedFrame>> {
use rayon::prelude::*;
let rows: u64 = (0..64u32)
.into_par_iter()
.with_max_len(1)
.map(|r| (0..20_000u64).fold(u64::from(r), |a, x| a.wrapping_mul(31).wrapping_add(x)))
.reduce(|| 0, u64::wrapping_add);
std::hint::black_box(rows);
self.0.decode(sample, pts)
}
fn flush(&mut self) -> Vec<DecodedFrame> {
self.0.flush()
}
fn reset(&mut self) {
self.0.reset()
}
fn name(&self) -> &str {
"par"
}
}
struct ParSamples(Samples);
impl VideoSamples for ParSamples {
fn count(&self) -> usize {
self.0.count()
}
fn pts(&self, i: usize) -> i64 {
self.0.pts(i)
}
fn sync_before(&self, i: usize) -> usize {
self.0.sync_before(i)
}
fn sample_at(&self, t: i64) -> Option<usize> {
self.0.sample_at(t)
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
self.0.read(i)
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
Ok(Box::new(ParDec(self.0.make_decoder()?)))
}
}
/// #480: render jobs on rayon asking one source for frames while its decoding (a decoder's
/// slices, rotating the frames) runs on rayon too. A worker that took part of that work and,
/// waiting inside it, picked up another frame request blocked on the cache's lock for good:
/// an export of rotated iPhone clips stopped making frames.
#[test]
fn rayon_frame_requests_while_decoding_on_rayon_never_deadlock() {
use rayon::prelude::*;
let s = Arc::new(ParSamples(samples(240, 30, 4, false)));
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for round in 0..40usize {
// export-like: many frame workers, frames of several GOPs of a fresh source at once
let c = GopCache::new(None).with_rotation(1);
(0..96usize).into_par_iter().with_max_len(1).for_each(|k| {
let i = (k * 37 + round * 53) % 240;
assert_eq!(index_of(&c.frame(&*s, i as i64 * 1000).expect("frame")), i);
});
}
let _ = tx.send(());
});
assert!(rx.recv_timeout(Duration::from_secs(120)).is_ok(), "frame requests deadlocked");
}
#[test]
fn running_decoder_continues_through_a_nearby_sync_sample() {
let s = samples(200, 30, 0, false);
@ -906,19 +1080,27 @@ mod tests {
fn intra_frames_decode_in_parallel_once_each() {
let s = Arc::new(samples(64, 1, 0, true));
let c = Arc::new(GopCache::new(None));
std::thread::scope(|scope| {
for t in 0..4usize {
let (s, c) = (s.clone(), c.clone());
scope.spawn(move || {
for k in 0..64usize {
let i = (k * 7 + t * 16) % 64;
assert_eq!(index_of(&c.frame(&*s, i as i64 * 1000).expect("frame")), i);
}
});
}
});
// a frame two threads miss at the same moment may decode twice; nothing more
assert!(s.decodes.load(Ordering::Relaxed) <= 64 + 4 * 4);
// First decode 64 distinct cold frames: each of the four workers owns
// 16 indices. Multiplication by seven permutes them without overlap.
// Then let every worker request all 64 cached frames in overlapping order.
// Concurrent misses for the same cold frame may legitimately decode twice,
// so they cannot support a scheduler-independent duplicate allowance.
for requests_per_worker in [16usize, 64] {
let start = Arc::new(std::sync::Barrier::new(4));
std::thread::scope(|scope| {
for t in 0..4usize {
let (s, c, start) = (s.clone(), c.clone(), start.clone());
scope.spawn(move || {
start.wait();
for k in 0..requests_per_worker {
let i = (k * 7 + t * 16) % 64;
assert_eq!(index_of(&c.frame(&*s, i as i64 * 1000).expect("frame")), i);
}
});
}
});
assert_eq!(s.decodes.load(Ordering::Relaxed), 64, "cold frames decode once; warm requests hit the cache");
}
assert_eq!(s.resets.load(Ordering::Relaxed), 0);
}
@ -1261,4 +1443,180 @@ mod tests {
}
assert!(reused() - before >= more as u64 - 2, "{} of {more} planes recycled", reused() - before);
}
/// An open-GOP stream like iPhone HEVC: 8 pictures a GOP; after the first, each GOP stores
/// its random-access picture (display 8·g) first, then two leading pictures shown before it
/// (8·g − 2, 8·g − 1, predicted from the GOP before), then the rest. pts = 1000 · display.
struct OpenGop {
/// (display index, sync) in decode order
order: Vec<(i64, bool)>,
resets: Arc<AtomicUsize>,
}
impl OpenGop {
fn new(gops: i64) -> Self {
let mut order: Vec<(i64, bool)> = (0..6).map(|d| (d, d == 0)).collect();
for g in 1..gops {
order.extend([(8 * g, true), (8 * g - 2, false), (8 * g - 1, false)]);
order.extend((8 * g + 1..8 * g + 6).map(|d| (d, false)));
}
Self { order, resets: Default::default() }
}
}
/// Leaves out the leading pictures of the random-access picture a run starts at (RASL
/// pictures, as the HEVC decoders do), outputs the others at once.
struct OpenGopDec {
run_start: Option<i64>,
resets: Arc<AtomicUsize>,
}
impl VideoDecoder for OpenGopDec {
fn decode(&mut self, _sample: &[u8], pts: i64) -> crate::Result<Vec<DecodedFrame>> {
let start = *self.run_start.get_or_insert(pts);
Ok(if pts < start { Vec::new() } else { vec![picture((pts, false))] })
}
fn flush(&mut self) -> Vec<DecodedFrame> {
Vec::new()
}
fn reset(&mut self) {
self.resets.fetch_add(1, Ordering::Relaxed);
self.run_start = None;
}
fn name(&self) -> &str {
"open gop"
}
}
impl VideoSamples for OpenGop {
fn count(&self) -> usize {
self.order.len()
}
fn pts(&self, i: usize) -> i64 {
self.order[i].0 * 1000
}
fn sync_before(&self, i: usize) -> usize {
(0..=i).rev().find(|&k| self.order[k].1).unwrap_or(0)
}
fn sample_at(&self, t: i64) -> Option<usize> {
self.order.iter().position(|&(d, _)| d == t / 1000)
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
Ok(vec![i as u8])
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
Ok(Box::new(OpenGopDec { run_start: None, resets: self.resets.clone() }))
}
}
/// A leading picture of an open GOP, asked for first (a clip trimmed to start on it, the
/// playhead parked on it), came out black: decoding started at the random-access picture
/// after it, which leaves it out.
#[test]
fn leading_pictures_of_an_open_gop_decode_from_the_gop_before() {
let s = OpenGop::new(6);
for d in [6usize, 7, 14, 15, 39] {
let c = GopCache::new(None);
assert_eq!(index_of(&c.frame(&s, d as i64 * 1000).expect("frame")), d);
}
// the random-access picture first, then its leading pictures: the run that started at
// it left them out, so they restart a GOP earlier
let c = GopCache::new(None);
assert_eq!(index_of(&c.frame(&s, 16_000).expect("frame")), 16);
assert_eq!(index_of(&c.frame(&s, 15_000).expect("frame")), 15);
assert_eq!(index_of(&c.frame(&s, 14_000).expect("frame")), 14);
// playing through: one seek, every picture in turn
let s = OpenGop::new(6);
let c = GopCache::new(None);
for d in 0..46usize {
assert_eq!(index_of(&c.frame(&s, d as i64 * 1000).expect("frame")), d);
}
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
}
/// Many pictures in one GOP; a decoder that emits a burst past the wanted sample (frame-
/// threaded H.264). Eviction used to pivot on the newest pts and drop the playhead frame.
struct Burst {
n: usize,
/// Pictures held before a decode dumps them all.
burst: usize,
held: VecDeque<i64>,
resets: Arc<AtomicUsize>,
decodes: Arc<AtomicUsize>,
}
impl VideoDecoder for Burst {
fn decode(&mut self, _sample: &[u8], pts: i64) -> crate::Result<Vec<DecodedFrame>> {
self.decodes.fetch_add(1, Ordering::Relaxed);
self.held.push_back(pts);
if self.held.len() < self.burst {
return Ok(Vec::new());
}
Ok(self.held.drain(..).map(|p| picture((p, false))).collect())
}
fn flush(&mut self) -> Vec<DecodedFrame> {
self.held.drain(..).map(|p| picture((p, false))).collect()
}
fn reset(&mut self) {
self.resets.fetch_add(1, Ordering::Relaxed);
self.held.clear();
}
fn name(&self) -> &str {
"burst"
}
}
impl VideoSamples for Burst {
fn count(&self) -> usize {
self.n
}
fn pts(&self, i: usize) -> i64 {
i as i64 * 1000
}
fn sync_before(&self, _i: usize) -> usize {
0
}
fn sample_at(&self, t: i64) -> Option<usize> {
let i = (t / 1000) as usize;
(i < self.n).then_some(i)
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
Ok(vec![i as u8])
}
fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
Ok(Box::new(Burst { n: self.n, burst: self.burst, held: VecDeque::new(), resets: self.resets.clone(), decodes: self.decodes.clone() }))
}
}
#[test]
fn long_gop_seek_keeps_the_target_and_does_not_walk_twice() {
let s = Burst { n: 200, burst: 90, held: VecDeque::new(), resets: Default::default(), decodes: Default::default() };
let mut c = GopCache::new(None);
c.budget = 0;
// Want a picture early in the GOP; the burst dumps 90 pictures at once (past MIN_FRAMES).
assert_eq!(index_of(&c.frame(&s, 10_000).expect("frame")), 10);
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
let decodes = s.decodes.load(Ordering::Relaxed);
assert!(decodes <= 90 + 1, "walked {decodes} samples");
// The wanted picture is still cached: asking again must not restart or re-feed.
assert_eq!(index_of(&c.frame(&s, 10_000).expect("cached")), 10);
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
assert_eq!(s.decodes.load(Ordering::Relaxed), decodes);
// Playback continues from the same run.
assert_eq!(index_of(&c.frame(&s, 11_000).expect("frame")), 11);
assert_eq!(s.resets.load(Ordering::Relaxed), 1);
}
#[test]
fn long_gop_continue_does_not_restart_for_the_next_frame() {
let s = samples(300, 250, 8, false);
let mut c = GopCache::new(None);
c.budget = 0;
assert_eq!(index_of(&c.frame(&s, 80_000).expect("frame")), 80);
let resets = s.resets.load(Ordering::Relaxed);
let decodes = s.decodes.load(Ordering::Relaxed);
assert_eq!(index_of(&c.frame(&s, 81_000).expect("frame")), 81);
assert_eq!(s.resets.load(Ordering::Relaxed), resets);
assert!(s.decodes.load(Ordering::Relaxed) - decodes <= 8 + 2);
}
}

4
crates/codecs/src/hw.rs
View File

@ -258,7 +258,7 @@ impl NalStreamInfo {
bit_depth_chroma: sps.bit_depth_chroma,
interlaced: !sps.frame_mbs_only,
profile_idc: sps.profile_idc,
color: vui_color(crop.2, crop.3, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
color: vui_color(crop.2, crop.3, vui.colour_primaries, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
par: sar_par(vui.sar),
// as the software decoder's DPB (`max_reorder`)
reorder: sps.max_num_reorder_frames().min(sps.max_dpb_frames()),
@ -304,7 +304,7 @@ impl NalStreamInfo {
bit_depth_chroma: sps.bit_depth_chroma,
interlaced: false,
profile_idc: sps.ptl.profile_idc,
color: vui_color(crop.2, crop.3, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
color: vui_color(crop.2, crop.3, vui.colour_primaries, vui.matrix_coefficients, vui.transfer_characteristics, vui.full_range),
par: sar_par(vui.sar),
reorder: sps.max_num_reorder as usize,
})

32
crates/codecs/src/lib.rs
View File

@ -13,6 +13,8 @@
//! - [`MpegSource`]: MPEG-2 transport streams (`.ts`, `.m2ts`, `.mts`), program streams (`.mpg`,
//! `.vob`, `.mod`) and MPEG-1/2 video elementary streams: MPEG-1/2, H.264 and HEVC video; MPEG
//! audio, AAC (ADTS / LATM), AC-3 and LPCM.
//! - [`AviSource`]: AVI (OpenDML included): Motion JPEG, H.264, HEVC and uncompressed video; PCM,
//! MP3 / MP2 and AC-3 audio.
//! - [`AudioFileSource`]: standalone compressed audio files (MP3, FLAC, AIFF, …).
//! - [`openers`]: the openers to register with the engine's media pool.
@ -20,6 +22,7 @@
pub mod apv;
pub mod audio;
pub mod avi;
pub mod gop;
pub mod hw;
mod hw_frame;
@ -36,6 +39,7 @@ use std::sync::{Arc, RwLock};
pub use apv::ApvSource;
pub use audio::AudioFileSource;
pub use avi::AviSource;
pub use gop::{FRAME_BUDGET, GopStats, cached_bytes, gop_stats, live_decoders};
pub use mkv::MkvSource;
pub use mp4::Mp4Source;
@ -70,9 +74,10 @@ impl From<CodecError> for filmcraft_media::MediaError {
}
/// Our own (pure-Rust) decoders, in the order they are tried.
const BUILTIN_FACTORIES: [VideoDecoderFactory; 9] = [
const BUILTIN_FACTORIES: [VideoDecoderFactory; 10] = [
video::h264_factory,
video::hevc_factory,
video::vp8_factory,
video::vp9_factory,
video::av1_factory,
video::apv_factory,
@ -125,12 +130,22 @@ pub fn software_video_decoder(entry: &filmcraft_isobmff::SampleEntry) -> Result<
/// Openers for the engine's media pool (MP4/MOV, Matroska/WebM, MXF, Ogg Opus/Vorbis, MPEG TS/PS and
/// MPEG-1/2 video elementary streams, APV raw bitstreams, standalone audio).
pub fn openers() -> Vec<filmcraft_media::Opener> {
vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, audio::opener]
vec![mp4::opener, mkv::opener, mxf::opener, ogg::opener, mpeg::opener, apv::opener, avi::opener, audio::opener]
}
fn reader_registry() -> &'static RwLock<Vec<filmcraft_media::ReaderOpener>> {
static R: std::sync::OnceLock<RwLock<Vec<filmcraft_media::ReaderOpener>>> = std::sync::OnceLock::new();
R.get_or_init(|| RwLock::new(vec![mp4::reader_opener, mkv::reader_opener, mxf::reader_opener, ogg::reader_opener, mpeg::reader_opener, apv::reader_opener]))
R.get_or_init(|| {
RwLock::new(vec![
mp4::reader_opener,
mkv::reader_opener,
mxf::reader_opener,
ogg::reader_opener,
mpeg::reader_opener,
apv::reader_opener,
avi::reader_opener,
])
})
}
/// Openers that read containers through a [`filmcraft_media::ByteReader`] (index now, samples on
@ -161,6 +176,15 @@ impl filmcraft_isobmff::ByteSource for Src {
}
}
impl filmcraft_avi::ByteSource for Src {
fn len(&self) -> u64 {
self.0.len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> {
self.0.read_at(offset, buf)
}
}
impl filmcraft_matroska::ByteSource for Src {
fn len(&self) -> u64 {
self.0.len()
@ -178,6 +202,8 @@ pub fn open_bytes(name: &str, bytes: Arc<[u8]>) -> std::result::Result<filmcraft
#[cfg(test)]
mod audio_timing_tests;
#[cfg(test)]
mod avi_tests;
#[cfg(test)]
mod rounded_pts_tests;
#[cfg(test)]
mod stream_color_tests;

92
crates/codecs/src/mkv.rs
View File

@ -58,7 +58,7 @@ pub struct MkvSource {
/// Video codec as an ISO-BMFF sample entry (for the decoder factories).
ventry: Option<SampleEntry>,
video: GopCache,
/// The alpha layer of a VP9 track with `AlphaMode` 1 (WebM transparency, #402).
/// The alpha layer of a VP8 / VP9 track with `AlphaMode` 1 (WebM transparency, #402).
alpha: Option<Arc<crate::webm_alpha::AlphaLayer>>,
/// The playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
audios: Vec<MkvAudio>,
@ -68,24 +68,24 @@ pub struct MkvSource {
///
/// Block timestamps are quantised to `TimestampScale` (usually 1 ms), so they are only accurate to a
/// tick: a 1024-sample AAC frame at 48 kHz lasts 21.33 ms. Packets that all decode to the same
/// length (AAC, MPEG audio, AC-3) therefore run on from one another, resynchronising to the
/// length (AAC, MPEG audio, AC-3, E-AC-3) therefore run on from one another, resynchronising to the
/// timestamp only across gaps of more than half a packet (and more than two ticks); see
/// [`crate::audio::contiguous_starts`]. Opus packets: [`opus_starts`]. Other codecs start at their
/// timestamps.
/// timestamps. Timestamps are taken with the exact `CodecDelay` off (see [`stamp_to_samples`]).
fn audio_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, rate: i64) -> Vec<i64> {
use crate::audio::{FixedFrames, PacketTime, contiguous_starts, fixed_packet_samples};
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
if let Some(h) = opus_head(&t.codec) {
return opus_starts(file, bytes, ti, &h, rate);
}
let (n, d) = tb(t);
let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64;
let at = stamp_to_samples(t, rate, codec_delay_samples(t, rate));
let first = || file.read_sample(bytes, ti, 0).unwrap_or_default();
let out_rate = u32::try_from(rate).unwrap_or(0);
let frame = match &t.codec {
Codec::Aac { asc } => fixed_packet_samples(FixedFrames::Aac(asc), &[], out_rate),
Codec::Mp3 | Codec::Mp2 => fixed_packet_samples(FixedFrames::MpegAudio, &first(), out_rate),
Codec::Ac3 => fixed_packet_samples(FixedFrames::Ac3, &first(), out_rate),
Codec::Eac3 => fixed_packet_samples(FixedFrames::Eac3, &first(), out_rate),
_ => None,
};
let Some(frame) = frame else {
@ -97,18 +97,13 @@ fn audio_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, rate: i64) -> Vec
/// Sample-exact Opus packet start positions (48 kHz frames, pre-skip removed).
///
/// The demuxer also subtracts a rounded `CodecDelay` from the timestamps. Starts are accumulated
/// from each packet's TOC duration, resynchronising to the timestamp only across real gaps (more
/// than two ticks off). Pre-skip is the exact `CodecDelay` (or the header's pre-skip when it is
/// absent).
/// Starts are accumulated from each packet's TOC duration, resynchronising to the timestamp only
/// across real gaps (more than two ticks off). Pre-skip is the exact `CodecDelay` (or the header's
/// pre-skip when it is absent).
fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_opus::OpusHead, rate: i64) -> Vec<i64> {
let Some(t) = file.tracks.get(ti) else { return Vec::new() };
let (n, d) = tb(t);
let delay_ns = t.codec_delay_ns as i128;
// The demuxer's rounding of CodecDelay to ticks, undone here.
let delay_ticks = codec_delay_ticks(t);
let skip = if t.codec_delay_ns > 0 { (delay_ns * rate as i128 / 1_000_000_000) as i64 } else { head.pre_skip as i64 };
let at = |pts: i64| ((pts.saturating_add(delay_ticks) as i128 * n as i128 * rate as i128 / d as i128) as i64).saturating_sub(skip);
let skip = if t.codec_delay_ns > 0 { codec_delay_samples(t, rate) } else { head.pre_skip as i64 };
let at = stamp_to_samples(t, rate, skip);
let packets = t.samples.iter().enumerate().map(|(i, s)| crate::audio::PacketTime {
stamp: at(s.pts),
stamped: own_stamp(s),
@ -117,6 +112,28 @@ fn opus_starts(file: &MkvFile, bytes: &crate::Src, ti: usize, head: &filmcraft_o
crate::audio::contiguous_starts(packets, stamp_tolerance(t, rate))
}
/// A block timestamp (with the demuxer's rounded `CodecDelay` taken off) as a position in sample
/// frames at `rate`, `skip` frames of priming removed (#790). The demuxer's rounding is undone first:
/// `CodecDelay` is rarely a whole number of ticks (1024 AAC samples at 48 kHz are 21.333 ms, 256
/// AC-3 samples at 44.1 kHz 5.805 ms), so starting the first packet at the rounded delay played the
/// whole track up to 16 samples late.
fn stamp_to_samples(t: &filmcraft_matroska::Track, rate: i64, skip: i64) -> impl Fn(i64) -> i64 {
let (n, d) = tb(t);
let delay_ticks = codec_delay_ticks(t);
move |pts: i64| {
let ticks = pts.saturating_add(delay_ticks) as i128;
let at = ticks.checked_mul(n as i128).and_then(|x| x.checked_mul(rate as i128)).map_or(ticks.signum() * i128::MAX, |x| x / d as i128);
i64::try_from(at).unwrap_or(if at < 0 { i64::MIN } else { i64::MAX }).saturating_sub(skip)
}
}
/// `CodecDelay` in sample frames at `rate`, rounded to the nearest (the element holds whole
/// nanoseconds: 1024 samples at 48 kHz are stored as 21 333 333 ns).
fn codec_delay_samples(t: &filmcraft_matroska::Track, rate: i64) -> i64 {
let ns = (t.codec_delay_ns as i128).saturating_mul(rate.max(0) as i128);
i64::try_from(ns.saturating_add(500_000_000) / 1_000_000_000).unwrap_or(i64::MAX)
}
/// `CodecDelay` in timestamp ticks, rounded as the demuxer rounds it (half away from zero) before
/// subtracting it from every timestamp of the track.
fn codec_delay_ticks(t: &filmcraft_matroska::Track) -> i64 {
@ -231,6 +248,7 @@ fn vp9_config(private: &[u8], v: Option<&filmcraft_matroska::VideoInfo>) -> VpcC
fn sample_entry(c: &Codec, private: &[u8], v: Option<&filmcraft_matroska::VideoInfo>, w: u16, h: u16) -> Option<SampleEntry> {
Some(match c {
Codec::Vp8 => SampleEntry::video(FourCc(*b"vp08"), CodecConfig::Unknown { fourcc: FourCc(*b"vp08"), raw: Vec::new() }, w, h),
Codec::Vp9 { private } => SampleEntry::video(FourCc(*b"vp09"), CodecConfig::Vp9(vp9_config(private, v)), w, h),
Codec::Av1 { av1c } => SampleEntry::video(FourCc(*b"av01"), CodecConfig::Av1(filmcraft_isobmff::Av1Config::parse(av1c).unwrap_or_default()), w, h),
Codec::Apv { apvc } => SampleEntry::apv(filmcraft_isobmff::ApvConfig::parse(apvc).unwrap_or_default(), w, h),
@ -357,7 +375,11 @@ impl MkvSource {
let rate = match t.default_duration_ns {
Some(ns) if ns > 0 => FrameRate::from_f64(1e9 / ns as f64),
_ => {
let mut d: Vec<i64> = t.samples.windows(2).take(240).map(|p| (p[1].pts - p[0].pts).abs()).filter(|d| *d > 0).collect();
// blocks are stored in decode order: with B-frames, neighbours in the file are
// up to a few frames apart, so step between presentation times (#923)
let mut pts: Vec<i64> = t.samples.iter().take(241).map(|s| s.pts).collect();
pts.sort_unstable();
let mut d: Vec<i64> = pts.windows(2).map(|p| p[1].saturating_sub(p[0])).filter(|d| *d > 0).collect();
d.sort_unstable();
let (n, dd) = tb(t);
let step = d.get(d.len() / 2).copied().unwrap_or(1).max(1) as f64 * n as f64 / dd as f64;
@ -428,9 +450,11 @@ impl MkvSource {
Some(MkvAudio { track: i, state, starts, preroll })
})
.collect();
// WebM transparency: VP9 alpha in each block's BlockAdditional (ID 1)
// WebM transparency: VP8 / VP9 alpha in each block's BlockAdditional (ID 1)
let alpha = vtrack.and_then(|i| file.tracks.get(i)).filter(|t| {
matches!(t.codec, Codec::Vp9 { .. }) && t.video.as_ref().is_some_and(|v| v.alpha_mode != 0) && t.samples.iter().any(|s| s.addition.is_some())
matches!(t.codec, Codec::Vp8 | Codec::Vp9 { .. })
&& t.video.as_ref().is_some_and(|v| v.alpha_mode != 0)
&& t.samples.iter().any(|s| s.addition.is_some())
});
let alpha = alpha.map(|t| Arc::new(crate::webm_alpha::AlphaLayer::new(bytes.clone(), t.samples.iter().map(|s| (s.pts, s.addition)))));
Ok(Self { info, bytes, file, vtrack, ventry, video: GopCache::new(explicit_color).with_rotation(rotation), alpha, audios })
@ -440,6 +464,21 @@ impl MkvSource {
self.file.read_sample(&self.bytes, track, i).map_err(|e| CodecError::Container(e.to_string()))
}
/// Whether sample `i` of a VP8 track is a hidden frame (`show_frame` 0, RFC 6386 9.1): only the
/// first byte of the frame is read.
fn vp8_hidden(&self, track: usize, i: usize) -> bool {
let Some(t) = self.file.tracks.get(track) else { return false };
let Some(s) = t.samples.get(i) else { return false };
let first = match t.frame_prefix(u64::from(s.size)).first() {
Some(&b) => Some(b),
None => {
let mut b = [0u8; 1];
(s.size > 0 && self.bytes.0.read_at(s.offset, &mut b).is_ok()).then_some(b[0])
}
};
first.is_some_and(|b| b & 0x10 == 0)
}
fn audio_decoder(&self, c: &Codec, rate: u32) -> crate::Result<PacketDecoder> {
use symphonia::core::codecs::{CODEC_TYPE_FLAC, CODEC_TYPE_MP3, CODEC_TYPE_VORBIS};
match c {
@ -447,7 +486,7 @@ impl MkvSource {
Codec::Opus { head } => PacketDecoder::opus(filmcraft_opus::OpusHead::parse(head).map_err(|e| CodecError::Unsupported(format!("Opus: {e}")))?),
Codec::Mp3 => PacketDecoder::new(CODEC_TYPE_MP3, rate, None),
Codec::Mp2 => PacketDecoder::mpeg_audio(2, rate),
Codec::Ac3 => PacketDecoder::ac3(),
Codec::Ac3 | Codec::Eac3 => PacketDecoder::ac3(),
// symphonia wants the STREAMINFO block body: skip `fLaC` + the 4-byte block header.
Codec::Flac { private } => PacketDecoder::new(CODEC_TYPE_FLAC, rate, private.get(8..42).map(<[u8]>::to_vec)),
Codec::Vorbis { headers } if headers.len() == 3 => {
@ -533,7 +572,20 @@ impl VideoSamples for MkvVideo<'_> {
self.src.file.tracks[self.track].sync_sample_before(i)
}
fn sample_at(&self, t: i64) -> Option<usize> {
self.src.file.tracks[self.track].sample_at_pts(t)
let track = &self.src.file.tracks[self.track];
let i = track.sample_at_pts(t)?;
if !matches!(track.codec, Codec::Vp8) {
return Some(i);
}
// VP8 has no superframes: an encoder stores a hidden (altref) frame as a block of its own
// with the time of the frame that follows it, which vpxenc moves 1 ms later. The hidden
// frame shows nothing, so its time asks for that next shown frame (as decoders that pass
// the two on as one packet show it).
let mut k = i;
while k + 1 < track.samples.len() && k < i + 8 && self.src.vp8_hidden(self.track, k) {
k += 1;
}
Some(k)
}
fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
self.src.read(self.track, i)

23
crates/codecs/src/mp4.rs
View File

@ -57,8 +57,9 @@ pub fn sniff(b: &[u8]) -> bool {
}
/// Audio packet start positions in source sample frames at `rate`: the running total of the sample
/// durations, except that packets which all decode to the same length (AAC, MPEG audio, AC-3) run
/// on from one another, resynchronising to the total only across gaps of more than half a packet.
/// durations, except that packets which all decode to the same length (AAC, MPEG audio, AC-3,
/// E-AC-3) run on from one another, resynchronising to the total only across gaps of more than half
/// a packet.
/// A remux from Matroska (OBS's) carries the millisecond rounding of its timestamps into the
/// durations: 1008, 1008, 1056… for 1024-sample AAC frames at 48 kHz (see
/// [`crate::audio::contiguous_starts`]).
@ -81,6 +82,7 @@ fn audio_starts(file: &Mp4File, bytes: &crate::Src, ti: usize, rate: u32) -> Vec
Some(CodecConfig::Aac(a)) => fixed_packet_samples(FixedFrames::Aac(&a.asc), &[], rate),
Some(CodecConfig::Mp3) => fixed_packet_samples(FixedFrames::MpegAudio, &first(), rate),
Some(CodecConfig::Ac3 { .. }) => fixed_packet_samples(FixedFrames::Ac3, &first(), rate),
Some(CodecConfig::Eac3 { .. }) => fixed_packet_samples(FixedFrames::Eac3, &first(), rate),
_ => None,
};
let Some(frame) = frame else { return stamps };
@ -538,14 +540,26 @@ fn audio_stream_info(t: &filmcraft_isobmff::Track, bytes: &crate::Src) -> Option
None => ap.sample_rate as u32,
}
},
if a.channel_config > 0 { a.channel_config as u32 } else { ap.channels },
// the configuration is an index (7 and 12 are both eight channels), not a count
if a.channels() > 0 { a.channels() } else { ap.channels },
None,
),
CodecConfig::Pcm(p) => (p.sample_rate as u32, p.channels, Some(p.bits as u32)),
CodecConfig::Opus(o) => (crate::audio::OPUS_RATE, (o.output_channels as u32).max(1), None),
_ => (if ap.sample_rate > 0.0 { ap.sample_rate as u32 } else { t.timescale }, ap.channels.max(1), None),
};
Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec: codec_label(&entry.codec), bits_per_sample: bits })
let codec = match &entry.codec {
// these entries name no layer (ffmpeg stores MP2 as `mp4a` + 0x6B): the first frame does
CodecConfig::Mp3 => {
let head = t.samples.first().and_then(|s| filmcraft_media::reader::read_range(&*bytes.0, s.offset, (s.size as usize).min(4)).ok());
match head.as_deref().and_then(crate::audio::mpeg_audio_layer) {
Some(1 | 2) => "MPEG Audio".into(),
_ => codec_label(&entry.codec),
}
}
c => codec_label(c),
};
Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec, bits_per_sample: bits })
}
fn codec_label(c: &CodecConfig) -> String {
@ -564,6 +578,7 @@ fn codec_label(c: &CodecConfig) -> String {
),
CodecConfig::Hevc(_) => "HEVC".into(),
CodecConfig::Vp9(c) => format!("VP9 (Profile {})", c.profile),
CodecConfig::Unknown { fourcc, .. } if fourcc.0 == *b"vp08" => "VP8".into(),
CodecConfig::Av1(c) => format!("AV1 ({} Profile)", ["Main", "High", "Professional"].get(c.seq_profile as usize).unwrap_or(&"Main")),
CodecConfig::Apv(a) => match filmcraft_apv::Profile::from_idc(a.profile_idc) {
Some(p) => p.name().into(),

189
crates/codecs/src/mpeg.rs
View File

@ -7,8 +7,9 @@
//! first random-access picture; display order comes from the PTS (or, without timestamps, the
//! GOP and temporal reference). Leading pictures of an open GOP decode from the previous GOP.
//! - Audio: MPEG audio layers I-III (symphonia), AAC in ADTS or LATM (our decoder), AC-3 (our
//! decoder), Blu-ray / AVCHD and DVD LPCM. Audio frames are placed on the timeline from their
//! PTS relative to the first video frame, sample-contiguously (re-anchored only at gaps).
//! decoder), Blu-ray / AVCHD and DVD LPCM. Every audio PID is a stream (capped). Audio frames
//! are placed on the timeline from their PTS relative to the first video frame,
//! sample-contiguously (re-anchored only at gaps).
use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, Mutex};
@ -104,7 +105,8 @@ pub struct MpegSource {
es: Option<(Arc<[u8]>, Vec<std::ops::Range<usize>>)>,
video: Option<VideoTrack>,
gop: GopCache,
audio: Option<AudioTrack>,
/// Playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
audios: Vec<AudioTrack>,
/// Why the video or audio stream cannot be decoded.
unsupported_video: Option<String>,
unsupported_audio: Option<String>,
@ -120,7 +122,7 @@ fn vkind(c: &Codec) -> Option<VKind> {
}
fn audio_supported(c: &Codec) -> bool {
matches!(c, Codec::MpegAudio | Codec::AacAdts | Codec::AacLatm | Codec::LpcmBluray | Codec::LpcmDvd | Codec::Ac3)
matches!(c, Codec::MpegAudio | Codec::AacAdts | Codec::AacLatm | Codec::LpcmBluray | Codec::LpcmDvd | Codec::Ac3 | Codec::Eac3)
}
/// The coded rate unless the timestamps say otherwise (3:2 pulldown, field-rate coding). Never
@ -201,8 +203,8 @@ impl MpegSource {
log::warn!("{name}: {w}");
}
let vstream = file.find(Kind::Video, |c| vkind(c).is_some()).or_else(|| file.find(Kind::Video, |_| true));
let astream = file.find(Kind::Audio, audio_supported).or_else(|| file.find(Kind::Audio, |_| true));
if vstream.is_none() && astream.is_none() {
let has_audio = file.streams.iter().any(|s| s.kind() == Kind::Audio && !s.units.is_empty());
if vstream.is_none() && !has_audio {
return Err(CodecError::Unsupported(format!("{}: no video or audio streams", file.format.name())));
}
let mut src = MpegSource {
@ -221,7 +223,7 @@ impl MpegSource {
es: None,
video: None,
gop: GopCache::new(None),
audio: None,
audios: Vec::new(),
unsupported_video: None,
unsupported_audio: None,
};
@ -256,7 +258,7 @@ impl MpegSource {
src.video = Some(track);
src.describe_video(rate_from_pts)?;
if let Some(file) = src.file.take() {
src.setup_audio(file, astream, origin);
src.setup_audio(file, origin);
}
}
None => {
@ -273,12 +275,12 @@ impl MpegSource {
bitrate: None,
hdr: None,
});
src.setup_audio(file, astream, None);
src.setup_audio(file, None);
}
}
} else {
src.info.kind = MediaKind::AudioOnly;
src.setup_audio(file, astream, None);
src.setup_audio(file, None);
}
Ok(src)
}
@ -326,7 +328,7 @@ impl MpegSource {
es: Some((all, aus)),
video: Some(track),
gop: GopCache::new(None),
audio: None,
audios: Vec::new(),
unsupported_video: None,
unsupported_audio: None,
};
@ -347,15 +349,15 @@ impl MpegSource {
/// Timeline position (samples at the audio stream's rate) of the first audio frame, and the
/// number of samples in the audio stream.
pub fn audio_extent(&self) -> Option<(i64, i64)> {
let a = self.audio.as_ref()?;
let a = self.audios.first()?;
let first = *a.starts.first()?;
let end = a.starts.last()? + *a.lens.last()? as i64;
Some((first, end - first))
}
/// The demuxed stream indices of the presented video and audio.
/// The demuxed stream indices of the presented video and first audio.
pub fn stream_indices(&self) -> (Option<usize>, Option<usize>) {
(self.video.as_ref().filter(|_| self.es.is_none()).map(|v| v.stream), self.audio.as_ref().map(|a| a.stream))
(self.video.as_ref().filter(|_| self.es.is_none()).map(|v| v.stream), self.audios.first().map(|a| a.stream))
}
/// Presentation timestamps (90 kHz) of the presented video frames in display order. Frames
@ -475,15 +477,31 @@ impl MpegSource {
Ok(())
}
/// Choose and index the audio stream; `origin`: PTS of the first video frame.
fn setup_audio(&mut self, file: File, astream: Option<usize>, origin: Option<i64>) {
let track = astream.and_then(|ai| self.audio_track(&file, ai, origin));
/// Index every audio stream the container carries (capped); `origin`: PTS of the first video frame.
fn setup_audio(&mut self, file: File, origin: Option<i64>) {
let indices: Vec<usize> = file
.streams
.iter()
.enumerate()
.filter(|(_, s)| s.kind() == Kind::Audio && !s.units.is_empty())
.map(|(i, _)| i)
.take(filmcraft_media::MAX_AUDIO_STREAMS)
.collect();
let mut tracks = Vec::new();
for ai in indices {
if let Some(t) = self.audio_track(&file, ai, origin) {
tracks.push(t);
}
}
self.file = Some(file);
self.audio = track;
self.audios = tracks;
}
/// Mark the audio stream as not decodable (it still shows in the media info).
/// Mark the audio stream as not decodable (it still shows in the media info when nothing plays).
fn audio_unsupported(&mut self, codec_name: &str, why: String) -> Option<AudioTrack> {
if !self.info.audio_streams.is_empty() {
return None;
}
self.unsupported_audio = Some(why);
self.info.audio_streams =
vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None }];
@ -505,7 +523,7 @@ impl MpegSource {
};
let mut latm = None;
let (codec, rate, channels, per_frame, bits) = match &st.codec {
Codec::MpegAudio | Codec::AacAdts | Codec::Ac3 => {
Codec::MpegAudio | Codec::AacAdts | Codec::Ac3 | Codec::Eac3 => {
let Some(fi) = filmcraft_mpegts::frame_info(&st.codec, &first) else {
return self.audio_unsupported(&codec_name, format!("{codec_name}: bad frame header"));
};
@ -594,14 +612,16 @@ impl MpegSource {
next = Some(s + n as i64);
}
let total = next.unwrap_or(0).max(0);
if self.info.video.is_none() || self.unsupported_video.is_some() {
if self.unsupported_audio.take().is_some() {
self.info.audio_streams.clear();
}
if self.info.audio_streams.is_empty() && (self.info.video.is_none() || self.unsupported_video.is_some()) {
self.info.duration = Tick::from_units(total, rate.max(1) as i64);
if self.info.video.is_none() {
self.info.kind = MediaKind::AudioOnly;
}
}
self.info.audio_streams =
vec![AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits }];
self.info.audio_streams.push(AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits });
Some(AudioTrack {
stream: ai,
codec,
@ -783,10 +803,16 @@ impl MediaSource for MpegSource {
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
if let Some(why) = &self.unsupported_audio {
self.audio_stream(0, start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
if self.audios.is_empty()
&& let Some(why) = &self.unsupported_audio
{
return Err(MediaError::Unsupported(why.clone()));
}
let a = self.audio.as_ref().ok_or(MediaError::NoStream("audio"))?;
let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
let ch = a.channels;
let ratio = a.rate as f64 / sample_rate as f64;
let exact = a.rate == sample_rate;
@ -852,6 +878,7 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedRe
#[cfg(test)]
mod tests {
use super::*;
use filmcraft_media::MediaSource;
/// A mutated transport stream whose PTS span gave ~0 fps panicked on open with "attempt to
/// divide by zero" (`FrameRate::tick_of`).
@ -868,6 +895,118 @@ mod tests {
assert_eq!(stream_rate(None, Some(50.0)), FrameRate::from_f64(50.0));
}
/// MPEG-1 layer II frame, 48 kHz 192 kb/s (576 bytes).
fn mp2_frame(k: u8) -> Vec<u8> {
let mut v = vec![0xFF, 0xFD, 0xA4, 0x00];
v.extend((0..572).map(|i| (i as u8) ^ k));
v
}
fn ts_bytes(prefix: u8, v: i64) -> [u8; 5] {
let v = v & ((1 << 33) - 1);
[(prefix << 4) | (((v >> 30) & 7) as u8) << 1 | 1, (v >> 22) as u8, (((v >> 15) & 0x7F) as u8) << 1 | 1, (v >> 7) as u8, ((v & 0x7F) as u8) << 1 | 1]
}
fn pes(stream_id: u8, pts: Option<i64>, payload: &[u8]) -> Vec<u8> {
let mut h = Vec::new();
if let Some(p) = pts {
h.extend(ts_bytes(2, p));
}
let flags = if pts.is_some() { 0x80 } else { 0 };
let len = 3 + h.len() + payload.len();
let mut v = vec![0, 0, 1, stream_id, (len >> 8) as u8, len as u8, 0x80, flags, h.len() as u8];
v.extend(h);
v.extend_from_slice(payload);
v
}
/// Tiny TS muxer: PAT, PMT and PES packets in 188-byte packets.
struct TsMux {
out: Vec<u8>,
cc: std::collections::HashMap<u16, u8>,
}
impl TsMux {
fn packet(&mut self, pid: u16, pusi: bool, payload: &[u8]) -> usize {
let cc = self.cc.entry(pid).or_insert(0);
let mut p = vec![0x47, (pusi as u8) << 6 | (pid >> 8) as u8, pid as u8];
let n = payload.len().min(184);
if n < 184 {
let al = 183 - n;
p.push(0x30 | *cc);
p.push(al as u8);
if al > 0 {
p.push(0);
p.extend(std::iter::repeat_n(0xFF, al - 1));
}
} else {
p.push(0x10 | *cc);
}
*cc = (*cc + 1) & 15;
p.extend_from_slice(&payload[..n]);
self.out.extend(&p);
n
}
fn section(&mut self, pid: u16, body: &[u8]) {
let mut s = body.to_vec();
s.extend(crc32_mpeg(&s).to_be_bytes());
let mut payload = vec![0];
payload.extend(s);
self.packet(pid, true, &payload);
}
fn psi(&mut self, streams: &[(u8, u16)]) {
let pat = [0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00];
self.section(0, &pat);
let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x00, 0xF0, 0];
for &(st, pid) in streams {
pmt.extend([st, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]);
}
let len = pmt.len() - 3 + 4;
pmt[1] = 0xB0 | (len >> 8) as u8;
pmt[2] = len as u8;
self.section(0x100, &pmt);
}
fn pes(&mut self, pid: u16, data: &[u8]) {
let mut rest = data;
let mut first = true;
while !rest.is_empty() {
let n = self.packet(pid, first, rest);
rest = &rest[n..];
first = false;
}
}
}
fn crc32_mpeg(data: &[u8]) -> u32 {
let mut crc = 0xFFFF_FFFFu32;
for &b in data {
crc ^= (b as u32) << 24;
for _ in 0..8 {
crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
}
}
crc
}
/// Dual MPEG audio in one transport stream: import used to keep only the first PID.
#[test]
fn every_audio_pid_is_a_stream() {
let mut mux = TsMux { out: Vec::new(), cc: Default::default() };
mux.psi(&[(0x03, 0x1100), (0x03, 0x1101)]);
for k in 0..4u16 {
mux.pes(0x1100, &pes(0xC0, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8)));
mux.pes(0x1101, &pes(0xC1, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8 + 16)));
}
let source = MpegSource::open("dual.ts", mux.out.into()).unwrap();
assert_eq!(source.info().audio_streams.len(), 2, "{:?}", source.info().audio_streams);
assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::<Vec<_>>(), vec![48_000, 48_000]);
let a0 = source.audio(0, 64, 48_000).unwrap();
let a1 = source.audio_stream(1, 0, 64, 48_000).unwrap();
assert_eq!(a0.channels.len(), 2);
assert_eq!(a1.channels.len(), 2);
assert!(source.audio_stream(2, 0, 64, 48_000).is_err());
}
#[test]
fn presentation_drops_leading_pictures_and_orders_by_key() {
// decode order I(2) B(0) B(1) P(5) B(3) B(4): the leading B pictures are not presented

2
crates/codecs/src/stream_color.rs
View File

@ -42,6 +42,8 @@ pub fn from_codec_config(codec: &CodecConfig) -> Option<ColorCodes> {
CodecConfig::Avc(c) => avc(c),
CodecConfig::Hevc(c) => hevc(c),
CodecConfig::Av1(c) => av1(&c.config_obus),
// VP8 has no colour signalling: its one colour space is BT.601 YUV, studio swing (RFC 6386 9.2)
CodecConfig::Unknown { fourcc, .. } if fourcc.0 == *b"vp08" => Some(ColorCodes { primaries: 2, transfer: 2, matrix: 6, full_range: Some(false) }),
_ => None,
}
}

164
crates/codecs/src/tests.rs
View File

@ -7,7 +7,7 @@ use std::sync::Arc;
use filmcraft_media::FrameRequest;
use filmcraft_time::{TICKS_PER_SECOND, Tick};
fn ffmpeg() -> Option<&'static str> {
pub(crate) fn ffmpeg() -> Option<&'static str> {
["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].into_iter().find(|p| std::path::Path::new(p).exists())
}
@ -224,20 +224,63 @@ fn matroska_hevc_flac() {
check_mkv("blue_hevc_flac.mkv", &["-c:v", "libx265", "-x265-params", "log-level=error", "-pix_fmt", "yuv420p"], &["-c:a", "flac"]);
}
/// FLAC in MP4 (`fLaC` + `dfLa`): the decoder gets the STREAMINFO from `dfLa` (it was given
/// nothing, and every FLAC track in an MP4 failed with "missing extra data").
#[test]
fn mp4_flac_audio() {
let args = ["-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=48000:d=2", "-c:a", "flac", "-strict", "-2"];
let Some(b) = fixture("tone_flac.mp4", &args) else { return };
let src = crate::open_bytes("tone_flac.mp4", b).expect("open");
let a = src.audio(24_000, 4800, 48_000).expect("audio");
let peak = a.channels[0].iter().fold(0f32, |m, s| m.max(s.abs()));
assert!(peak > 0.05 && peak < 1.0, "audio peak {peak}");
}
#[test]
fn matroska_prores_pcm() {
check_mkv("blue_prores_pcm.mkv", &["-c:v", "prores_ks", "-profile:v", "2"], &["-c:a", "pcm_s16le"]);
}
/// `bytes` with its first `DefaultDuration` element (0x23E383) overwritten by an EBML Void of the
/// same length, as a muxer that leaves the optional element out would write the track.
fn without_default_duration(bytes: &[u8]) -> Vec<u8> {
let mut out = bytes.to_vec();
let at = out.windows(3).position(|w| w == [0x23, 0xE3, 0x83]).expect("DefaultDuration");
// ffmpeg writes a one-byte size (0x80 | n)
let total = 4 + (out[at + 3] & 0x7F) as usize;
out[at] = 0xEC;
out[at + 1] = 0x80 | (total - 2) as u8;
out[at + 2..at + total].fill(0);
out
}
#[test]
fn matroska_reordered_video_without_default_duration_keeps_its_rate() {
// blocks are in decode order, so B-frames put file neighbours 2–3 frames apart (#923)
for (bf, name) in [(0, "rate25_bf0.mkv"), (1, "rate25_bf1.mkv"), (2, "rate25_bf2.mkv")] {
let params = format!("bframes={bf}:b-adapt=0:b-pyramid=0:keyint=250:scenecut=0");
let args =
["-f", "lavfi", "-i", "testsrc2=size=64x48:rate=25", "-frames:v", "25", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-x264-params", &params, "-an"];
let Some(control) = fixture(name, &args) else { return };
let stripped = without_default_duration(&control);
for (label, bytes) in [("with DefaultDuration", control.to_vec()), ("without", stripped)] {
let src = crate::open_bytes(name, bytes.into()).expect("open");
let rate = src.info().video.as_ref().expect("video").frame_rate;
assert_eq!((rate.num, rate.den), (25, 1), "{bf} B-frames, {label}");
assert_eq!(src.info().duration, Tick::from_rational(1, 1, 1), "{bf} B-frames, {label}");
}
}
}
/// Path of a fixture under `target/fixtures/codecs/`, generated with ffmpeg when missing
/// (`None` when ffmpeg or the encoder is unavailable).
fn fixture_path(name: &str, args: &[&str]) -> Option<PathBuf> {
pub(crate) fn fixture_path(name: &str, args: &[&str]) -> Option<PathBuf> {
fixture(name, args)?;
Some(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/codecs").join(name))
}
/// ffmpeg's decode of `src` as raw planar video (the oracle).
fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option<Vec<u8>> {
pub(crate) fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option<Vec<u8>> {
let name = format!("{}.{pix_fmt}.yuv", src.file_name()?.to_str()?);
let path = src.to_str()?;
let b = fixture(&name, &["-i", path, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", pix_fmt])?;
@ -245,7 +288,7 @@ fn reference_yuv(src: &std::path::Path, pix_fmt: &str) -> Option<Vec<u8>> {
}
/// Planes of a decoded YUV frame as little-endian bytes (ffmpeg rawvideo layout).
fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec<u8> {
pub(crate) fn yuv_bytes(f: &filmcraft_frame::VideoFrame) -> Vec<u8> {
match &f.data {
filmcraft_frame::PixelData::Yuv8 { planes, .. } => planes.iter().flat_map(|p| p.iter().copied()).collect(),
filmcraft_frame::PixelData::Yuv16 { planes, .. } => planes.iter().flat_map(|p| p.iter().flat_map(|s| s.to_le_bytes())).collect(),
@ -449,6 +492,119 @@ fn vp9_webm_alpha_layer_matches_libvpx() {
}
}
/// #432: VP8 in WebM with hidden altref frames (vpxenc, two-pass with ARNR: 11 of its 161
/// frames are not shown). Frames requested out of order (seeks back and forth, then sequential
/// playback across the hidden frames) are sample-exact against libvpx's decode, and the stream's
/// colour is BT.601 (RFC 6386 9.2). Skipped without ffmpeg or vpxenc.
#[test]
fn vp8_webm_random_access_matches_libvpx() {
let frames = 150usize;
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/codecs");
let path = dir.join("vp8_hidden_altref.webm");
if !path.exists() {
let Some(y4m) =
fixture_path("mandelbrot_352x288_150.y4m", &["-f", "lavfi", "-i", "mandelbrot=s=352x288:r=25", "-frames:v", "150", "-pix_fmt", "yuv420p"])
else {
return;
};
let tmp = dir.join(format!("vp8_hidden_altref.{}.webm", std::process::id()));
let enc = Command::new("vpxenc")
.args(["--codec=vp8", "--passes=2", "--auto-alt-ref=1", "--lag-in-frames=25", "--arnr-maxframes=7", "--arnr-strength=5"])
.args(["--target-bitrate=300", "--kf-max-dist=150", "-q", "-o"])
.arg(&tmp)
.arg(&y4m)
.status();
if !enc.is_ok_and(|s| s.success()) {
eprintln!("vpxenc not available; skipping");
return;
}
let _ = std::fs::rename(&tmp, &path);
}
let p = path.to_string_lossy().into_owned();
let Some(reference) =
fixture("vp8_hidden_altref.webm.yuv", &["-c:v", "libvpx", "-i", &p, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "yuv420p"])
else {
return;
};
let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into();
let src = crate::open_bytes("vp8_hidden_altref.webm", bytes).unwrap();
let info = src.info().clone();
assert_eq!(info.container, "WebM");
let v = info.video.as_ref().unwrap();
assert_eq!(v.codec, "VP8");
assert_eq!((v.width, v.height), (352, 288));
assert_eq!(v.color.matrix, filmcraft_color::Matrix::Bt601);
assert_eq!(reference.len(), frames * 352 * 288 * 3 / 2);
let frame_len = reference.len() / frames;
let rate = info.frame_rate();
let order: Vec<usize> = [147, 3, 90, 26, 24, 149, 0, 112, 61].into_iter().chain(40..frames).collect();
for k in order {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
assert_eq!((f.width, f.height), (352, 288));
assert_eq!(f.color.matrix, filmcraft_color::Matrix::Bt601);
assert!(yuv_bytes(&f) == reference[k * frame_len..(k + 1) * frame_len], "frame {k} differs from libvpx");
}
}
/// #432 / #402: a VP8 WebM with an alpha layer (`AlphaMode` 1, a second VP8 stream in each
/// block's `BlockAdditional`, as browsers and GIF converters write transparent WebM) decodes
/// with that alpha; frames requested out of order match libvpx exactly, colour and alpha.
#[test]
fn vp8_webm_alpha_layer_matches_libvpx() {
let (w, h, frames) = (176usize, 144usize, 30usize);
let Some(path) = fixture_path(
"alpha_vp8.webm",
&[
"-f",
"lavfi",
"-i",
"testsrc2=s=176x144:r=25,format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='mod(X*2+N*7\\,256)'",
"-frames:v",
"30",
"-c:v",
"libvpx",
"-pix_fmt",
"yuva420p",
"-g",
"10",
"-auto-alt-ref",
"0",
"-b:v",
"600k",
],
) else {
return;
};
let src_path = path.to_string_lossy().into_owned();
let Some(reference) =
fixture("alpha_vp8.webm.yuva420p.yuv", &["-c:v", "libvpx", "-i", &src_path, "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "yuva420p"])
else {
return;
};
let (luma, chroma) = (w * h, (w / 2) * (h / 2));
let frame_len = luma * 2 + chroma * 2;
if reference.len() != frame_len * frames {
eprintln!("libvpx reference has {} bytes, not {}; skipping", reference.len(), frame_len * frames);
return;
}
let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into();
let src = crate::open_bytes("alpha_vp8.webm", bytes).unwrap();
let info = src.info().clone();
assert_eq!(info.container, "WebM");
assert!(info.video.as_ref().unwrap().has_alpha);
let rate = info.frame_rate();
let order: Vec<usize> = [29, 3, 17, 12, 0, 25, 21, 9].into_iter().chain(10..frames).collect();
for k in order {
let f = src.video_frame(FrameRequest::full(rate.tick_of(k as i64))).unwrap();
let r = &reference[k * frame_len..(k + 1) * frame_len];
let filmcraft_frame::PixelData::Yuv8 { planes, alpha, .. } = &f.data else { panic!("frame {k}: not 8-bit YUV: {}", f.format_label()) };
assert_eq!(planes[0].as_slice(), &r[..luma], "frame {k}: luma");
assert_eq!(planes[1].as_slice(), &r[luma..luma + chroma], "frame {k}: U");
let alpha = alpha.as_ref().unwrap_or_else(|| panic!("frame {k}: no alpha plane"));
assert_eq!(alpha.as_slice(), &r[luma + 2 * chroma..], "frame {k}: alpha");
}
}
/// Every sample flagged as sync (as in an MP4 without `stss`): the GOP cache must still start
/// decoding at a real VP9 key frame.
#[test]

315
crates/codecs/src/video.rs
View File

@ -140,11 +140,14 @@ pub(crate) fn tight_plane<T: Copy>(src: Vec<T>, stride: usize, w: usize, h: usiz
}
/// Colour of an H.264 / HEVC picture from its VUI code points (ITU-T H.273), as the decoders
/// report it: BT.709 by default with the matrix guessed from the size, the signalled matrix and
/// transfer when known, and full range when flagged. Shared by the software and hardware
/// decoders so the two are interchangeable.
pub fn vui_color(width: u32, height: u32, matrix: u8, transfer: u8, full_range: bool) -> filmcraft_color::ColorInfo {
/// report it: BT.709 by default with the matrix guessed from the size, the signalled primaries,
/// matrix and transfer when known, and full range when flagged. Shared by the software and
/// hardware decoders so the two are interchangeable.
pub fn vui_color(width: u32, height: u32, primaries: u8, matrix: u8, transfer: u8, full_range: bool) -> filmcraft_color::ColorInfo {
let mut color = filmcraft_color::ColorInfo { matrix: filmcraft_frame::default_matrix(width, height), ..filmcraft_color::ColorInfo::REC709 };
if let Some(p) = primaries_from_code(primaries) {
color.primaries = p;
}
if let Some(m) = filmcraft_color::Matrix::from_code(matrix) {
color.matrix = m;
}
@ -178,13 +181,69 @@ pub fn hvcc_length_size_and_tid(hvcc: &[u8]) -> (usize, Option<u8>) {
/// A factory returns `None` when it does not handle the entry.
pub type VideoDecoderFactory = fn(&SampleEntry) -> Option<Result<Box<dyn VideoDecoder>>>;
/// The typical Huffman tables of ITU-T T.81 Annex K.3 as one DHT segment (luminance and
/// chrominance, DC and AC): what a Motion JPEG frame without its own tables is coded with (AVI's
/// "MJPG", the AVI1 convention of capture cards and cameras).
const DEFAULT_DHT: &[u8] = &[
0xFF, 0xC4, 0x01, 0xA2, //
0x00, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, //
0x01, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, //
0x10, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7D, //
0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1, 0x08, //
0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, //
0x29, 0x2A, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, //
0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, //
0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, //
0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xE1, 0xE2, //
0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, //
0x11, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77, //
0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, //
0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62, 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25, 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, //
0x27, 0x28, 0x29, 0x2A, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, //
0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, //
0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, //
0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, //
0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA,
];
/// A JPEG with the [`DEFAULT_DHT`] inserted before its first scan when it defines no Huffman
/// table of its own (else unchanged, borrowed).
pub fn with_default_huffman(jpeg: &[u8]) -> std::borrow::Cow<'_, [u8]> {
use std::borrow::Cow;
if jpeg.get(..2) != Some(&[0xFF, 0xD8]) {
return Cow::Borrowed(jpeg);
}
// walk the marker segments up to the start of scan
let mut at = 2usize;
while let (Some(&0xFF), Some(&m)) = (jpeg.get(at), jpeg.get(at + 1)) {
match m {
0xFF => at += 1, // fill byte
0xC4 => return Cow::Borrowed(jpeg),
0xDA => {
let mut out = Vec::with_capacity(jpeg.len() + DEFAULT_DHT.len());
out.extend_from_slice(jpeg.get(..at).unwrap_or_default());
out.extend_from_slice(DEFAULT_DHT);
out.extend_from_slice(jpeg.get(at..).unwrap_or_default());
return Cow::Owned(out);
}
0xD0..=0xD9 | 0x01 => at += 2,
_ => {
let Some(len) = jpeg.get(at + 2..at + 4).map(|b| usize::from(u16::from_be_bytes([b[0], b[1]]))) else { break };
at += 2 + len;
}
}
}
Cow::Borrowed(jpeg)
}
/// Motion-JPEG / Photo-JPEG (each sample is a complete JPEG).
pub struct MjpegDecoder;
impl VideoDecoder for MjpegDecoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let sample = with_default_huffman(sample);
// mjpa samples may contain two fields; decode the first image (field-merging lands with interlace support).
let img = image::load_from_memory_with_format(sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?;
let img = image::load_from_memory_with_format(&sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?;
let rgba = img.to_rgba8();
let (w, h) = rgba.dimensions();
Ok(vec![DecodedFrame { pts, frame: VideoFrame::rgba8(w, h, rgba.into_raw()), draft: false }])
@ -224,7 +283,7 @@ impl H264Decoder {
fn make_inner(avcc: &[u8], threads: usize) -> Result<filmcraft_h264::Decoder> {
let mut dec = filmcraft_h264::Decoder::with_threads(threads);
if !avcc.is_empty() {
dec.configure_avcc(avcc).map_err(|e| CodecError::Decode(e.to_string()))?;
dec.configure_avcc(avcc).map_err(h264_error)?;
}
Ok(dec)
}
@ -250,7 +309,7 @@ impl H264Decoder {
let y = tight_plane(p.y, p.y_stride, w, h);
let u = tight_plane(p.u, p.uv_stride, cw, ch);
let v = tight_plane(p.v, p.uv_stride, cw, ch);
let color = vui_color(p.width, p.height, p.color.matrix, p.color.transfer, p.color.full_range);
let color = vui_color(p.width, p.height, p.color.primaries, p.color.matrix, p.color.transfer, p.color.full_range);
let par = sar_par(p.sar);
let frame = VideoFrame {
width: p.width,
@ -264,9 +323,19 @@ impl H264Decoder {
}
}
/// An H.264 decoder error as a [`CodecError`]: a feature the decoder does not implement (High
/// 4:2:2, field pictures...) is [`CodecError::Unsupported`], so the media pool shows the
/// unreadable slate instead of a black picture (#626); anything else is a decode error.
fn h264_error(e: filmcraft_h264::Error) -> CodecError {
match e {
filmcraft_h264::Error::Unsupported(what) => CodecError::Unsupported(format!("H.264: {what}")),
e => CodecError::Decode(e.to_string()),
}
}
impl VideoDecoder for H264Decoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let pics = self.dec.decode(sample, pts).map_err(|e| CodecError::Decode(e.to_string()))?;
let pics = self.dec.decode(sample, pts).map_err(h264_error)?;
Ok(pics.into_iter().map(Self::convert).collect())
}
fn flush(&mut self) -> Vec<DecodedFrame> {
@ -328,7 +397,7 @@ pub struct HevcDecoder {
impl HevcDecoder {
pub fn new(hvcc: Vec<u8>) -> Result<Self> {
let dec = filmcraft_hevc::Decoder::from_hvcc(&hvcc).map_err(|e| CodecError::Decode(e.to_string()))?;
let dec = filmcraft_hevc::Decoder::from_hvcc(&hvcc).map_err(hevc_error)?;
let (length_size, highest_tid) = hvcc_length_size_and_tid(&hvcc);
Ok(Self { hvcc, dec, length_size, highest_tid })
}
@ -361,16 +430,24 @@ impl HevcDecoder {
alpha: None,
},
};
let color = vui_color(p.width, p.height, p.color.matrix, p.color.transfer, p.color.full_range);
let color = vui_color(p.width, p.height, p.color.primaries, p.color.matrix, p.color.transfer, p.color.full_range);
let par = sar_par(p.sar);
let frame = VideoFrame { width: p.width, height: p.height, data, color, par, pts: filmcraft_time::Tick::ZERO };
DecodedFrame { pts: p.pts, frame, draft: p.draft }
}
}
/// An HEVC decoder error as a [`CodecError`] (see [`h264_error`]).
fn hevc_error(e: filmcraft_hevc::Error) -> CodecError {
match e {
filmcraft_hevc::Error::Unsupported(what) => CodecError::Unsupported(format!("HEVC: {what}")),
e => CodecError::Decode(e.to_string()),
}
}
impl VideoDecoder for HevcDecoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let pics = self.dec.decode(sample, pts).map_err(|e| CodecError::Decode(e.to_string()))?;
let pics = self.dec.decode(sample, pts).map_err(hevc_error)?;
Ok(pics.into_iter().map(Self::convert).collect())
}
fn flush(&mut self) -> Vec<DecodedFrame> {
@ -529,6 +606,60 @@ pub fn vp9_factory(e: &SampleEntry) -> Option<Result<Box<dyn VideoDecoder>>> {
}
}
/// Our pure-Rust VP8 decoder (RFC 6386; #432): WebM / Matroska `V_VP8` and MP4 `vp08`.
pub struct Vp8Decoder {
dec: filmcraft_vp8::Decoder,
}
impl Default for Vp8Decoder {
fn default() -> Self {
Self::new()
}
}
impl Vp8Decoder {
pub fn new() -> Self {
Self { dec: filmcraft_vp8::Decoder::new() }
}
fn convert(p: filmcraft_vp8::Picture) -> DecodedFrame {
use filmcraft_frame::{Chroma, PixelData};
use std::sync::Arc;
// color_space 0 is BT.601 YUV (RFC 6386 9.2), studio swing; a container's Colour wins
let color = filmcraft_color::ColorInfo { matrix: filmcraft_color::Matrix::Bt601, ..filmcraft_color::ColorInfo::REC709 };
let data = PixelData::Yuv8 { planes: [Arc::new(p.y), Arc::new(p.u), Arc::new(p.v)], chroma: Chroma::C420, alpha: None };
let frame = VideoFrame { width: p.width, height: p.height, data, color, par: (1, 1), pts: filmcraft_time::Tick::ZERO };
DecodedFrame { pts: p.pts, frame, draft: false }
}
}
impl VideoDecoder for Vp8Decoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
let pic = self.dec.decode(sample, pts).map_err(|e| CodecError::Decode(e.to_string()))?;
Ok(pic.into_iter().map(Self::convert).collect())
}
fn flush(&mut self) -> Vec<DecodedFrame> {
// no reordering: every picture comes out with its frame
Vec::new()
}
fn reset(&mut self) {
self.dec.reset();
}
fn name(&self) -> &str {
"FilmCraft VP8"
}
fn is_random_access(&self, sample: &[u8]) -> Option<bool> {
Some(filmcraft_vp8::is_keyframe(sample))
}
fn is_disposable(&self, sample: &[u8]) -> bool {
self.dec.is_disposable(sample)
}
}
pub fn vp8_factory(e: &SampleEntry) -> Option<Result<Box<dyn VideoDecoder>>> {
(e.format.0 == *b"vp08").then(|| Ok(Box::new(Vp8Decoder::new()) as Box<dyn VideoDecoder>))
}
/// Our AV1 decoder. The `av1C` configuration OBUs (sequence header) are fed before the first
/// sample and again after every reset.
pub struct Av1Decoder {
@ -1017,3 +1148,165 @@ mod disposable_tests {
assert!(!hevc_disposable(&sample(&[&nal(0, 1)]), 4, None));
}
}
/// Uncompressed video layouts (AVI `BI_RGB` and YUV FourCCs).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RawFormat {
/// 24-bit B, G, R; rows padded to 4 bytes.
Bgr24,
/// 32-bit B, G, R, X.
Bgrx32,
/// Packed 4:2:2: Y0 U Y1 V (`YUY2`, `YUYV`).
Yuy2,
/// Packed 4:2:2: U Y0 V Y1 (`UYVY`, `2vuy`).
Uyvy,
/// Planar 4:2:0: Y, U, V (`I420`, `IYUV`).
I420,
/// Planar 4:2:0: Y, V, U (`YV12`).
Yv12,
/// Y plane, interleaved U V at half resolution (`NV12`).
Nv12,
/// Luma only (`Y800`, `GREY`, `Y8 `).
Gray8,
}
impl RawFormat {
/// The layout of an AVI video stream: `BI_RGB` with its bit count, or a YUV FourCC.
pub fn from_avi(compression: [u8; 4], bit_count: u16) -> Option<RawFormat> {
if matches!(u32::from_le_bytes(compression), 0 | 3) {
return match bit_count {
24 => Some(RawFormat::Bgr24),
32 => Some(RawFormat::Bgrx32),
_ => None,
};
}
let mut upper = compression;
upper.make_ascii_uppercase();
Some(match &upper {
b"YUY2" | b"YUYV" | b"YUNV" | b"V422" => RawFormat::Yuy2,
b"UYVY" | b"2VUY" | b"UYNV" | b"HDYC" => RawFormat::Uyvy,
b"I420" | b"IYUV" => RawFormat::I420,
b"YV12" => RawFormat::Yv12,
b"NV12" => RawFormat::Nv12,
b"Y800" | b"GREY" | b"GRAY" | b"Y8 " => RawFormat::Gray8,
_ => return None,
})
}
pub fn label(self) -> &'static str {
match self {
RawFormat::Bgr24 => "Uncompressed RGB 24-bit",
RawFormat::Bgrx32 => "Uncompressed RGB 32-bit",
RawFormat::Yuy2 | RawFormat::Uyvy => "Uncompressed YUV 4:2:2",
RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 => "Uncompressed YUV 4:2:0",
RawFormat::Gray8 => "Uncompressed greyscale",
}
}
}
/// Uncompressed frames (each sample one picture of a [`RawFormat`]).
pub struct RawVideoDecoder {
format: RawFormat,
width: u32,
height: u32,
/// RGB rows are stored bottom-up (a positive `BITMAPINFOHEADER` height).
bottom_up: bool,
}
impl RawVideoDecoder {
pub fn new(format: RawFormat, width: u32, height: u32, bottom_up: bool) -> Result<Self> {
if width == 0 || height == 0 || width > 16_384 || height > 16_384 {
return Err(CodecError::Unsupported(format!("{width}x{height} uncompressed video")));
}
Ok(Self { format, width, height, bottom_up })
}
fn yuv(&self, planes: [Vec<u8>; 3], chroma: filmcraft_frame::Chroma) -> VideoFrame {
use std::sync::Arc;
let [y, u, v] = planes;
VideoFrame {
width: self.width,
height: self.height,
data: filmcraft_frame::PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma, alpha: None },
color: vui_color(self.width, self.height, 2, 2, 2, false),
par: (1, 1),
pts: filmcraft_time::Tick::ZERO,
}
}
fn picture(&self, s: &[u8]) -> Result<VideoFrame> {
let (w, h) = (self.width as usize, self.height as usize);
let short = || CodecError::Decode(format!("{} frame of {} bytes is too short for {w}x{h}", self.format.label(), s.len()));
let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
Ok(match self.format {
RawFormat::Bgr24 | RawFormat::Bgrx32 => {
let bpp = if self.format == RawFormat::Bgr24 { 3 } else { 4 };
let stride = (w * bpp).div_ceil(4) * 4;
if s.len() < stride * h {
return Err(short());
}
let mut rgba = Vec::with_capacity(w * h * 4);
for row in 0..h {
let src = if self.bottom_up { h - 1 - row } else { row };
for px in s.get(src * stride..src * stride + w * bpp).unwrap_or_default().chunks_exact(bpp) {
rgba.extend_from_slice(&[px[2], px[1], px[0], 255]);
}
}
VideoFrame::rgba8(self.width, self.height, rgba)
}
RawFormat::Yuy2 | RawFormat::Uyvy => {
let stride = cw * 4;
if s.len() < stride * h {
return Err(short());
}
let (yo, uo, vo, y2) = if self.format == RawFormat::Yuy2 { (0, 1, 3, 2) } else { (1, 0, 2, 3) };
let (mut y, mut u, mut v) = (Vec::with_capacity(w * h), Vec::with_capacity(cw * h), Vec::with_capacity(cw * h));
for row in s.chunks_exact(stride).take(h) {
for (k, q) in row.as_chunks::<4>().0.iter().enumerate() {
y.push(q[yo]);
if 2 * k + 1 < w {
y.push(q[y2]);
}
u.push(q[uo]);
v.push(q[vo]);
}
}
self.yuv([y, u, v], filmcraft_frame::Chroma::C422)
}
RawFormat::I420 | RawFormat::Yv12 | RawFormat::Nv12 | RawFormat::Gray8 => {
let luma = w * h;
let need = if self.format == RawFormat::Gray8 { luma } else { luma + 2 * cw * ch };
if s.len() < need {
return Err(short());
}
let y = s.get(..luma).unwrap_or_default().to_vec();
let (u, v) = match self.format {
RawFormat::Gray8 => (vec![128; cw * ch], vec![128; cw * ch]),
RawFormat::Nv12 => s.get(luma..luma + 2 * cw * ch).unwrap_or_default().as_chunks::<2>().0.iter().map(|p| (p[0], p[1])).unzip(),
_ => {
let a = s.get(luma..luma + cw * ch).unwrap_or_default().to_vec();
let b = s.get(luma + cw * ch..luma + 2 * cw * ch).unwrap_or_default().to_vec();
if self.format == RawFormat::Yv12 { (b, a) } else { (a, b) }
}
};
self.yuv([y, u, v], filmcraft_frame::Chroma::C420)
}
})
}
}
impl VideoDecoder for RawVideoDecoder {
fn decode(&mut self, sample: &[u8], pts: i64) -> Result<Vec<DecodedFrame>> {
Ok(vec![DecodedFrame { pts, frame: self.picture(sample)?, draft: false }])
}
fn flush(&mut self) -> Vec<DecodedFrame> {
Vec::new()
}
fn reset(&mut self) {}
fn name(&self) -> &str {
self.format.label()
}
fn intra_only(&self) -> bool {
true
}
}

11
crates/codecs/src/webm_alpha.rs
View File

@ -1,9 +1,10 @@
//! WebM alpha (#402): a VP9 track with `AlphaMode` 1 carries, in each block's `BlockAdditional`
//! with `BlockAddID` 1, a second VP9 stream coded independently of the colour one. Its luma plane,
//! WebM alpha (#402): a VP8 or VP9 track with `AlphaMode` 1 carries, in each block's
//! `BlockAdditional` with `BlockAddID` 1, a second stream of the same codec coded independently of
//! the colour one. Its luma plane,
//! full range, is the frame's alpha (the WebM convention, as libvpx-based decoders read it).
//!
//! [`AlphaDecoder`] wraps the colour decoder: every sample it decodes, it also decodes that
//! sample's alpha frame with a second (software) VP9 decoder and attaches the luma plane to the
//! sample's alpha frame with a second (software) decoder and attaches the luma plane to the
//! colour picture with the same timestamp, holding a colour picture back until its alpha is out. The alpha stream has key frames of its own, which need
//! not coincide with the colour ones: after a seek (or any other break in the order samples are
//! fed), the alpha decoder restarts from the nearest alpha key frame at or before the sample.
@ -21,7 +22,7 @@ use crate::video::{DecodedFrame, VideoDecoder};
/// How far back the alpha decoder looks for a key frame when it resynchronises.
const MAX_RESYNC: usize = 600;
/// Decoded alpha pictures waiting for their colour picture, and colour pictures waiting for their
/// alpha (VP9 pictures come out in order, so a few are plenty; past that a picture goes out opaque
/// alpha (VP8 / VP9 pictures come out in order, so a few are plenty; past that a picture goes out opaque
/// rather than stall).
const MAX_PENDING: usize = 8;
@ -106,7 +107,7 @@ impl AlphaDecoder {
self.alpha.reset();
self.pending.clear();
self.alpha_max = None;
let is_key = |k: usize| self.layer.read(k).is_some_and(|d| filmcraft_vp9::is_keyframe(&d));
let is_key = |k: usize| self.layer.read(k).is_some_and(|d| self.alpha.is_random_access(&d) == Some(true));
let start = (i.saturating_sub(MAX_RESYNC)..=i).rev().find(|&k| is_key(k));
if let Some(start) = start {
for k in start..i {

108
crates/codecs/tests/aac_channels_oracle.rs Normal file
View File

@ -0,0 +1,108 @@
//! Multichannel AAC in MP4 against ffmpeg (#922). The AudioSpecificConfig's channel configuration
//! is an index into ISO/IEC 14496-3 Table 1.19 (with Amd 4), not a channel count: 7 and 12 are
//! both 7.1 (eight channels) and 11 is 6.1 (seven). Each channel carries its own tone, so a
//! dropped or swapped channel shows up against ffmpeg's decode.
mod common;
use std::io::Cursor;
use common::*;
use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
const RATE: u32 = 48_000;
/// A one-second AAC MP4 of `layout` (`n` channels), channel `i` a tone of 200 + 100·i Hz, except
/// the LFE (fourth in all these layouts) at 60 Hz: AAC band-limits it.
fn tones(ff: &std::path::Path, name: &str, layout: &str, n: usize) -> Option<std::path::PathBuf> {
let mut graph: String =
(0..n).map(|i| format!("sine=frequency={}:sample_rate={RATE}:duration=1[a{i}];", if i == 3 { 60 } else { 200 + 100 * i })).collect();
graph.extend((0..n).map(|i| format!("[a{i}]")));
graph.push_str(&format!("join=inputs={n}:channel_layout={layout}[out]"));
fixture(ff, name, &["-filter_complex", &graph, "-map", "[out]", "-c:a", "aac", "-b:a", "512k"])
}
/// The AAC access units of `path`'s audio track, rewrapped in a new MP4 whose AudioSpecificConfig
/// says `channel_config` (AAC LC, 48 kHz), keeping each sample's duration and the priming edit.
fn rewrap(path: &std::path::Path, name: &str, channel_config: u8, channels: u32) -> std::path::PathBuf {
let data = bytes(path);
let file = filmcraft_isobmff::open(&data).unwrap();
let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio).unwrap();
// object type 2, rate index 3 (48 kHz), then the configuration
let asc = (((2u16 << 11) | (3 << 7) | ((channel_config as u16) << 3)).to_be_bytes()).to_vec();
let mut mux = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap();
let id = mux.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).unwrap();
for s in &track.samples {
let au = &data[s.offset as usize..s.offset as usize + s.size as usize];
mux.write_sample(id, WriteSample { data: au, duration: s.duration, composition_offset: 0, is_sync: true }).unwrap();
}
let out = dir().join(name);
std::fs::write(&out, mux.finish().unwrap().into_inner()).unwrap();
out
}
/// Our decode of the whole file, channel by channel.
fn ours(path: &std::path::Path, name: &str) -> (u32, Vec<Vec<f32>>) {
let src = filmcraft_codecs::open_bytes(name, bytes(path)).unwrap();
let ch = src.info().audio().map(|a| a.channels).unwrap();
(ch, src.audio(0, RATE as usize * 9 / 10, RATE).unwrap().channels)
}
/// ffmpeg's decode, deinterleaved into `n` channels.
fn theirs(ff: &std::path::Path, path: &std::path::Path, n: usize) -> Vec<Vec<f32>> {
let all = ffmpeg_audio_f32(ff, path, &[]);
(0..n).map(|c| all.iter().skip(c).step_by(n).copied().collect()).collect()
}
/// Largest sample difference over the first `len` samples, best over a small lag either way (the
/// two decoders may place the priming differently by a few samples).
fn diff(a: &[f32], b: &[f32]) -> f32 {
let len = a.len().min(b.len()).saturating_sub(64);
(0..=32).flat_map(|l| [(l, 0), (0, l)]).map(|(x, y)| (32..len).map(|i| (a[i + x] - b[i + y]).abs()).fold(0.0, f32::max)).fold(f32::MAX, f32::min)
}
fn peak(x: &[f32]) -> f32 {
x.iter().fold(0.0, |m, v| m.max(v.abs()))
}
#[test]
fn aac_channel_configurations_report_and_decode_every_channel() {
let ff = filmcraft_testkit::require_ffmpeg!();
// ffmpeg 8 writes 7.1 as configuration 7 and ffmpeg 9 as 12: the same five elements
let Some(seven) = tones(&ff, "aac_71.mp4", "7.1", 8) else {
eprintln!("SKIPPED: this ffmpeg can't encode 7.1 AAC");
return;
};
let asc = filmcraft_isobmff::open(bytes(&seven)).unwrap().tracks.iter().find_map(|t| match &t.entries.first()?.codec {
filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.channel_config),
_ => None,
});
let twelve = match asc {
Some(7) => rewrap(&seven, "aac_71_cfg12.mp4", 12, 8),
Some(12) => seven.clone(),
other => panic!("ffmpeg wrote 7.1 AAC as configuration {other:?}"),
};
let seven = if asc == Some(7) { seven } else { rewrap(&seven, "aac_71_cfg7.mp4", 7, 8) };
// configuration 7: eight channels reported and returned, the last one too
let (ch, got) = ours(&seven, "aac_71_cfg7.mp4");
assert_eq!((ch, got.len()), (8, 8), "configuration 7");
assert!(got.iter().all(|c| peak(c) > 0.05), "configuration 7: a silent channel");
// configuration 12 (7.1 back): decoded, in ffmpeg's (and WAV's) L R C LFE Lb Rb Ls Rs order
let (ch, got) = ours(&twelve, "aac_71_cfg12.mp4");
assert_eq!((ch, got.len()), (8, 8), "configuration 12");
let want = theirs(&ff, &twelve, 8);
for (c, (a, b)) in got.iter().zip(&want).enumerate() {
let d = diff(a, b);
println!("configuration 12, channel {c}: peak {:.3}, max |ours - ffmpeg| {d:.5}", peak(a));
assert!(peak(a) > 0.05 && d < 0.01, "configuration 12, channel {c}: off ffmpeg's decode by {d}");
}
// controls: 5.1 (configuration 6) and 6.1 (ffmpeg writes a PCE)
for (layout, n, name) in [("5.1", 6, "aac_51.mp4"), ("6.1", 7, "aac_61.mp4")] {
let path = tones(&ff, name, layout, n).unwrap();
let (ch, got) = ours(&path, name);
assert_eq!((ch as usize, got.len()), (n, n), "{layout}");
}
}

85
crates/codecs/tests/eac3_oracle.rs Normal file
View File

@ -0,0 +1,85 @@
//! E-AC-3 (Dolby Digital Plus) audio in MP4, Matroska and MPEG-TS against ffmpeg: ffmpeg encodes
//! and muxes the files and decodes them as the oracle. Zero-bit mantissas carry decoder-specific
//! dither (A/52 §7.3.4), so the criterion is SNR against ffmpeg's decode, sample-aligned.
mod common;
use common::*;
const STEREO: &str = "aevalsrc=exprs=0.3*sin(2*PI*441*t)+0.1*sin(2*PI*3100*t)|0.25*sin(2*PI*660*t)+0.1*sin(2*PI*(500+4000*t)*t):s=48000";
const SURROUND: &str =
"aevalsrc=exprs=0.3*sin(2*PI*300*t)|0.3*sin(2*PI*400*t)|0.3*sin(2*PI*500*t)|0.3*sin(2*PI*60*t)|0.2*sin(2*PI*700*t)|0.2*sin(2*PI*800*t):s=48000:c=5.1";
/// Worst channel SNR (dB) of `ours` (planar) against ffmpeg's interleaved `want`, with ours
/// shifted by `lag` samples, over the samples both cover past the first 1000.
fn snr(ours: &[Vec<f32>], want: &[f32], ch: usize, lag: i64) -> f64 {
let n = want.len() / ch;
(0..ch)
.map(|c| {
let (mut sig, mut err) = (0f64, 0f64);
for i in 1000..n - 1000 {
let Some(&a) = ours[c].get((i as i64 + lag) as usize) else { continue };
let (a, b) = (a as f64, want[i * ch + c] as f64);
sig += b * b;
err += (a - b) * (a - b);
}
10.0 * (sig / err.max(1e-30)).log10()
})
.fold(f64::INFINITY, f64::min)
}
#[test]
fn eac3_in_mp4_matroska_and_transport_streams() {
let ff = filmcraft_testkit::require_ffmpeg!();
let cases: [(&str, &str, u32, &[&str]); 4] = [
("eac3_51.mp4", SURROUND, 6, &["-b:a", "384k"]),
("eac3_51.mkv", SURROUND, 6, &["-b:a", "384k"]),
("eac3_stereo.mov", STEREO, 2, &["-b:a", "192k"]),
("eac3_stereo.ts", STEREO, 2, &["-b:a", "192k"]),
];
for (name, input, channels, enc) in cases {
let args: Vec<&str> = ["-f", "lavfi", "-i", input, "-t", "2", "-c:a", "eac3"].into_iter().chain(enc.iter().copied()).collect();
let f = fixture(&ff, name, &args).unwrap_or_else(|| panic!("could not write {name}"));
let src = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!((a.sample_rate, a.channels), (48_000, channels), "{name}: {}", a.codec);
let want = ffmpeg_audio_f32(&ff, &f, &[]);
let ch = channels as usize;
let n = want.len() / ch;
// MPEG-TS audio starts at its first PTS relative to the presentation start
let start = match name.ends_with(".ts") {
true => filmcraft_codecs::MpegSource::open(name, bytes(&f)).unwrap().audio_extent().map_or(0, |e| e.0),
false => 0,
};
let ours = src.audio(start, n, 48_000).unwrap().channels;
assert_eq!(ours.len(), ch, "{name}");
// Matroska stores the encoder's 256-sample priming as CodecDelay, which we trim exactly
// (checked to the sample against the encoded signal in `matroska_audio_oracle`). Whether
// ffmpeg's CLI trims it too depends on its version (9.x does; 2023-24 builds keep the
// priming samples), so for .mkv its decode may start 256 samples earlier than ours.
// Elsewhere the samples line up exactly, whatever the version.
let lags: Vec<i64> = match name.ends_with(".mkv") {
true => (-32..=32).chain(-256 - 32..=-256 + 32).collect(),
false => (-32..=32).collect(),
};
let (worst, lag) = lags.into_iter().map(|lag| (snr(&ours, &want, ch, lag), lag)).fold((f64::MIN, 0), |a, b| if b.0 > a.0 { b } else { a });
let ok = if name.ends_with(".mkv") { lag == 0 || lag == -256 } else { lag == 0 };
assert!(ok, "{name}: decode is {lag} samples off ffmpeg's");
// a window read on a fresh source (random access) gives the same samples
let fresh = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap();
let at = n / 3;
let win = fresh.audio(start + at as i64, 4800, 48_000).unwrap().channels;
let mut seek_err = 0f32;
for c in 0..ch {
for (k, v) in win[c].iter().enumerate() {
seek_err = seek_err.max((v - ours[c][at + k]).abs());
}
}
println!(
"{name}: {} ({} Hz, {ch} ch), worst channel SNR vs ffmpeg {worst:.1} dB (lag {lag}), random access max diff {seek_err:.1e}",
a.codec, a.sample_rate
);
assert!(worst >= 50.0, "{name}: SNR {worst:.1} dB");
assert!(seek_err <= 1e-6, "{name}: random-access audio differs by {seek_err}");
}
}

61
crates/codecs/tests/frame_color_oracle.rs Normal file
View File

@ -0,0 +1,61 @@
//! Decoded H.264 / HEVC frames carry the colour primaries their VUI signals (#924). ffmpeg writes
//! MP4 without a `colr` box unless asked, so for most files the bitstream is the only place the
//! primaries are described: stream info already read them, the decoded frames reported BT.709.
mod common;
use common::*;
use filmcraft_color::{ColorInfo, ColorSpace, Matrix, Primaries, Range, Transfer};
use filmcraft_media::FrameRequest;
use filmcraft_time::Tick;
/// One 64×64 frame of `codec` with the colour `params` (x264 / x265 syntax), and our stream and
/// frame colour for it.
fn colors(ff: &std::path::Path, name: &str, codec: &[&str]) -> Option<(ColorInfo, ColorInfo)> {
let mut args = vec!["-f", "lavfi", "-i", "testsrc2=size=64x64:rate=1", "-frames:v", "1"];
args.extend_from_slice(codec);
let path = fixture(ff, name, &args)?;
let isobmff = filmcraft_isobmff::open(bytes(&path)).unwrap();
let track = isobmff.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).unwrap();
assert!(track.entries[0].video.as_ref().unwrap().color.is_none(), "{name}: the fixture has a colr box");
let src = filmcraft_codecs::open_bytes(name, bytes(&path)).unwrap();
let stream = src.info().video.as_ref().unwrap().color;
let frame = src.video_frame(FrameRequest::full(Tick::ZERO)).unwrap().color;
Some((stream, frame))
}
#[test]
fn decoded_frames_keep_the_signalled_primaries() {
let ff = filmcraft_testkit::require_ffmpeg!();
let x264 = |p: &'static str| ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-x264-params", p];
let x265 = |p: &'static str, pix: &'static str| ["-c:v", "libx265", "-pix_fmt", pix, "-tag:v", "hvc1", "-x265-params", p];
let sdr2020 = ColorInfo { matrix: Matrix::Bt709, transfer: Transfer::Bt709, primaries: Primaries::Bt2020, range: Range::Limited };
let cases: [(&str, Vec<&str>, ColorInfo, ColorSpace); 4] = [
("prim2020_h264.mp4", x264("colorprim=bt2020:transfer=bt709:colormatrix=bt709").to_vec(), sdr2020, ColorSpace::Rec2020),
("prim2020_hevc.mp4", x265("log-level=error:colorprim=bt2020:transfer=bt709:colormatrix=bt709", "yuv420p").to_vec(), sdr2020, ColorSpace::Rec2020),
// control: BT.709 throughout
(
"prim709_h264.mp4",
x264("colorprim=bt709:transfer=bt709:colormatrix=bt709").to_vec(),
ColorInfo { primaries: Primaries::Bt709, ..sdr2020 },
ColorSpace::Rec709,
),
// HDR keeps its transfer and now its primaries too
(
"prim2020_pq_hevc.mp4",
x265("log-level=error:colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc", "yuv420p10le").to_vec(),
ColorInfo { matrix: Matrix::Bt2020Ncl, transfer: Transfer::Pq, primaries: Primaries::Bt2020, range: Range::Limited },
ColorSpace::Rec2100Pq,
),
];
for (name, codec, want, space) in cases {
let Some((stream, frame)) = colors(&ff, name, &codec) else {
eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
continue;
};
println!("{name}: stream {stream:?}, frame {frame:?}");
assert_eq!(stream, want, "{name}: stream");
assert_eq!(frame, want, "{name}: decoded frame");
assert_eq!(ColorSpace::from_info(&frame), space, "{name}");
}
}

97
crates/codecs/tests/matroska_audio_oracle.rs
View File

@ -48,3 +48,100 @@ fn matroska_audio_plays_without_holes() {
assert!(r_ours < 2.0 * r_theirs + 0.005, "{name}: clicks between packets (|Δ²| {r_ours} vs ffmpeg's {r_theirs})");
}
}
/// Mean |difference| between `ours[i]` and `theirs[i + lag]` over `from..from + len`.
fn mean_err(ours: &[f32], theirs: &[f32], from: usize, len: usize, lag: i64) -> f64 {
(from..from + len).map(|i| (ours[i] as f64 - theirs[(i as i64 + lag) as usize] as f64).abs()).sum::<f64>() / len as f64
}
/// The lag in `lags` at which `ours` best matches `theirs` (least [`mean_err`]), and that error.
fn best_lag(ours: &[f32], theirs: &[f32], from: usize, len: usize, lags: std::ops::RangeInclusive<i64>) -> (i64, f64) {
lags.map(|lag| (lag, mean_err(ours, theirs, from, len, lag))).min_by(|a, b| a.1.total_cmp(&b.1)).unwrap()
}
/// A reference signal we control: low-passed white noise from a fixed xorshift seed. Its
/// autocorrelation dies out within a few samples, so even a one-sample misalignment shows.
fn noise(n: usize) -> Vec<f32> {
let (mut state, mut y) = (0x9E37_79B9_7F4A_7C15u64, 0f32);
(0..n)
.map(|_| {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
let white = (state >> 40) as f32 / (1u64 << 24) as f32 - 0.5;
y = 0.5 * y + 0.4 * white;
y
})
.collect()
}
/// #790: ffmpeg stores the encoder priming as `CodecDelay` (1024 samples = 21.333 ms for AAC at
/// 48 kHz, 256 = 5.805 ms for AC-3 at 44.1 kHz, 1105 = 25.057 ms for MP3), and block timestamps to
/// the millisecond. Taking the rounded delay as the start of the first packet played the whole
/// track up to 16 samples late.
///
/// The strict check is version-independent: ffmpeg encodes a signal we generate, and our decode
/// must line up with that signal to the sample (the stream's `CodecDelay` trimmed exactly). ffmpeg's
/// own decode is compared only after a bounded best-lag search: how its CLI handles Matroska
/// priming depends on the version (9.x trims `CodecDelay`, 2023-24 builds keep the priming
/// samples), so its lag is reported, not asserted.
#[test]
fn matroska_audio_is_sample_aligned_with_ffmpeg() {
let ff = filmcraft_testkit::require_ffmpeg!();
let cases: [(&str, u32, &[&str]); 6] = [
("ref_aac_48k.mkv", 48_000, &["-c:a", "aac", "-b:a", "256k"]),
("ref_aac_44k.mkv", 44_100, &["-c:a", "aac", "-b:a", "256k"]),
("ref_ac3_44k.mkv", 44_100, &["-c:a", "ac3", "-b:a", "448k"]),
("ref_ac3_48k.mkv", 48_000, &["-c:a", "ac3", "-b:a", "448k"]),
("ref_eac3_48k.mkv", 48_000, &["-c:a", "eac3", "-b:a", "384k"]),
("ref_mp3_44k.mkv", 44_100, &["-c:a", "libmp3lame", "-b:a", "320k"]),
];
for (name, rate, codec) in cases {
let reference = noise(rate as usize * 2);
let raw = format!("ref_noise_{rate}.f32");
let raw_path = filmcraft_testkit::fixtures::generate(&dir().join(&raw), |tmp| {
std::fs::write(tmp, reference.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>()).is_ok()
})
.unwrap();
let (raw_arg, rate_arg) = (raw_path.to_str().unwrap().to_string(), rate.to_string());
let args: Vec<&str> = ["-f", "f32le", "-ar", &rate_arg, "-ac", "1", "-i", &raw_arg].iter().copied().chain(codec.iter().copied()).collect();
let Some(path) = fixture(&ff, name, &args) else {
eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
continue;
};
let file = bytes(&path);
let mkv = filmcraft_matroska::open(file.clone()).unwrap();
let delay_ns = mkv.tracks.first().map_or(0, |t| t.codec_delay_ns) as i64;
let delay = (delay_ns * rate as i64 + 500_000_000) / 1_000_000_000;
assert!(delay > 0, "{name}: no CodecDelay");
let src = filmcraft_codecs::open_bytes(name, file).unwrap();
let frames = rate as usize * 3 / 2;
let mut ours = Vec::with_capacity(frames);
while ours.len() < frames {
let n = 1000.min(frames - ours.len());
ours.extend_from_slice(&src.audio(ours.len() as i64, n, rate).unwrap().channels[0]);
}
let (from, len, bound) = (rate as usize / 4, rate as usize / 4, delay + 32);
// strict: the encoder's input, to the sample
let (lag, err) = best_lag(&ours, &reference, from, len, -bound..=bound);
let level = reference[from..from + len].iter().map(|v| v.abs() as f64).sum::<f64>() / len as f64;
assert_eq!(lag, 0, "{name}: our decode is {lag} samples off the encoded signal (CodecDelay {delay} samples)");
assert!(err < 0.25 * level, "{name}: decode error {err:.2e} against a signal level of {level:.2e}");
// ffmpeg's decode, wherever its version puts the priming
let theirs = ffmpeg_audio_f32(&ff, &path, &[]);
assert!(theirs.len() >= frames, "{name}: ffmpeg decoded {} samples", theirs.len());
let (ff_lag, ff_err) = best_lag(&ours, &theirs, from, len, -bound..=bound);
let priming = match ff_lag {
0 => "trims CodecDelay".to_string(),
l if l == delay => "keeps the priming samples".to_string(),
l => format!("is {l} samples off"),
};
println!("{name}: CodecDelay {delay} samples; vs the encoded signal lag 0, error {err:.2e} (level {level:.2e}); ffmpeg {priming}, error {ff_err:.2e}");
assert!(ff_err < 1e-3, "{name}: differs from ffmpeg's decode by {ff_err:.2e} at its best lag {ff_lag}");
// read from the middle, as a seek does
let at = rate as i64 / 2;
let mid = src.audio(at, 2000, rate).unwrap().channels[0].clone();
let diff = mid.iter().zip(&ours[at as usize..]).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max);
assert!(diff < 1e-3, "{name}: a read from {at} differs from continuous decoding by {diff}");
}
}

155
crates/codecs/tests/media_oracle.rs
View File

@ -1,6 +1,7 @@
//! Image sequences and Broadcast WAV against ffmpeg: ffmpeg writes numbered stills (PNG, JPEG,
//! TIFF, BMP, WebP when available) and BWF files; the sequence frames must equal ffmpeg's decode of
//! the same image2 sequence (lossless formats exactly, JPEG within a few levels), and the BWF
//! Image sequences, stills and Broadcast WAV against ffmpeg: ffmpeg writes numbered stills (PNG,
//! JPEG, TIFF, BMP, WebP when available, Targa, DPX), single Targa / DPX stills in every layout it
//! has, and BWF files; our frames must equal ffmpeg's decode of the same files (lossless formats
//! exactly, JPEG within a few levels, high bit depths within one 8-bit step), and the BWF
//! TimeReference must give the start timecode.
mod common;
@ -114,6 +115,154 @@ fn webp_sequence_lossless() {
check(&ff, "webp", &["-c:v", "libwebp", "-lossless", "1"], 0);
}
#[test]
fn targa_and_dpx_sequences_exact() {
let ff = filmcraft_testkit::require_ffmpeg!();
check(&ff, "tga", &[], 0);
check(&ff, "dpx", &["-pix_fmt", "gbrp10be"], 1);
}
/// One `ext` still in `pix_fmt` (95 × 63, so scan lines need padding; translucent when the format
/// has alpha) opened like an imported file must match ffmpeg's RGBA decode within `tol` (exactly
/// when `tol` is 0).
fn still_matches(ff: &Path, ext: &str, pix_fmt: &str, extra: &[&str], tol: u8) {
let name = format!("still_{pix_fmt}{}.{ext}", extra.join(""));
let alpha = if pix_fmt.contains('a') && pix_fmt != "pal8" { ",format=rgba,geq=r='r(X,Y)':g='g(X,Y)':b='b(X,Y)':a='X*2+Y'" } else { "" };
let vf = format!("scale=95:63{alpha}");
let mut args = vec!["-f", "lavfi", "-i", "testsrc2=size=96x64", "-frames:v", "1", "-vf", &vf, "-pix_fmt", pix_fmt];
args.extend_from_slice(extra);
let Some(f) = fixture(ff, &name, &args) else {
eprintln!("SKIPPED: ffmpeg cannot write {name}");
return;
};
let src = filmcraft_codecs::open_bytes(&name, bytes(&f)).unwrap_or_else(|e| panic!("{name}: {e}"));
assert_eq!(src.info().kind, MediaKind::Still, "{name}");
let v = src.info().video.as_ref().unwrap();
assert_eq!((v.width, v.height), (95, 63), "{name}");
let ours = rgba(&src.video_frame(FrameRequest::full(filmcraft_time::Tick(0))).unwrap());
// deeper than 8 bits: ffmpeg's 16-bit decode, rounded to 8 bits (its own 8-bit conversion dithers)
let want: Vec<u8> = if tol == 0 {
ffmpeg_out(ff, &["-i", f.to_str().unwrap(), "-f", "rawvideo", "-pix_fmt", "rgba", "-"])
} else {
let wide = ffmpeg_out(ff, &["-i", f.to_str().unwrap(), "-f", "rawvideo", "-pix_fmt", "rgba64le", "-"]);
wide.as_chunks::<2>().0.iter().map(|c| ((u32::from(u16::from_le_bytes(*c)) * 255 + 32767) / 65535) as u8).collect()
};
assert_eq!(ours.len(), want.len(), "{name}");
let d = ours.iter().zip(&want).map(|(a, b)| a.abs_diff(*b)).max().unwrap();
assert!(d <= tol, "{name}: differs by {d}");
assert_eq!(v.has_alpha, want.as_chunks::<4>().0.iter().any(|p| p[3] != 255), "{name}");
eprintln!("{name}: max diff {d}");
}
#[test]
fn targa_stills_every_layout() {
let ff = filmcraft_testkit::require_ffmpeg!();
for pix_fmt in ["bgr24", "bgra", "rgb555le", "gray", "pal8"] {
still_matches(&ff, "tga", pix_fmt, &[], 0); // run-length encoded (ffmpeg's default)
still_matches(&ff, "tga", pix_fmt, &["-rle", "0"], 0);
}
}
#[test]
fn dpx_stills_every_layout() {
let ff = filmcraft_testkit::require_ffmpeg!();
for (pix_fmt, tol) in [("rgb24", 0), ("rgba", 0), ("gray", 0), ("rgb48be", 1), ("rgba64be", 1), ("gray16be", 1), ("gbrp10be", 1), ("gbrp12be", 1)] {
still_matches(&ff, "dpx", pix_fmt, &[], tol);
}
}
/// A big-endian DPX file holding `data` as its one image element (`w` × `h`, the given
/// descriptor, bit depth, packing and colorimetric code; reference codes undefined).
fn dpx_file(w: u32, h: u32, descriptor: u8, bits: u8, packing: u16, colorimetric: u8, data: &[u8]) -> Vec<u8> {
let mut f = vec![0u8; 2048];
let put = |f: &mut Vec<u8>, at: usize, v: &[u8]| f[at..at + v.len()].copy_from_slice(v);
put(&mut f, 0, b"SDPX");
put(&mut f, 4, &2048u32.to_be_bytes());
put(&mut f, 8, b"V2.0");
put(&mut f, 16, &(2048 + data.len() as u32).to_be_bytes());
put(&mut f, 770, &1u16.to_be_bytes());
put(&mut f, 772, &w.to_be_bytes());
put(&mut f, 776, &h.to_be_bytes());
let el = 780;
put(&mut f, el + 4, &u32::MAX.to_be_bytes());
put(&mut f, el + 12, &u32::MAX.to_be_bytes());
f[el + 20] = descriptor;
f[el + 22] = colorimetric;
f[el + 23] = bits;
put(&mut f, el + 24, &packing.to_be_bytes());
put(&mut f, el + 28, &2048u32.to_be_bytes());
f.extend_from_slice(data);
f
}
/// `pix_fmt` raw frame of ffmpeg's test picture at `w` × `h`.
fn raw_picture(ff: &Path, w: u32, h: u32, pix_fmt: &str) -> Vec<u8> {
let alpha = if pix_fmt.contains('a') { ",format=rgba,geq=r='r(X,Y)':g='g(X,Y)':b='b(X,Y)':a='X*2+Y'" } else { "" };
let vf = format!("scale={w}:{h}{alpha}");
ffmpeg_out(ff, &["-f", "lavfi", "-i", "testsrc2=size=96x64", "-frames:v", "1", "-vf", &vf, "-f", "rawvideo", "-pix_fmt", pix_fmt, "-"])
}
/// A DPX file we built must import exactly like ffmpeg decodes it, within `tol` of its 16-bit
/// decode rounded to 8 bits.
fn dpx_matches(ff: &Path, name: &str, file: Vec<u8>, tol: u8) {
let path = dir().join(name);
std::fs::create_dir_all(dir()).unwrap();
std::fs::write(&path, &file).unwrap();
let src = filmcraft_codecs::open_bytes(name, file.into()).unwrap_or_else(|e| panic!("{name}: {e}"));
assert_eq!(src.info().kind, MediaKind::Still, "{name}");
let ours = rgba(&src.video_frame(FrameRequest::full(filmcraft_time::Tick(0))).unwrap());
let wide = ffmpeg_out(ff, &["-i", path.to_str().unwrap(), "-f", "rawvideo", "-pix_fmt", "rgba64le", "-"]);
let want: Vec<u8> = wide.as_chunks::<2>().0.iter().map(|c| ((u32::from(u16::from_le_bytes(*c)) * 255 + 32767) / 65535) as u8).collect();
assert_eq!(ours.len(), want.len(), "{name}");
let d = ours.iter().zip(&want).map(|(a, b)| a.abs_diff(*b)).max().unwrap();
assert!(d <= tol, "{name}: differs by {d}");
eprintln!("{name}: max diff {d}");
}
/// Packed 12-bit RGB (packing 0: components back to back, low bits first, lines on 32-bit
/// boundaries) and 8-bit Y'CbCr (4:2:2, 4:4:4, 4:4:4:4 with alpha; BT.709, BT.601 and unspecified
/// colorimetry), layouts ffmpeg reads but does not write, built here from ffmpeg's test picture
/// (#841).
#[test]
fn dpx_packed_and_ycbcr_match_ffmpeg() {
let ff = filmcraft_testkit::require_ffmpeg!();
let (w, h) = (95u32, 63u32);
let rgb: Vec<u32> = raw_picture(&ff, w, h, "rgb48le").as_chunks::<2>().0.iter().map(|c| u32::from(u16::from_le_bytes(*c)) >> 4).collect();
let mut data = Vec::new();
for line in rgb.chunks(w as usize * 3) {
let (mut acc, mut n) = (0u64, 0u32);
for &v in line {
acc |= u64::from(v) << n;
n += 12;
while n >= 32 {
data.extend_from_slice(&(acc as u32).to_be_bytes());
acc >>= 32;
n -= 32;
}
}
if n > 0 {
data.extend_from_slice(&(acc as u32).to_be_bytes());
}
}
dpx_matches(&ff, "dpx_packed12.dpx", dpx_file(w, h, 50, 12, 0, 6, &data), 1);
// Y'CbCr: even width, scan lines a multiple of 4 bytes
let (w, h) = (96u32, 64u32);
let px = (w * h) as usize;
let uyvy = raw_picture(&ff, w, h, "uyvy422");
let yuv = raw_picture(&ff, w, h, "yuv444p");
let yuva = raw_picture(&ff, w, h, "yuva444p");
let interleave = |planes: &[u8], order: &[usize]| -> Vec<u8> { (0..px).flat_map(|i| order.iter().map(move |&p| planes[p * px + i])).collect() };
// planes are Y, U (Cb), V (Cr), A; DPX stores Cb Y Cr A
let cbycr = interleave(&yuv, &[1, 0, 2]);
let cbycra = interleave(&yuva, &[1, 0, 2, 3]);
for colorimetric in [6u8, 7, 0] {
dpx_matches(&ff, &format!("dpx_cbycry_{colorimetric}.dpx"), dpx_file(w, h, 100, 8, 0, colorimetric, &uyvy), 1);
dpx_matches(&ff, &format!("dpx_cbycr_{colorimetric}.dpx"), dpx_file(w, h, 102, 8, 0, colorimetric, &cbycr), 1);
}
dpx_matches(&ff, "dpx_cbycra.dpx", dpx_file(w, h, 103, 8, 0, 6, &cbycra), 1);
}
#[test]
fn missing_frames_hold_the_previous_frame() {
let ff = filmcraft_testkit::require_ffmpeg!();

84
crates/codecs/tests/mp4_oracle.rs
View File

@ -163,3 +163,87 @@ fn cropped_pictures_match_ffmpeg() {
}
}
}
/// An open GOP (x265 `open-gop`, like iPhone HEVC): each CRA after the first is followed in the
/// file by RASL pictures shown before it, predicted from the GOP before. Asked for first (a clip
/// trimmed to start on one, the playhead parked on one), such a picture came out black: decoding
/// started at the CRA after it, which leaves its RASL pictures out. Every frame, asked of a fresh
/// source and in an order that visits each CRA before its leading pictures, is ffmpeg's, from MP4
/// and from Matroska.
#[test]
fn open_gop_leading_pictures_decode_alone() {
let ff = filmcraft_testkit::require_ffmpeg!();
let args = [
"-f",
"lavfi",
"-i",
"testsrc2=size=160x96:rate=30:duration=2,noise=alls=10:allf=t",
"-c:v",
"libx265",
"-x265-params",
"log-level=error:keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3",
"-tag:v",
"hvc1",
"-pix_fmt",
"yuv420p",
];
let Some(mp4) = fixture(&ff, "open_gop_x265.mp4", &args) else {
eprintln!("SKIPPED: ffmpeg without libx265");
return;
};
let mkv = fixture(&ff, "open_gop_x265.mkv", &["-i", mp4.to_str().unwrap(), "-c", "copy"]).unwrap();
let raw = ffmpeg_frames(&ff, &mp4, "yuv420p");
let frame_bytes = 160 * 96 * 3 / 2;
let want: Vec<_> = raw.chunks(frame_bytes).map(|f| raw_planes(f, 160, 96, 80, 48, 1)).collect();
assert_eq!(want.len(), 60);
for path in [&mp4, &mkv] {
let name = path.file_name().unwrap().to_str().unwrap();
let open = || filmcraft_codecs::open_bytes(name, bytes(path)).unwrap();
let rate = open().info().frame_rate();
let frame = |src: &filmcraft_media::SharedSource, i: usize| {
let f = src.video_frame(filmcraft_media::FrameRequest::full(rate.tick_of(i as i64))).unwrap_or_else(|e| panic!("{name}: frame {i}: {e}"));
assert!(planes(&f) == want[i], "{name}: frame {i} differs from ffmpeg's");
};
for i in 0..want.len() {
frame(&open(), i);
}
let src = open();
for cra in [12, 24, 36, 48] {
for i in [cra, cra - 1, cra - 2, cra - 3] {
frame(&src, i);
}
}
}
}
/// #800: MP2 in MP4 (`mp4a` + object type 0x6B, which names no layer) and in MOV (QuickTime
/// `.mp2`) decodes sample-exactly as ffmpeg does: same length, the encoder delay (481 samples)
/// trimmed by the edit list, and the same samples from the first one on, also after a seek. It
/// used to go to the layer III decoder (MP4: silence) or not decode at all (MOV).
#[test]
fn mp2_in_mp4_and_mov_matches_ffmpeg() {
let ff = filmcraft_testkit::require_ffmpeg!();
for (name, rate) in [("mp2_stereo_224k_48k.mp4", 48_000u32), ("mp2_stereo_224k_44k.mov", 44_100)] {
let input = format!("anoisesrc=d=1:c=pink:r={rate}:a=0.3");
let Some(f) = fixture(&ff, name, &["-f", "lavfi", "-i", &input, "-ac", "2", "-c:a", "mp2", "-b:a", "224k"]) else {
eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
continue;
};
let src = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (rate, 2, "MPEG Audio"), "{name}");
let want = ffmpeg_audio_f32(&ff, &f, &[]);
let n = want.len() / 2;
assert!(n > rate as usize * 9 / 10, "{name}: ffmpeg decoded {n} samples");
let worst = |from: usize, len: usize| {
let got = src.audio(from as i64, len, rate).unwrap();
(0..len).map(|k| (0..2).map(|c| (got.channels[c][k] - want[2 * (from + k) + c]).abs()).fold(0f32, f32::max)).fold(0f32, f32::max)
};
let level = want.iter().fold(0f32, |m, v| m.max(v.abs()));
assert!(level > 0.1, "{name}: the source is silent ({level})");
// a seek into the middle first, then the whole track from the start
let (mid, all) = (worst(n / 2 + 77, 3000), worst(0, n));
println!("{name}: {n} samples, max error mid {mid:.2e}, whole {all:.2e}");
assert!(mid <= 1e-4 && all <= 1e-4, "{name}: differs from ffmpeg by {mid} (mid) / {all} (whole)");
}
}

59
crates/codecs/tests/mpeg_oracle.rs
View File

@ -16,6 +16,9 @@ use filmcraft_media::{FrameRequest, MediaError, MediaSource};
const NOISE: &[&str] =
&["-f", "lavfi", "-i", "aevalsrc=exprs=0.3*sin(2*PI*441*t)+0.1*sin(2*PI*3100*t)|0.25*sin(2*PI*660*t)+0.1*sin(2*PI*(500+4000*t)*t):s=48000"];
/// A mono tone.
const MONO: &[&str] = &["-f", "lavfi", "-i", "sine=frequency=1000:sample_rate=48000"];
fn spec(name: &str) -> Vec<String> {
let cat = |p: &[&[&str]]| p.concat().into_iter().map(String::from).collect::<Vec<_>>();
match name {
@ -142,6 +145,12 @@ fn spec(name: &str) -> Vec<String> {
cat(&[&["-f", "lavfi", "-i", "testsrc2=size=320x240:rate=25"], NOISE, &["-t", "0.6", "-c:v", "mpeg2video", "-bf", "2", "-g", "6", "-c:a", "mp2"]])
}
"audio.mp2" => cat(&[NOISE, &["-t", "1", "-c:a", "mp2", "-b:a", "224k", "-f", "mp2"]]),
// mono MP2 at ffmpeg's default bit rate, 384 kbit/s, which ISO/IEC 11172-3 does not allow
"audio_mono.mp2" => cat(&[MONO, &["-t", "1", "-c:a", "mp2", "-f", "mp2"]]),
"ts_mpeg2_mono_mp2.ts" => {
cat(&[&["-f", "lavfi", "-i", "testsrc2=size=320x240:rate=25"], MONO, &["-t", "1", "-c:v", "mpeg2video", "-c:a", "mp2", "-f", "mpegts"]])
}
"mkv_mono_mp2.mkv" => cat(&[MONO, &["-t", "1", "-c:a", "mp2"]]),
"es_mpeg2.m2v" => {
cat(&[&["-f", "lavfi", "-i", "testsrc2=size=352x288:rate=25"], &["-t", "1", "-c:v", "mpeg2video", "-bf", "2", "-g", "9", "-f", "mpeg2video"]])
}
@ -164,6 +173,9 @@ pub const ALL: &[&str] = &[
"mp4_mpeg2.mp4",
"mkv_mpeg2.mkv",
"audio.mp2",
"audio_mono.mp2",
"ts_mpeg2_mono_mp2.ts",
"mkv_mono_mp2.mkv",
"es_mpeg2.m2v",
];
@ -515,6 +527,53 @@ fn standalone_mp2_file() {
assert!(worst <= 2e-4, "{worst}");
}
/// #809: mono MP2 at 384 kbit/s, ffmpeg's default, played as silence in every container because
/// symphonia rejects the bit rate / mode combinations ISO/IEC 11172-3 does not allow.
#[test]
fn mono_mp2_at_ffmpegs_default_bit_rate() {
let ff = filmcraft_testkit::require_ffmpeg!();
let f = make(&ff, "audio_mono.mp2");
assert_eq!(std::fs::read(&f).unwrap()[2] >> 4, 14, "384 kbit/s");
let worst = check_audio(&ff, &make(&ff, "ts_mpeg2_mono_mp2.ts"), 2e-4);
println!("ts_mpeg2_mono_mp2.ts: max error {worst}");
// Matroska stores the encoder's 481-sample priming as `CodecDelay`. Whether ffmpeg's CLI trims
// it depends on its version (9.x does; 2023-24 builds keep the priming samples), and until
// #790 we trim it rounded to the millisecond. So for .mkv ffmpeg's decode is searched at
// lags within a sample or two of 0 (trimmed) and of `CodecDelay` (kept). The plain .mp2 file
// has no such ambiguity and must line up exactly.
for name in ["audio_mono.mp2", "mkv_mono_mp2.mkv"] {
let f = make(&ff, name);
let file = bytes(&f);
let src = filmcraft_codecs::open_bytes(name, file.clone()).unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!((a.sample_rate, a.channels), (48_000, 1), "{name}");
let want = ffmpeg_audio_f32(&ff, &f, &[]);
let (delay, lags): (usize, Vec<i64>) = match name.ends_with(".mkv") {
true => {
let mkv = filmcraft_matroska::open(file).unwrap();
let ns = mkv.tracks.first().map_or(0, |t| t.codec_delay_ns);
let delay = ((ns * 48_000 + 500_000_000) / 1_000_000_000) as usize;
assert!(delay > 0, "{name}: no CodecDelay");
(delay, (-2..=2).chain(delay as i64 - 2..=delay as i64 + 2).collect())
}
false => (0, vec![0]),
};
// ours from sample 2 on, against ffmpeg's from sample 2 + lag (inside both decodes)
let n = want.len() - delay - 6;
let buf = src.audio(0, n, 48_000).unwrap();
let (worst, lag) = lags
.into_iter()
.map(|lag| {
let err = (2..n - 2).map(|i| (want[(i as i64 + lag) as usize] - buf.channels[0][i]).abs()).fold(0f32, f32::max);
(err, lag)
})
.fold((f32::INFINITY, 0), |m, x| if x.0 < m.0 { x } else { m });
let peak = want.iter().fold(0f32, |m, x| m.max(x.abs()));
println!("{name}: max error {worst} at lag {lag} (CodecDelay {delay}), peak {peak}");
assert!(peak > 0.1 && worst <= 2e-4, "{name}: peak {peak}, max error {worst} at lag {lag}");
}
}
#[test]
fn truncated_and_corrupt_files_never_panic() {
let ff = filmcraft_testkit::require_ffmpeg!();

69
crates/codecs/tests/rf64_metadata.rs Normal file
View File

@ -0,0 +1,69 @@
//! #925: a chunk after an RF64 `data` sentinel must not become audio.
use std::sync::Arc;
use filmcraft_time::Tick;
fn chunk(id: &[u8; 4], body: &[u8]) -> Vec<u8> {
let mut out = id.to_vec();
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(body);
if body.len() % 2 == 1 {
out.push(0);
}
out
}
/// Four 16-bit mono frames of silence at 48 kHz, as RIFF or RF64, optionally with a trailing iXML.
fn fixture(rf64: bool, trailing: bool) -> Vec<u8> {
let mut fmt = Vec::new();
for v in [1u16, 1] {
fmt.extend_from_slice(&v.to_le_bytes());
}
fmt.extend_from_slice(&48_000u32.to_le_bytes());
fmt.extend_from_slice(&96_000u32.to_le_bytes());
fmt.extend_from_slice(&2u16.to_le_bytes());
fmt.extend_from_slice(&16u16.to_le_bytes());
let mut out = if rf64 { b"RF64\xff\xff\xff\xffWAVE".to_vec() } else { b"RIFF\0\0\0\0WAVE".to_vec() };
if rf64 {
let mut ds = vec![0u8; 28];
ds[8..16].copy_from_slice(&8u64.to_le_bytes());
ds[16..24].copy_from_slice(&4u64.to_le_bytes());
out.extend(chunk(b"ds64", &ds));
}
out.extend(chunk(b"fmt ", &fmt));
let offset = out.len();
out.extend(chunk(b"data", &[0u8; 8]));
if rf64 {
out[offset + 4..offset + 8].copy_from_slice(&u32::MAX.to_le_bytes());
}
if trailing {
out.extend(chunk(b"iXML", b"<BWFXML></BWFXML>"));
}
let size = out.len() - 8;
if rf64 {
out[20..28].copy_from_slice(&(size as u64).to_le_bytes());
} else {
out[4..8].copy_from_slice(&(size as u32).to_le_bytes());
}
out
}
#[test]
fn rf64_metadata_stays_outside_audio() {
for (name, rf64, trailing) in [("rf64.wav", true, false), ("riff_ixml.wav", false, true), ("rf64_ixml.wav", true, true)] {
let bytes: Arc<[u8]> = fixture(rf64, trailing).into();
let src = filmcraft_codecs::open_bytes(name, bytes.clone()).unwrap();
let stream = filmcraft_media::reader::open_reader(
name,
Arc::new(filmcraft_media::reader::MemReader(bytes)),
&filmcraft_codecs::reader_openers(),
&filmcraft_codecs::openers(),
)
.unwrap();
for s in [&src, &stream] {
assert_eq!(s.info().duration, Tick::from_units(4, 48_000), "{name}");
assert_eq!(s.audio(0, 20, 48_000).unwrap().channels.remove(0), vec![0.0; 20], "{name}");
}
}
}

2
crates/color/src/lib.rs
View File

@ -20,7 +20,7 @@ pub use grade::GradeSpace;
pub use log::LogCurve;
pub use lut::{Lut, Lut1d, Lut3d, LutFormat};
pub use spaces::{ColorPipeline, ColorSpace, Curve, Gamut, WorkingSpace};
pub use transform::{DecodeTable, InputTransform, OutputTransform, REFERENCE_WHITE_NITS, ToneMap};
pub use transform::{DecodeTable, GpuCurve, GpuInputParams, InputTransform, OutputTransform, REFERENCE_WHITE_NITS, ToneMap};
/// YUV ↔ RGB matrix coefficients.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]

15
crates/color/src/lut.rs
View File

@ -186,7 +186,7 @@ impl LutFormat {
}
fn floats(line: &str) -> Result<Vec<f32>, String> {
line.split_whitespace().map(|x| x.parse::<f32>().map_err(|_| format!("not a number: {x:?}"))).collect()
line.split_whitespace().map(|x| x.parse::<f32>().ok().filter(|v| v.is_finite()).ok_or_else(|| format!("not a finite number: {x:?}"))).collect()
}
impl Lut {
@ -252,6 +252,9 @@ impl Lut {
if v.len() != 2 {
return Err(err(&format!("{key} needs 2 values")));
}
if v[1] <= v[0] {
return Err(err(&format!("{key} maximum must exceed its minimum")));
}
let r = Some([v[0], v[1]]);
if key == "LUT_1D_INPUT_RANGE" { range1 = r } else { range3 = r }
}
@ -591,6 +594,16 @@ mod tests {
assert!(Lut::parse_cube("TITLE \"x\"\n").is_err());
assert!(Lut::parse_cube("LUT_3D_SIZE 2\n0 0 0\nLUT_1D_SIZE 2\n").is_err());
assert!(Lut::parse_cube("LUT_3D_SIZE 2\nDOMAIN_MIN 1 1 1\nDOMAIN_MAX 0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n0 0 0\n").is_err());
for key in ["LUT_1D_INPUT_RANGE", "LUT_3D_INPUT_RANGE"] {
let (size, rows) = if key.starts_with("LUT_1D") { ("LUT_1D_SIZE 2", 2) } else { ("LUT_3D_SIZE 2", 8) };
let table = "0 0 0\n".repeat(rows);
for range in ["0 0", "1 0", "NaN 1", "0 inf"] {
assert!(Lut::parse_cube(&format!("{size}\n{key} {range}\n{table}")).is_err(), "accepted {key} {range}");
}
assert!(Lut::parse_cube(&format!("{size}\n{key} -0.5 2\n{table}")).is_ok());
}
assert!(Lut::parse_cube("LUT_1D_SIZE 2\n0 0 0\nNaN 1 1\n").is_err());
assert!(Lut::parse_cube("LUT_1D_SIZE 2\nDOMAIN_MAX inf 1 1\n0 0 0\n1 1 1\n").is_err());
// comments, blank lines, Windows newlines, inline comments
let txt = "# hi\r\nTITLE \"t\"\r\n\r\nLUT_3D_SIZE 2 # two\r\n0 0 0\n1 0 0\n0 1 0\n1 1 0\n0 0 1\n1 0 1\n0 1 1\n1 1 1\n";
let l = Lut3d::parse_cube(txt).unwrap();

67
crates/color/src/transform.rs
View File

@ -259,6 +259,54 @@ impl InputTransform {
pub fn convert(&self, rgb: [f32; 3]) -> [f32; 3] {
self.apply(rgb.map(|v| self.table.lookup(v)))
}
/// The transform as plain numbers for a shader, which evaluates the curve and the BT.2390
/// EETF directly instead of sampling the tables. `None` for what a shader does not do: log
/// curves and the camera-gamut compression.
pub fn gpu_params(&self) -> Option<GpuInputParams> {
let curve = match self.source.curve() {
Curve::Sdr | Curve::Srgb => GpuCurve::Sdr,
Curve::Linear => GpuCurve::Linear,
Curve::Pq => GpuCurve::Pq,
Curve::Hlg => GpuCurve::Hlg,
Curve::Log(_) => return None,
};
if self.compress {
return None;
}
Some(GpuInputParams {
curve,
hlg: self.hlg,
matrix: (!self.identity_matrix).then_some(self.matrix),
tone: self.tone.as_ref().map(|t| (t.src_peak_nits as f32, t.dst_peak_nits as f32)),
desaturate: self.desaturate,
})
}
}
/// How a shader decodes a channel ([`decode_channel`] without the table): SDR with the sRGB curve,
/// linear, PQ in working units (cd/m² over [`REFERENCE_WHITE_NITS`]), HLG to scene light.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum GpuCurve {
Sdr,
Linear,
Pq,
Hlg,
}
/// [`InputTransform`] for a shader ([`InputTransform::gpu_params`]), stage by stage as
/// [`InputTransform::apply`] runs them.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct GpuInputParams {
pub curve: GpuCurve,
/// HLG OOTF: BT.2020 luma weights and the scale to working units.
pub hlg: Option<([f32; 3], f32)>,
/// Into the working primaries (row-major), when they differ.
pub matrix: Option<[[f32; 3]; 3]>,
/// BT.2390 tone mapping (source peak, target peak) in cd/m², on max(R,G,B) as a ratio.
pub tone: Option<(f32, f32)>,
/// Luma weights to desaturate into the working gamut with ([`desaturate_into_gamut`]).
pub desaturate: Option<[f32; 3]>,
}
/// Whether gamut `a` lies inside gamut `b` (so no compression is needed going a → b).
@ -462,6 +510,25 @@ mod tests {
}
}
#[test]
fn gpu_params_cover_display_referred_media_only() {
// HLG / PQ / BT.2020 into Rec. 709: every stage the shader has (#965)
let p = |cs| InputTransform::new(cs, Range::Limited, &ColorPipeline::REC709, None).gpu_params();
let hlg = p(ColorSpace::Rec2100Hlg).unwrap();
assert_eq!(hlg.curve, GpuCurve::Hlg);
assert!(hlg.hlg.is_some() && hlg.matrix.is_some() && hlg.desaturate.is_some());
assert_eq!(hlg.tone, Some((HDR_PEAK_NITS as f32, REFERENCE_WHITE_NITS as f32)));
let pq = InputTransform::new(ColorSpace::Rec2100Pq, Range::Limited, &ColorPipeline::REC709, Some(4000.0)).gpu_params().unwrap();
assert_eq!((pq.curve, pq.hlg, pq.tone), (GpuCurve::Pq, None, Some((4000.0, REFERENCE_WHITE_NITS as f32))));
let sdr2020 = p(ColorSpace::Rec2020).unwrap();
assert_eq!((sdr2020.curve, sdr2020.tone), (GpuCurve::Sdr, None));
assert!(sdr2020.matrix.is_some() && sdr2020.desaturate.is_some());
// log curves and camera-gamut compression stay on the CPU
for cs in ColorSpace::ALL.into_iter().filter(|c| c.is_log()) {
assert_eq!(p(cs), None, "{cs:?}");
}
}
#[test]
fn sdr_in_hdr_keeps_values_and_display_tone_maps() {
let pipe = hdr(WorkingSpace::Rec2100Pq);

237
crates/edit/src/lib.rs
View File

@ -214,6 +214,26 @@ fn transitions_follow_cuts(before: &Sequence, after: &mut Sequence) {
}
}
/// Keep a clip's one-sided transitions (a fade in from nothing, a fade out to nothing) on its edges
/// after a regular trim moved them, no longer than the clip (#374). Transitions between two clips
/// are left alone: a regular trim cannot move their cut.
fn fades_follow_edges(track: &mut Track, clip: ClipId) {
let Some((start, end, len)) = track.item(clip).map(|i| (i.start, i.end(), i.duration)) else { return };
for tr in &mut track.transitions {
match (tr.from, tr.to) {
(None, Some(to)) if to == clip => {
tr.duration = tr.duration.min(len);
tr.start = start;
}
(Some(from), None) if from == clip => {
tr.duration = tr.duration.min(len);
tr.start = end - tr.duration;
}
_ => {}
}
}
}
fn place(track: &mut Track, item: TrackItem) {
let idx = track.items.partition_point(|i| i.start <= item.start);
track.items.insert(idx, item);
@ -227,6 +247,15 @@ fn track_mut(seq: &mut Sequence, id: TrackId) -> Result<&mut Track> {
seq.track_mut(id).ok_or(EditError::NoTrack(id))
}
/// The clip's track, or [`EditError::Locked`] when that track may not change.
fn unlocked_track(seq: &Sequence, clip: ClipId) -> Result<TrackId> {
let (tid, _) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
if seq.track(tid).ok_or(EditError::NoTrack(tid))?.locked {
return Err(EditError::Locked);
}
Ok(tid)
}
/// Overwrite edits: each item replaces material on its track.
pub fn overwrite(seq: &mut Sequence, placements: Vec<(TrackId, TrackItem)>, ctx: &mut EditCtx) -> Result<Vec<ClipId>> {
let mut links = HashMap::new();
@ -448,6 +477,10 @@ pub fn ripple_delete_items(seq: &mut Sequence, items: &[ClipId]) -> Result<Vec<T
/// Close the gap containing `t` on a track (Ripple Delete on a gap).
pub fn close_gap(seq: &mut Sequence, track: TrackId, t: Tick) -> Result<()> {
let tr = seq.track(track).ok_or(EditError::NoTrack(track))?;
if tr.locked {
// the other tracks would move while this one stayed put
return Err(EditError::Locked);
}
if tr.item_at(t).is_some() {
return Err(EditError::Nothing);
}
@ -479,7 +512,10 @@ pub fn move_items(seq: &mut Sequence, moves: &[(ClipId, TrackId, Tick)], insert_
let mut work = seq.clone();
let mut placed = Vec::new();
for (c, dest, start) in moves {
let (_, it) = work.find_item(*c).ok_or(EditError::NoItem(*c))?;
let (src, it) = work.find_item(*c).ok_or(EditError::NoItem(*c))?;
if work.track(src).is_some_and(|t| t.locked) {
return Err(EditError::Locked);
}
let mut it = it.clone();
if work.track(*dest).ok_or(EditError::NoTrack(*dest))?.locked {
return Err(EditError::Locked);
@ -506,6 +542,60 @@ pub fn move_items(seq: &mut Sequence, moves: &[(ClipId, TrackId, Tick)], insert_
Ok(())
}
/// Copy items to (track, time) with overwrite or insert semantics, leaving the originals where
/// they are (Option-drag). `moves`: (item, destination track, start of the copy). The copies get
/// new ids; copies of linked clips are linked to each other, not to the originals, and the
/// transitions that would travel with a move are copied with them. Returns the copies' ids in
/// the order of `moves`.
pub fn copy_items(seq: &mut Sequence, moves: &[(ClipId, TrackId, Tick)], insert_mode: bool, ctx: &mut EditCtx) -> Result<Vec<ClipId>> {
let mut work = seq.clone();
let mut placed = Vec::new();
for (c, dest, start) in moves {
let (_, it) = work.find_item(*c).ok_or(EditError::NoItem(*c))?;
let mut it = it.clone();
if work.track(*dest).ok_or(EditError::NoTrack(*dest))?.locked {
return Err(EditError::Locked);
}
it.start = (*start).max(Tick::ZERO);
placed.push((*dest, it));
}
// worked out on the original ids, then moved over to the copies
let carried = carried_transitions(&work, &placed);
let mut ids = HashMap::new();
let mut links = HashMap::new();
for (_, it) in &mut placed {
let id = ClipId(ctx.alloc());
ids.insert(it.id, id);
it.id = id;
if let Some(l) = it.link {
it.link = Some(*links.entry(l).or_insert_with(|| ctx.alloc()));
}
}
// a copy whose linked partner is not copied is not linked
let mut counts: HashMap<u64, usize> = HashMap::new();
for l in placed.iter().filter_map(|(_, it)| it.link) {
*counts.entry(l).or_default() += 1;
}
for (_, it) in &mut placed {
if it.link.is_some_and(|l| counts.get(&l).copied().unwrap_or(0) < 2) {
it.link = None;
}
}
let out = if insert_mode { insert(&mut work, placed, ctx)? } else { overwrite(&mut work, placed, ctx)? };
for (dest, mut tr) in carried {
tr.id = TransitionId(ctx.alloc());
tr.from = tr.from.and_then(|c| ids.get(&c).copied());
tr.to = tr.to.and_then(|c| ids.get(&c).copied());
let t = track_mut(&mut work, dest)?;
if [tr.from, tr.to].into_iter().flatten().all(|c| t.item(c).is_some()) && (tr.from.is_some() || tr.to.is_some()) {
t.transitions.push(tr);
t.sort();
}
}
*seq = work;
Ok(out)
}
/// The transitions that travel with a move (#227, #374): those whose clips all move, onto one
/// track and by one offset (a fade at a moved clip's edge, or the cut between two clips that move
/// together), shifted to where those clips land.
@ -555,7 +645,8 @@ fn neighbours(track: &Track, id: ClipId) -> (Tick, Tick) {
/// Compute the clamped delta a trim may apply (for UI feedback while dragging).
pub fn clamp_trim(seq: &Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta: Tick, ctx: &EditCtx) -> Result<Tick> {
let (tid, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let tid = unlocked_track(seq, clip)?;
let (_, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let tr = seq.track(tid).ok_or(EditError::NoTrack(tid))?;
let (prev_end, next_start) = neighbours(tr, clip);
let media = media_len(ctx, it);
@ -597,9 +688,35 @@ pub fn clamp_trim(seq: &Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delt
Ok(d)
}
/// What moving one edge by `d` (already clamped) does to the item itself: its length, the media
/// it starts at (a reversed clip consumes media from its other end; a frame hold has none to
/// move), and, for the In edge with `move_start`, where it starts. The trims and the timeline's
/// live trim preview share it, so the preview shows the media the edit will keep (#374).
pub fn move_edge(it: &mut TrackItem, edge: Edge, d: Tick, move_start: bool) {
let speed = it.speed.abs();
match edge {
Edge::In => {
if !it.reverse && it.frame_hold.is_none() {
it.source_in += src_of(d, speed);
}
it.duration -= d;
if move_start {
it.start += d;
}
}
Edge::Out => {
if it.reverse {
it.source_in -= src_of(d, speed);
}
it.duration += d;
}
}
}
/// Trim one edge of an item (Selection-tool edge drag = Regular; Ripple tool = Ripple).
/// Linked partners should be trimmed by the caller with the same delta.
pub fn trim(seq: &mut Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, clip)?;
let d = clamp_trim(seq, clip, edge, mode, delta, ctx)?;
if d == Tick::ZERO {
return Ok(d);
@ -608,27 +725,10 @@ pub fn trim(seq: &mut Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta:
let old_end = it.end();
let old_start = it.start;
let own_link = it.link;
let speed = it.speed.abs();
let mut work = seq.clone();
{
let (_, it) = work.find_item_mut(clip).ok_or(EditError::NoItem(clip))?;
match edge {
Edge::In => {
if !it.reverse && it.frame_hold.is_none() {
it.source_in += src_of(d, speed);
}
it.duration -= d;
if mode == TrimMode::Regular {
it.start += d;
}
}
Edge::Out => {
if it.reverse {
it.source_in -= src_of(d, speed);
}
it.duration += d;
}
}
move_edge(it, edge, d, mode == TrimMode::Regular);
}
if mode == TrimMode::Ripple {
let (at, shift) = match edge {
@ -658,6 +758,8 @@ pub fn trim(seq: &mut Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta:
}
}
transitions_follow_cuts(seq, &mut work);
} else if let Ok(track) = track_mut(&mut work, tid) {
fades_follow_edges(track, clip);
}
work.check().map_err(EditError::Other)?;
*seq = work;
@ -671,6 +773,9 @@ pub fn trim(seq: &mut Sequence, clip: ClipId, edge: Edge, mode: TrimMode, delta:
/// overwrite material on a sync-locked track, or when such a linked clip has no room to follow.
pub fn ripple_trim_group(seq: &mut Sequence, clips: &[ClipId], edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
let Some(&first) = clips.first() else { return Ok(Tick::ZERO) };
for c in clips {
unlocked_track(seq, *c)?;
}
let mut d = delta;
for c in clips {
let x = clamp_trim(seq, *c, edge, TrimMode::Ripple, d, ctx)?;
@ -686,25 +791,11 @@ pub fn ripple_trim_group(seq: &mut Sequence, clips: &[ClipId], edge: Edge, delta
let mut origins: Vec<(TrackId, Tick)> = Vec::new();
for c in clips {
let (tid, it) = work.find_item_mut(*c).ok_or(EditError::NoItem(*c))?;
let speed = it.speed.abs();
let from = match edge {
Edge::In => it.start + Tick(1),
Edge::Out => it.end(),
};
match edge {
Edge::In => {
if !it.reverse && it.frame_hold.is_none() {
it.source_in += src_of(d, speed);
}
it.duration -= d;
}
Edge::Out => {
if it.reverse {
it.source_in -= src_of(d, speed);
}
it.duration += d;
}
}
move_edge(it, edge, d, false);
if !origins.iter().any(|(t, _)| *t == tid) {
origins.push((tid, from));
}
@ -787,6 +878,8 @@ pub fn ripple_trim_group(seq: &mut Sequence, clips: &[ClipId], edge: Edge, delta
/// Rolling edit between two adjacent items on one track: moves the cut by `delta`.
pub fn roll(seq: &mut Sequence, left: ClipId, right: ClipId, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, left)?;
unlocked_track(seq, right)?;
let (_, l) = seq.find_item(left).ok_or(EditError::NoItem(left))?;
let (_, r) = seq.find_item(right).ok_or(EditError::NoItem(right))?;
if l.end() != r.start {
@ -834,6 +927,7 @@ pub fn roll(seq: &mut Sequence, left: ClipId, right: ClipId, delta: Tick, ctx: &
/// Slip: change which part of the media an item shows without moving it.
pub fn slip(seq: &mut Sequence, clip: ClipId, delta_media: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, clip)?;
let (_, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let used = src_of(it.duration, it.speed);
let max_in = media_len(ctx, it).map(|m| m - used).unwrap_or(Tick::MAX);
@ -847,6 +941,7 @@ pub fn slip(seq: &mut Sequence, clip: ClipId, delta_media: Tick, ctx: &mut EditC
/// Slide: move an item between its neighbours, trimming them to keep the gapless span.
pub fn slide(seq: &mut Sequence, clip: ClipId, delta: Tick, ctx: &mut EditCtx) -> Result<Tick> {
unlocked_track(seq, clip)?;
let (tid, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let it = it.clone();
let tr = seq.track(tid).ok_or(EditError::NoTrack(tid))?;
@ -925,27 +1020,65 @@ pub fn slide_items(seq: &mut Sequence, clips: &[ClipId], delta: Tick, ctx: &mut
Ok(d)
}
/// Rate stretch: change an item's duration by dragging an edge, adjusting speed to keep the same media.
pub fn rate_stretch(seq: &mut Sequence, clip: ClipId, edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<f64> {
/// How far `clip` can rate-stretch on `edge` without overlapping a neighbour or going shorter than
/// one frame. Linked partners use this so they all take the tightest room.
pub fn clamp_rate_stretch(seq: &Sequence, clip: ClipId, edge: Edge, delta: Tick, ctx: &EditCtx) -> Result<Tick> {
let (tid, it) = seq.find_item(clip).ok_or(EditError::NoItem(clip))?;
let tr = seq.track(tid).ok_or(EditError::NoTrack(tid))?;
let (prev_end, next_start) = neighbours(tr, clip);
let src_len = it.source_out() - it.source_in;
let d = match edge {
Ok(match edge {
Edge::Out => delta.clamp(-(it.duration - ctx.min_duration), next_start - it.end()),
Edge::In => delta.clamp(-(it.start - prev_end), it.duration - ctx.min_duration),
};
let new_dur = match edge {
Edge::Out => it.duration + d,
Edge::In => it.duration - d,
};
let speed = src_len.0 as f64 / new_dur.0 as f64;
let (_, it) = seq.find_item_mut(clip).ok_or(EditError::NoItem(clip))?;
if edge == Edge::In {
it.start += d;
})
}
/// Rate stretch: change an item's duration by dragging an edge, adjusting speed to keep the same media.
pub fn rate_stretch(seq: &mut Sequence, clip: ClipId, edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<f64> {
rate_stretch_items(seq, &[clip], edge, delta, ctx)
}
/// Rate stretch a clip and its linked partners as one edit: the tightest neighbour on any member
/// clamps them all, so they keep one speed. A neighbour on the sound track cannot leave the picture
/// stretching further (and the pair still linked at different rates).
pub fn rate_stretch_items(seq: &mut Sequence, clips: &[ClipId], edge: Edge, delta: Tick, ctx: &mut EditCtx) -> Result<f64> {
// clips on locked tracks are left as they are; with every clip locked the edit is refused
let mut open = Vec::with_capacity(clips.len());
for &c in clips {
match unlocked_track(seq, c) {
Ok(_) => open.push(c),
Err(EditError::Locked) => {}
Err(e) => return Err(e),
}
}
if open.is_empty() {
return Err(EditError::Locked);
}
let clips = open.as_slice();
let mut d = delta;
for c in clips {
let x = clamp_rate_stretch(seq, *c, edge, d, ctx)?;
if x.0.signum() != d.0.signum() {
d = Tick::ZERO;
} else if x.abs() < d.abs() {
d = x;
}
}
let mut speed = 1.0;
for c in clips {
let (_, it) = seq.find_item(*c).ok_or(EditError::NoItem(*c))?;
let src_len = it.source_out() - it.source_in;
let new_dur = match edge {
Edge::Out => it.duration + d,
Edge::In => it.duration - d,
};
speed = src_len.0 as f64 / new_dur.0 as f64;
let (_, it) = seq.find_item_mut(*c).ok_or(EditError::NoItem(*c))?;
if edge == Edge::In {
it.start += d;
}
it.duration = new_dur;
it.speed = speed;
}
it.duration = new_dur;
it.speed = speed;
Ok(speed)
}
@ -964,6 +1097,9 @@ pub fn set_speed_group(seq: &mut Sequence, clips: &[ClipId], speed: f64, reverse
if !(speed.is_finite() && speed > 0.0) {
return Err(EditError::Other("speed must be positive".into()));
}
for clip in clips {
unlocked_track(seq, *clip)?;
}
let mut work = seq.clone();
// per member track: where its later material starts, and by how much the member's length changed
let mut origins: Vec<(TrackId, Tick, Tick)> = Vec::new();
@ -987,6 +1123,7 @@ pub fn set_speed_group(seq: &mut Sequence, clips: &[ClipId], speed: f64, reverse
it.speed = speed;
it.reverse = reverse;
it.duration = new_dur;
fades_follow_edges(work.track_mut(tid).ok_or(EditError::NoTrack(tid))?, *clip);
origins.push((tid, old_end, delta));
}
let main_from = origins.first().map(|o| o.1).unwrap_or_default();

123
crates/edit/src/tests.rs
View File

@ -274,6 +274,17 @@ fn regular_trim_respects_neighbours_and_handles() {
fx.seq.check().unwrap();
}
#[test]
fn trim_on_a_locked_track_is_refused() {
let mut fx = Fx::new();
let v1 = fx.v(0);
let a = fx.put(v1, 0, 100, 0);
fx.seq.track_mut(v1).unwrap().locked = true;
let mut n = fx.next;
assert_eq!(trim(&mut fx.seq, a, Edge::Out, TrimMode::Regular, -f(20), &mut Fx::ctx(&mut n)), Err(EditError::Locked));
assert_eq!(fx.spans(v1), vec![(0, 100)]);
}
#[test]
fn held_regular_in_trim_stops_at_previous_clip() {
let mut fx = Fx::new();
@ -464,6 +475,58 @@ fn transitions_stay_on_their_cuts() {
fx.seq.check().unwrap();
}
#[test]
fn fades_follow_a_regular_trim() {
let mut fx = Fx::new();
let v1 = fx.v(0);
let a = fx.put(v1, 10, 20, 10);
let fade = |id, start, from, to| Transition {
id: TransitionId(id),
effect: filmcraft_project::find_effect("cross_dissolve").unwrap().instance(),
start: f(start),
duration: f(6),
from,
to,
align: Default::default(),
reverse: false,
};
fx.seq.track_mut(v1).unwrap().transitions = vec![fade(1, 10, None, Some(a)), fade(2, 24, Some(a), None)];
let spans = |fx: &Fx| fx.seq.track(v1).unwrap().transitions.iter().map(|t| (t.start, t.duration)).collect::<Vec<_>>();
let mut n = fx.next;
trim(&mut fx.seq, a, Edge::In, TrimMode::Regular, f(2), &mut Fx::ctx(&mut n)).unwrap();
trim(&mut fx.seq, a, Edge::Out, TrimMode::Regular, f(4), &mut Fx::ctx(&mut n)).unwrap();
// the fade in starts with the clip (12), the fade out ends with it (34)
assert_eq!(spans(&fx), vec![(f(12), f(6)), (f(28), f(6))]);
// a clip trimmed shorter than its fades keeps them inside it
trim(&mut fx.seq, a, Edge::Out, TrimMode::Regular, -f(18), &mut Fx::ctx(&mut n)).unwrap();
assert_eq!(spans(&fx), vec![(f(12), f(4)), (f(12), f(4))]);
fx.seq.check().unwrap();
}
#[test]
fn fades_follow_a_non_ripple_speed_change() {
// #918: faster without ripple shortens the clip; the fade-out stays on the new out point.
let mut fx = Fx::new();
let v1 = fx.v(0);
let a = fx.put(v1, 0, 100, 0);
fx.seq.track_mut(v1).unwrap().transitions = vec![Transition {
id: TransitionId(1),
effect: filmcraft_project::find_effect("cross_dissolve").unwrap().instance(),
start: f(90),
duration: f(10),
from: Some(a),
to: None,
align: Default::default(),
reverse: false,
}];
let mut n = fx.next;
set_speed(&mut fx.seq, a, 2.0, false, false, &mut Fx::ctx(&mut n)).unwrap();
let tr = &fx.seq.track(v1).unwrap().transitions[0];
assert_eq!(fx.spans(v1), vec![(0, 50)]);
assert_eq!((tr.start, tr.duration), (f(40), f(10)));
fx.seq.check().unwrap();
}
#[test]
fn transitions_travel_with_moved_clips() {
// V1: a [0,10) fading in, b [10,20), c [20,30); b→c crossfade, c fading out
@ -509,6 +572,29 @@ fn rate_stretch_and_speed() {
assert_eq!(fx.spans(v1), vec![(0, 5)]);
}
#[test]
fn linked_rate_stretch_uses_the_tighter_neighbour() {
// picture and sound are 60 frames; a later clip on A1 starts at 70, so the sound has only 10
// frames of room. Stretching the pair +20 must not leave the picture at 80f / 0.75x and the
// sound at 70f / 6/7x while still linked.
let mut fx = Fx::new();
let (v1, a1) = (fx.v(0), fx.a(0));
let pic = fx.put(v1, 0, 60, 0);
let snd = fx.put(a1, 0, 60, 0);
fx.put(a1, 70, 10, 0);
for c in [pic, snd] {
fx.seq.find_item_mut(c).unwrap().1.link = Some(1);
}
let mut n = fx.next;
let speed = rate_stretch_items(&mut fx.seq, &[pic, snd], Edge::Out, f(20), &mut Fx::ctx(&mut n)).unwrap();
assert!((speed - 60.0 / 70.0).abs() < 1e-9);
let (p, s) = (fx.seq.find_item(pic).unwrap().1, fx.seq.find_item(snd).unwrap().1);
assert_eq!((R.frame_at(p.duration), R.frame_at(s.duration)), (70, 70));
assert!((p.speed - s.speed).abs() < 1e-12);
assert_eq!(fx.spans(a1), vec![(0, 70), (70, 10)]);
fx.seq.check().unwrap();
}
#[test]
fn speed_on_a_linked_group_ripples_once() {
// V1: a [0,10), b [10,20). A1: a's sound [0,10), b's sound [10,20).
@ -782,3 +868,40 @@ fn roll_reversed_clips_keeps_media_mapping() {
let (_, r) = fx.seq.find_item(ib).unwrap();
assert_eq!((r.start, r.duration, r.source_in), (f(40), f(20), f(50)));
}
/// #374: moving an edge must not change which media sits under any timeline time the clip still
/// covers (the timeline's live trim preview draws its waveform from this item), and must leave the
/// item as the trim itself does.
#[test]
fn moving_an_edge_keeps_the_media_under_the_timeline() {
let fx = Fx::new();
let v1 = fx.v(0);
for (speed, reverse, hold) in [(1.0, false, None), (2.0, false, None), (0.5, false, None), (1.0, true, None), (2.0, true, None), (1.0, false, Some(f(50)))]
{
for (edge, d) in [(Edge::In, f(7)), (Edge::In, -f(4)), (Edge::Out, -f(9)), (Edge::Out, f(3))] {
let mut fresh = Fx::new();
let c = fresh.put(v1, 100, 40, 200);
{
let it = fresh.seq.find_item_mut(c).unwrap().1;
(it.speed, it.reverse, it.frame_hold) = (speed, reverse, hold);
}
let before = fresh.seq.find_item(c).unwrap().1.clone();
let mut moved = before.clone();
move_edge(&mut moved, edge, d, true);
let label = format!("speed {speed} reverse {reverse} hold {hold:?} {edge:?} {d:?}");
// every frame still covered shows the same media as before the move
let mut t = moved.start.max(before.start);
while t < moved.end().min(before.end()) {
let (a, b) = (before.source_time_at(t), moved.source_time_at(t));
assert!((a - b).0.abs() <= 1, "{label}: at {t:?} {a:?} became {b:?}");
t += f(1);
}
// and the item is what the real (unclamped) trim makes of it
let mut n = fresh.next;
let got = trim(&mut fresh.seq, c, edge, TrimMode::Regular, d, &mut Fx::ctx(&mut n)).unwrap();
assert_eq!(got, d, "{label}: not clamped");
let trimmed = fresh.seq.find_item(c).unwrap().1;
assert_eq!((trimmed.start, trimmed.duration, trimmed.source_in), (moved.start, moved.duration, moved.source_in), "{label}");
}
}
}

1
crates/engine/Cargo.toml
View File

@ -46,4 +46,5 @@ web-time = "1"
[dev-dependencies]
filmcraft-testkit = { workspace = true }
filmcraft-isobmff = { workspace = true }
image = { version = "0.25", default-features = false, features = ["png"] }

16
crates/engine/src/audio_effects_tests.rs
View File

@ -111,3 +111,19 @@ fn shared_names_resolve_by_track_kind() {
let it = s.active_sequence().unwrap().find_item(ClipId(v)).unwrap().1.clone();
assert!(it.effects.iter().any(|e| e.effect == "invert"));
}
/// #696: applying an audio effect to the video side of an A/V pair lands on the linked audio.
#[test]
fn audio_effect_on_video_clip_applies_to_linked_audio() {
let mut s = demo();
let seq = s.active_sequence().unwrap();
let v = seq.video_tracks[0].items[0].clone();
let a = seq.audio_tracks[0].items.iter().find(|i| i.link == v.link).expect("linked audio").clone();
let n0 = a.effects.len();
s.execute("effects.apply", json!({"clips": [v.id.0], "effect": "parametric_eq"})).unwrap();
let a = s.active_sequence().unwrap().find_item(a.id).unwrap().1.clone();
let v = s.active_sequence().unwrap().find_item(v.id).unwrap().1.clone();
assert_eq!(a.effects.len(), n0 + 1);
assert_eq!(a.effects.last().unwrap().effect, "parametric_eq");
assert!(v.effects.iter().all(|e| e.effect != "parametric_eq"), "EQ stays off the picture clip");
}

26
crates/engine/src/audio_placement_tests.rs
View File

@ -168,6 +168,32 @@ fn missing_audio_tracks_are_cloned_from_the_destination() {
assert_eq!(again, ids);
}
/// #823: a container with two PCM tracks imports both streams, and overwrite places linked clips.
#[test]
fn import_keeps_every_audio_stream_and_places_linked_clips() {
use filmcraft_isobmff::{Brand, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions};
let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
for (rate, value) in [(48_000u32, 8192i16), (24_000, -16384)] {
let entry = SampleEntry::pcm(PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: rate as f64 });
let track = mux.add_track(TrackConfig::new(entry, rate)).unwrap();
let data: Vec<u8> = (0..rate / 10).flat_map(|_| value.to_le_bytes()).collect();
mux.write_sample(track, WriteSample { data: &data, duration: rate / 10, composition_offset: 0, is_sync: true }).unwrap();
}
let bytes = mux.finish().unwrap().into_inner();
let mut s = Session::default();
let item = crate::commands::import_bytes(&mut s, "/two-audio.mov", bytes.into(), None).unwrap();
let streams = s.project.item(item).unwrap().as_media().unwrap().info.audio_streams.len();
assert_eq!(streams, 2);
s.execute("file.newSequence", json!({"name": "S", "fromItem": item.0, "video": 1, "audio": 1})).unwrap();
let seq = s.state.active_sequence.unwrap();
let q = s.project.sequence(seq).unwrap();
assert_eq!(q.audio_tracks.len(), 2);
let (a, b) = (&q.audio_tracks[0].items[0], &q.audio_tracks[1].items[0]);
assert_eq!((a.audio_stream, b.audio_stream), (0, 1));
assert_eq!(a.link, b.link);
assert!(a.link.is_some());
}
#[test]
fn hostile_track_counts_are_an_error_not_a_crash() {
let s = demo();

17
crates/engine/src/autosave.rs
View File

@ -572,15 +572,16 @@ pub fn scan_candidates(data_dir: &Path, skip: Option<&Path>) -> Vec<RecoveryCand
if !dir.is_dir() || Some(dir.as_path()) == skip || !lock_is_stale(&dir) {
continue;
}
let meta = fs::read(dir.join(META)).ok().and_then(|b| serde_json::from_slice::<JournalMeta>(&b).ok());
match meta {
Some(meta) if dir.join(SNAPSHOT).is_file() => {
out.push(RecoveryCandidate { id: e.file_name().to_string_lossy().into_owned(), dir, meta });
}
_ => {
let _ = fs::remove_dir_all(&dir);
}
let snap = dir.join(SNAPSHOT);
if !snap.is_file() {
// Empty leftover dirs (clean exit, or a crash before the first snapshot).
let _ = fs::remove_dir_all(&dir);
continue;
}
// Keep the snapshot even when session.json is missing or unreadable: the project
// is still recoverable. Meta fields fall back to defaults.
let meta = fs::read(dir.join(META)).ok().and_then(|b| serde_json::from_slice::<JournalMeta>(&b).ok()).unwrap_or_default();
out.push(RecoveryCandidate { id: e.file_name().to_string_lossy().into_owned(), dir, meta });
}
out.sort_by(|a, b| b.meta.saved_unix.cmp(&a.meta.saved_unix).then(b.id.cmp(&a.id)));
out

22
crates/engine/src/autosave_tests.rs
View File

@ -176,6 +176,28 @@ fn auto_save_ring_rotates_and_can_save_the_project() {
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn scan_keeps_snapshot_when_session_metadata_is_unreadable() {
// #894: a stale session with a snapshot must not be deleted because session.json is missing
// or corrupt — the recovered project would vanish from the list and the file would be gone.
let d = temp_dir("recover-nometa");
let root = autosave::recovery_root(&d);
let sess = root.join("123-1");
std::fs::create_dir_all(&sess).unwrap();
let snap = filmcraft_format::encode(&filmcraft_project::Project::default(), false);
std::fs::write(sess.join("snapshot.fcproj"), snap).unwrap();
std::fs::write(sess.join("session.json"), b"not json {").unwrap();
let found = autosave::scan_candidates(&d, None);
assert_eq!(found.len(), 1);
assert_eq!(found[0].id, "123-1");
assert!(sess.join("snapshot.fcproj").is_file(), "snapshot must survive the scan");
std::fs::remove_file(sess.join("session.json")).unwrap();
let found = autosave::scan_candidates(&d, None);
assert_eq!(found.len(), 1, "missing metadata still lists the snapshot");
assert!(sess.join("snapshot.fcproj").is_file());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn portable_marker_keeps_data_next_to_the_executable() {
// #139: a portable install must not write to %APPDATA%.

58
crates/engine/src/avi_import_tests.rs Normal file
View File

@ -0,0 +1,58 @@
//! #598: AVI files import (ffmpeg-made, skipped without it): the Project item describes the
//! streams, a sequence made from it plays picture and sound, and the Media Browser's streaming
//! probe opens it without reading it whole.
use serde_json::json;
use crate::Session;
use filmcraft_project::ItemId;
fn fixture() -> Option<std::path::PathBuf> {
let ffmpeg = filmcraft_testkit::oracle::ffmpeg_or_skip("avi import")?;
let path = filmcraft_testkit::fixtures_dir("engine/avi").join("h264_mp3.avi");
filmcraft_testkit::fixtures::generate(&path, |tmp| {
std::process::Command::new(&ffmpeg)
.args([
"-y",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=size=160x120:rate=25:duration=2",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=48000:duration=2",
])
.args(["-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p", "-c:a", "libmp3lame", "-f", "avi"])
.arg(tmp)
.status()
.is_ok_and(|s| s.success())
})
}
#[test]
fn avi_imports_plays_and_probes() {
let Some(path) = fixture() else { return };
let mut s = Session::default();
let r = s.execute("file.import", json!({"paths": [path.to_string_lossy()]})).unwrap();
assert!(r["errors"].as_array().is_none_or(|e| e.is_empty()), "{r}");
let item = ItemId(r["items"][0].as_u64().unwrap());
let info = s.project.item(item).unwrap().as_media().unwrap().info.clone();
assert_eq!(info.container, "AVI");
let v = info.video.as_ref().unwrap();
assert_eq!((v.width, v.height, v.codec.as_str()), (160, 120, "H.264"));
assert_eq!(info.audio_streams.first().map(|a| a.codec.as_str()), Some("MP3"));
assert!((info.duration.seconds() - 2.0).abs() < 0.1, "{}", info.duration.seconds());
// a sequence from the clip has its picture and sound
s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap();
let q = s.active_sequence().unwrap().clone();
assert!(!q.video_tracks[0].items.is_empty() && !q.audio_tracks[0].items.is_empty());
let src = s.source(item).unwrap();
assert!(src.video_frame(filmcraft_media::FrameRequest::full(filmcraft_time::Tick(filmcraft_time::TICKS_PER_SECOND))).is_ok());
assert!(src.audio(48_000, 4_800, 48_000).unwrap().channels[0].iter().any(|v| v.abs() > 0.05));
// the Media Browser's probe streams it (index now, frames on demand)
let probed = s.media.probe_file(&path.to_string_lossy(), &*s.services).unwrap();
assert_eq!(probed.info().container, "AVI");
}

4
crates/engine/src/clip_ops.rs
View File

@ -1715,8 +1715,8 @@ fn replace_with(s: &mut Session, p: &Value, how: Replace) -> Result<Value> {
}
let Some((_, it)) = q.find_item_mut(*c) else { continue };
let src_in = match how {
// the Source monitor frame lines up with the sequence playhead
Replace::MatchFrame => src_ph - Tick(((ph - it.start).0 as f64 * it.speed.abs()).round() as i64),
// the Source monitor frame lines up with the sequence playhead (reverse-aware)
Replace::MatchFrame => src_ph - (it.moving_source_time_at(ph) - it.source_in),
_ if keep_in => {
let kept = keep_within.map(|(first, last)| it.source_in.clamp(first, last)).unwrap_or(it.source_in);
if kept != it.source_in {

79
crates/engine/src/clip_ops_tests.rs
View File

@ -510,10 +510,14 @@ fn frame_blending_renders_in_between_frames() {
*a = (*a + b) * 0.5;
}
assert!(mean_abs_diff(&avg, &blended) < mean_abs_diff(&sampled, &blended), "closer to the average than to either frame");
// optical flow falls back to blending
// optical flow synthesises a motion-compensated frame instead of the cross-fade
s.execute("clip.timeInterpolation.opticalFlow", json!({})).unwrap();
s.execute("playhead.set", json!({"time": t.0})).unwrap();
assert!(mean_abs_diff(&s.render_program(0.125).unwrap(), &blended) < 1e-6);
let flowed = s.render_program(0.125).unwrap();
// the moving traffic is motion compensated, the static city stays as blended
let d = mean_abs_diff(&flowed, &blended);
assert!(d > 1e-6, "optical flow should not be the plain cross-fade: {d}");
assert!(d < mean_abs_diff(&blended, &sampled), "optical flow stays near the blend on mostly static footage: {d}");
// at 100 % nothing is blended
s.execute("clip.speedDuration", json!({"speed": 100.0})).unwrap();
assert!(filmcraft_render::interpolation_blend(&clip(&s, c.id.0), t, src_rate).is_none());
@ -602,6 +606,32 @@ fn replace_with_clip_from_source_match_frame_and_bin() {
assert!(s.execute("clip.replaceFromBin", json!({"item": music.0})).is_err());
}
/// Match Frame on a reversed clip maps the Source monitor frame through reverse playback, not as
/// if the clip still ran forward (source 50 at sequence 20 used to become source 69).
#[test]
fn replace_from_source_match_frame_on_a_reversed_clip() {
let mut s = demo();
let rate = s.sequence_rate();
s.execute("sequence.linkedSelection", json!({"on": false})).unwrap();
let forest = item_named(&s, "Misty_Forest.mp4");
s.execute("file.newSequence", json!({"name": "MatchFrameReverse"})).unwrap();
s.execute("timeline.place", json!({"item": forest.0, "time": 0})).unwrap();
let c = s.active_sequence().unwrap().video_tracks[0].items[0].clone();
s.execute("timeline.trim", json!({"clip": c.id.0, "edge": "out", "mode": "regular", "delta": (rate.tick_of(60) - c.duration).0})).unwrap();
select(&mut s, &[c.id.0]);
s.execute("clip.speedDuration", json!({"speed": 100, "reverse": true, "ripple": false})).unwrap();
let got = clip(&s, c.id.0);
assert_eq!(rate.frame_at(got.duration), 60);
assert!(got.reverse);
s.execute("source.open", json!({"item": forest.0})).unwrap();
s.execute("playhead.set", json!({"time": rate.tick_of(20).0})).unwrap();
s.execute("source.setPlayhead", json!({"frame": 50})).unwrap();
s.execute("clip.replaceFromSourceMatchFrame", json!({})).unwrap();
let got = clip(&s, c.id.0);
assert_eq!(got.source_time_at(rate.tick_of(20)), rate.tick_of(50));
assert_ne!(got.source_time_at(rate.tick_of(20)), rate.tick_of(69));
}
/// Replace With Clip ▸ From Bin restarts the clip at the new item's In point (as Premiere does);
/// `keepSourceIn` keeps the clip's place in the media instead.
#[test]
@ -864,6 +894,29 @@ fn color_matte_color_changes_and_undoes() {
assert!(s.execute("project.matteColor", json!({"item": matte, "color": "green"})).is_err());
}
/// #918: non-ripple Speed/Duration 200% halves a 100-frame clip; a 10-frame fade-out must
/// move from frames 90–100 to 40–50 (the new clip end), not stay past the clip.
#[test]
fn speed_duration_moves_fade_out_to_the_new_clip_end() {
let mut s = Session::default();
s.execute("file.newSequence", json!({"name": "Speed fade", "video": 1, "audio": 0})).unwrap();
let rate = s.sequence_rate();
let seconds = rate.tick_of(100).seconds();
let matte = s.execute("file.newColorMatte", json!({"color": "#ff0000", "seconds": seconds})).unwrap()["item"].as_u64().unwrap();
let c = s.execute("timeline.place", json!({"item": matte, "track": "V1", "seconds": 0.0})).unwrap()["clips"][0].as_u64().unwrap();
select(&mut s, &[c]);
s.execute("sequence.applyVideoTransition", json!({"edge": "out", "frames": 10})).unwrap();
let before = s.active_sequence().unwrap().video_tracks[0].transitions[0].clone();
assert_eq!((rate.frame_at(before.start), rate.frame_at(before.end())), (90, 100));
s.execute("clip.speedDuration", json!({"speed": 200})).unwrap();
let it = clip(&s, c);
let fade = &s.active_sequence().unwrap().video_tracks[0].transitions[0];
assert_eq!((rate.frame_at(it.start), rate.frame_at(it.end())), (0, 50));
assert_eq!((rate.frame_at(fade.start), rate.frame_at(fade.end())), (40, 50));
assert_eq!(fade.from, Some(filmcraft_project::ClipId(c)));
assert_eq!(fade.to, None);
}
/// A damaged project whose subclip is its own parent (#66): `media_duration` followed the chain
/// without a bound and overflowed the stack. It gives up after a few hops now.
#[test]
@ -984,3 +1037,25 @@ fn keyboard_slip_keeps_linked_clips_in_sync_past_the_shorter_handle() {
s.execute("timeline.slipLeft5", json!({})).unwrap();
assert_eq!((clip(&s, v).source_in, clip(&s, a).source_in), (Tick(fr.0 * 30), Tick(fr.0 * 35)));
}
#[test]
fn linked_rate_stretch_keeps_one_speed_when_a_neighbour_blocks_the_sound() {
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Stretch"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 1, "width": 16, "height": 16})).unwrap();
let r = s.execute("file.newOfflineFile", json!({"name": "A", "seconds": 8, "fps": 25, "video": true, "audio": true})).unwrap();
let a = r["item"].as_u64().unwrap();
let r = s.execute("file.newOfflineFile", json!({"name": "B", "seconds": 2, "fps": 25, "video": true, "audio": true})).unwrap();
let b = r["item"].as_u64().unwrap();
let fr = s.sequence_rate().tick_of(1);
let r = s.execute("timeline.place", json!({"item": a, "track": "V1", "audioTrack": "A1", "time": 0, "duration": fr.0 * 60})).unwrap();
let (v, snd) = (r["clips"][0].as_u64().unwrap(), r["clips"][1].as_u64().unwrap());
s.execute("sequence.linkedSelection", json!({"on": false})).unwrap();
s.execute("timeline.place", json!({"item": b, "track": "A1", "time": fr.0 * 70, "duration": fr.0 * 10})).unwrap();
s.execute("sequence.linkedSelection", json!({"on": true})).unwrap();
let r = s.execute("timeline.rateStretch", json!({"clip": v, "edge": "out", "deltaFrames": 20})).unwrap();
let (pic, sound) = (clip(&s, v), clip(&s, snd));
assert_eq!((pic.duration, sound.duration), (Tick(fr.0 * 70), Tick(fr.0 * 70)));
assert!((pic.speed - sound.speed).abs() < 1e-12);
assert!((r["speed"].as_f64().unwrap() - 60.0 / 70.0).abs() < 1e-9);
}

696
crates/engine/src/commands.rs
View File

File diff suppressed because it is too large Load Diff

6
crates/engine/src/explicit_targets_tests.rs
View File

@ -234,15 +234,15 @@ fn project_delete_takes_explicit_bins_and_items_without_a_selection() {
assert_eq!(disabled(s.execute("project.delete", json!({}))), "select an item in the Project panel");
// an empty bin, by its id alone
let r = s.execute("project.delete", json!({"items": [empty]})).unwrap();
assert_eq!(r, json!({"items": 0, "bins": 1}));
assert_eq!(r, json!({"items": 0, "bins": 1, "offline": 0}));
assert!(s.project.root.find_bin(filmcraft_project::BinId(empty)).is_none());
// a bin with a clip in it, and an item, each by id alone
let r = s.execute("project.delete", json!({"items": [outer]})).unwrap();
assert_eq!(r, json!({"items": 1, "bins": 1}));
assert_eq!(r, json!({"items": 1, "bins": 1, "offline": 0}));
assert!(s.project.item(ocean).is_none());
let dunes = item_named(&s, "Desert_Dunes.mp4");
let r = s.execute("project.delete", json!({"items": [dunes.0]})).unwrap();
assert_eq!(r, json!({"items": 1, "bins": 0}));
assert_eq!(r, json!({"items": 1, "bins": 0, "offline": 0}));
assert_eq!(s.project.items.len(), before - 2);
assert!(s.state.project_selection.is_empty(), "the selection is left as it was");
assert!(!s.is_enabled("project.delete"), "menu enablement still follows the selection");

70
crates/engine/src/export_tests.rs
View File

@ -137,7 +137,7 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() {
Some(a) => (a[0].as_u64().unwrap(), a[1].as_u64().unwrap()),
None => (1920, 1080),
};
if matches!(fmt.as_str(), "png" | "tiff" | "bmp") {
if matches!(fmt.as_str(), "png" | "tiff" | "bmp" | "jpg" | "tga" | "dpx" | "exr") {
let files = files_in(&sub);
let ext = Path::new(&out).extension().unwrap().to_string_lossy().to_string();
assert_eq!(frames, 7, "frames 23..=29 at 23.976 fps");
@ -148,6 +148,10 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() {
let codec = match fmt.as_str() {
"png" => "png",
"tiff" => "tiff",
"jpg" => "mjpeg",
"tga" => "targa",
"dpx" => "dpx",
"exr" => "exr",
_ => "bmp",
};
assert_eq!(v["codec_name"], codec, "{name}");
@ -159,10 +163,11 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() {
let Some(j) = probe(&out) else { continue };
let fmt_name = j["format"]["format_name"].as_str().unwrap();
match fmt.as_str() {
"wav" | "aiff" => {
"wav" | "aiff" | "flac" => {
let a = stream(&j, "audio").unwrap();
let bits = settings["audio"]["bits"].as_u64().unwrap();
let codec = match (fmt.as_str(), bits) {
("flac", _) => "flac",
("wav", 24) => "pcm_s24le",
("wav", _) => "pcm_s16le",
(_, 24) => "pcm_s24be",
@ -174,6 +179,16 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() {
assert!((num(&j["format"]["duration"]) - 0.25).abs() < 0.01, "{name}: {}", j["format"]["duration"]);
continue;
}
"aac" => {
// AAC Audio: an MPEG-4 audio file, sound only
assert!(out.ends_with(".m4a"), "{name}: {out}");
assert!(fmt_name.contains("m4a"), "{name}: {fmt_name}");
assert_eq!(j["streams"].as_array().unwrap().len(), 1, "{name}");
let a = stream(&j, "audio").unwrap();
assert_eq!((a["codec_name"].as_str(), a["sample_rate"].as_str(), a["channels"].as_u64()), (Some("aac"), Some("48000"), Some(2)), "{name}");
assert!((num(&j["format"]["duration"]) - 0.25).abs() < 0.03, "{name}: {}", j["format"]["duration"]);
continue;
}
"gif" => {
let v = stream(&j, "video").unwrap();
assert_eq!(v["codec_name"], "gif");
@ -217,9 +232,16 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() {
"proxy" => ("Proxy", 45.0),
"lt" => ("LT", 102.0),
"standard" => ("Standard", 147.0),
"4444" => ("4444", 330.0),
"4444xq" => ("XQ", 500.0),
_ => ("HQ", 220.0),
};
assert_eq!(v["profile"], profile, "{name}");
if matches!(profile, "4444" | "XQ") {
let pix = v["pix_fmt"].as_str().unwrap_or_default();
let alpha = settings["alpha"].as_bool().unwrap_or(false);
assert!(pix.contains("444") && pix.starts_with("yuva") == alpha, "{name}: {pix}");
}
assert!(vbr <= mbps * px * 1.5, "{name}: {vbr:.1} Mb/s vs nominal {:.1}", mbps * px);
assert_eq!(a["codec_name"], "pcm_s24le", "{name}");
}
@ -271,6 +293,25 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() {
/// H.264 bitrate control over three seconds of the demo: the adaptive Match Source preset
/// targets width × height × fps × bpp, the 1080p delivery presets their fixed targets; ffprobe's
/// measured video bitrate stays within the VBR maximum and close to the target.
#[test]
fn aac_audio_keeps_a_dot_aac_path_as_adts_and_adds_m4a_otherwise() {
let mut s = demo();
let dir = Scratch::new("aac-audio");
let range = json!({"range": "custom", "startSeconds": 1.0, "endSeconds": 1.25, "wait": true, "format": "aac"});
let run = |s: &mut Session, path: String| {
let mut p = range.clone();
p["path"] = json!(path);
s.execute("file.exportMedia", p).unwrap()["path"].as_str().unwrap().to_string()
};
let adts = run(&mut s, dir.path("Mix.aac"));
assert!(adts.ends_with("Mix.aac"), "{adts}");
let b = std::fs::read(&adts).unwrap();
assert_eq!((b[0], b[1]), (0xFF, 0xF1), "an ADTS header");
let m4a = run(&mut s, dir.path("Mix"));
assert!(m4a.ends_with("Mix.m4a"), "{m4a}");
assert_eq!(&std::fs::read(&m4a).unwrap()[4..8], b"ftyp");
}
#[test]
fn h264_presets_hit_their_bitrates() {
let mut s = demo();
@ -318,7 +359,7 @@ fn user_presets_persist_with_favourites_import_and_export() {
let favs = s.execute("export.presets.list", json!({"favorites": true})).unwrap()["presets"].as_array().unwrap().len();
assert_eq!(favs, 2);
let hits = s.execute("export.presets.list", json!({"query": "prores"})).unwrap()["presets"].as_array().unwrap().len();
assert_eq!(hits, 5, "four QuickTime ProRes presets and MXF OP1a ProRes 422 HQ");
assert_eq!(hits, 8, "seven QuickTime ProRes presets and MXF OP1a ProRes 422 HQ");
// persisted: a fresh session with the same data directory sees both
let mut s2 = demo();
s2.export_presets.set_dir(&dir.0);
@ -627,3 +668,26 @@ fn default_export_dir_never_needs_a_real_filesystem() {
let s = Session { path: Some("/projects/show/edit.fcproj".into()), ..Default::default() };
assert_eq!(crate::export_tools::default_export_dir(&s), std::path::PathBuf::from("/projects/show"));
}
#[test]
fn unknown_settings_keys_are_errors_not_ignored() {
// A flat parameter put inside `settings` used to be dropped without a word: the export ran
// without loudness normalization. A misspelt nested key was dropped the same way.
let s = demo();
let err = crate::export_tools::settings_from_params(&s, &json!({"format": "h264", "settings": {"loudnessLufs": -14}}), "file.exportMedia").unwrap_err();
assert!(err.to_string().contains("`loudnessLufs`"), "{err}");
let err = crate::export_tools::settings_from_params(&s, &json!({"settings": {"effects": {"loudness": {"targetLoudness": -14}}}}), "file.exportMedia")
.unwrap_err();
assert!(err.to_string().contains("`effects.loudness.targetLoudness`"), "{err}");
// Both spellings of a snake_case field are known (#531), and null sets nothing.
for key in ["target_lufs", "targetLufs"] {
let (_, ok) = crate::export_tools::settings_from_params(
&s,
&json!({"settings": {"effects": {"loudness": {"enabled": true, key: -14}}, "captionSidecar": null}}),
"file.exportMedia",
)
.unwrap();
assert!(ok.effects.loudness.enabled);
assert_eq!(ok.effects.loudness.target_lufs, -14.0, "{key}");
}
}

63
crates/engine/src/export_tools.rs
View File

@ -159,6 +159,26 @@ fn merge(base: &mut Value, patch: &Value) {
}
}
/// The first key of `patch` (as a dotted path) that the resolved settings `known` don't have,
/// matched as [`merge`] matches them (the camelCase spelling of a snake_case field counts).
/// A null value is accepted anywhere: it sets nothing.
fn unknown_key(known: &Value, patch: &Value, prefix: &str) -> Option<String> {
let (Value::Object(k), Value::Object(p)) = (known, patch) else { return None };
for (key, v) in p {
let path = if prefix.is_empty() { key.clone() } else { format!("{prefix}.{key}") };
match k.get(key).or_else(|| k.get(&camel_to_snake(key))) {
None if !v.is_null() => return Some(path),
Some(kv) => {
if let Some(found) = unknown_key(kv, v, &path) {
return Some(found);
}
}
None => {}
}
}
None
}
/// A frame rate from a number of frames per second (29.97 → 30000/1001) or "num/den".
pub fn parse_rate(v: &Value) -> Option<FrameRate> {
if let Some(s) = v.as_str() {
@ -191,6 +211,12 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
let mut v = serde_json::to_value(&settings).map_err(|e| EngineError::Other(e.to_string()))?;
merge(&mut v, patch);
settings = serde_json::from_value(v).map_err(|e| bad(cmd, format!("settings: {e}")))?;
// serde ignores keys it doesn't know, so a misspelt or misplaced key (a flat parameter
// such as `loudnessLufs` put inside `settings`) would silently do nothing.
let known = serde_json::to_value(&settings).map_err(|e| EngineError::Other(e.to_string()))?;
if let Some(path) = unknown_key(&known, patch, "") {
return Err(bad(cmd, format!("settings: unknown key `{path}` (see export.presets.get for the settings' keys)")));
}
}
// flat overrides (the original file.exportMedia parameters and a few common ones)
if let Some(f) = str_p(p, "format") {
@ -260,6 +286,12 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option
if let Some(v) = str_p(p, "apvProfile") {
settings.apv_profile = v.to_string();
}
if let Some(v) = str_p(p, "audioCodec") {
settings.audio.codec = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "audioCodec: auto | aac | pcm | flac"))?;
}
if let Some(v) = u64_p(p, "flacLevel") {
settings.audio.flac_level = v.min(8) as u8;
}
if let Some(v) = str_p(p, "mxfVideoCodec") {
settings.mxf_video_codec = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "mxfVideoCodec: dnxhr | proRes | h264"))?;
}
@ -360,7 +392,14 @@ fn file_safe(n: &str) -> String {
fn with_extension(path: &str, settings: &ExportSettings) -> String {
let ext = settings.extension();
match Path::new(path).extension().and_then(|e| e.to_str()) {
Some(e) if e.eq_ignore_ascii_case(ext) || (settings.format == Format::TiffSequence && e.eq_ignore_ascii_case("tiff")) => path.to_string(),
Some(e)
if e.eq_ignore_ascii_case(ext)
|| (settings.format == Format::TiffSequence && e.eq_ignore_ascii_case("tiff"))
|| (settings.format == Format::JpegSequence && e.eq_ignore_ascii_case("jpeg"))
|| (settings.format == Format::Aac && e.eq_ignore_ascii_case("aac")) =>
{
path.to_string()
}
Some(_) if settings.is_image_sequence() || matches!(settings.format, Format::Aiff) => format!("{}.{ext}", path.rsplit_once('.').map_or(path, |x| x.0)),
_ => format!("{path}.{ext}"),
}
@ -433,8 +472,9 @@ pub fn spawn_export(s: &mut Session, project: Arc<Project>, seq: ItemId, mut set
let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1;
let job = crate::Job { id, label, progress: Default::default(), result: Default::default() };
// Export always renders full-resolution media, whatever the proxy toggle says.
let provider = s.media.full_res_provider(project.clone(), s.services.clone());
let provider = s.media.full_res_provider(project.clone(), s.services.clone()).recording();
let (prog, res) = (job.progress.clone(), job.result.clone());
let offline = s.export_offline.entry(id).or_default().clone();
if cfg!(target_arch = "wasm32") && !wait && filmcraft_export::stepped(format) {
// No threads: the host advances the export between UI frames (`Session::pump_jobs`).
let mut exporter = filmcraft_export::Exporter::new(project, seq, &settings, &prog).map_err(|e| EngineError::Other(e.to_string()))?;
@ -445,6 +485,7 @@ pub fn spawn_export(s: &mut Session, project: Arc<Project>, seq: ItemId, mut set
}
let run = move || {
let r = filmcraft_export::export(&project, seq, &settings, &provider, &prog).map_err(|e| e.to_string());
*offline.lock().unwrap_or_else(|x| x.into_inner()) = provider.offline_used();
if let Err(e) = &r {
*prog.error.lock().unwrap_or_else(|x| x.into_inner()) = Some(e.clone());
prog.finished.store(true, Ordering::Relaxed);
@ -507,6 +548,7 @@ pub fn export_media(s: &mut Session, p: &Value) -> Result<Value> {
return Err(EngineError::Other(format!("export failed: {e}")));
}
out["result"] = v["result"].clone();
out["offlineMedia"] = offline_media_json(s, id);
}
Ok(out)
}
@ -561,6 +603,22 @@ impl ExportQueue {
}
}
/// A `jobs.list` entry: [`crate::Job::to_json`], plus `offlineMedia` for an export job, so callers
/// that poll an export (the bridge, MCP long exports) can report it like `file.exportMedia` does.
pub(crate) fn job_json(s: &Session, j: &crate::Job) -> Value {
let mut v = j.to_json();
if s.export_offline.contains_key(&j.id) {
v["offlineMedia"] = offline_media_json(s, j.id);
}
v
}
/// The offline items export job `job` drew the slate for, as JSON (`[]` while it runs or when none).
fn offline_media_json(s: &Session, job: u64) -> Value {
let list = s.export_offline.get(&job).map(|o| o.lock().unwrap_or_else(|e| e.into_inner()).clone()).unwrap_or_default();
serde_json::to_value(list).unwrap_or_else(|_| json!([]))
}
fn item_json(s: &Session, it: &QueueItem) -> Value {
let job = it.job.and_then(|id| s.jobs.iter().find(|j| j.id == id));
let (progress, status_text) = match job {
@ -581,6 +639,7 @@ fn item_json(s: &Session, it: &QueueItem) -> Value {
"statusText": status_text,
"job": it.job,
"error": it.error,
"offlineMedia": it.job.map_or_else(|| json!([]), |j| offline_media_json(s, j)),
})
}

22
crates/engine/src/file_tests.rs
View File

@ -93,3 +93,25 @@ fn project_is_named_after_its_file() {
assert_eq!(s.project.name, "Apollo 11 - Tranquility");
let _ = std::fs::remove_dir_all(&d);
}
/// Opening the demo after a saved project must not keep that path: Save would overwrite it.
#[test]
fn open_demo_project_clears_saved_path_so_save_does_not_overwrite() {
let d = temp_dir("demo-path");
let original = d.join("original.fcproj").to_string_lossy().to_string();
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Original"})).unwrap();
s.execute("file.newBin", json!({"name": "ORIGINAL"})).unwrap();
s.execute("file.saveAs", json!({"path": original})).unwrap();
s.execute("file.openDemoProject", json!({})).unwrap();
assert!(s.path.is_none(), "demo is unsaved, like New Project");
let err = s.execute("file.save", json!({})).unwrap_err().to_string();
assert!(err.contains("no path") || err.contains("Save As"), "{err}");
let mut reopen = Session::default();
reopen.execute("file.open", json!({"path": original})).unwrap();
assert!(
reopen.project.root.children.iter().any(|e| matches!(e, filmcraft_project::BinEntry::Bin(b) if b.name == "ORIGINAL")),
"original.fcproj still has the ORIGINAL bin"
);
let _ = std::fs::remove_dir_all(&d);
}

51
crates/engine/src/frame_export_tests.rs
View File

@ -101,3 +101,54 @@ fn frame_export_snapshots_are_independent_of_later_monitor_selection() {
Arc::make_mut(&mut s.project).sequence_mut(seq).unwrap().settings.frame_rate = FrameRate { num: 1, den: i64::MAX };
assert!(keyboard::frame_export_target(&s, &json!({})).is_err());
}
/// A source whose pictures are still on their way, as a WebCodecs decode or a Blob read on the web.
struct Loading(filmcraft_media::MediaInfo);
impl filmcraft_media::MediaSource for Loading {
fn info(&self) -> &filmcraft_media::MediaInfo {
&self.0
}
fn video_frame(&self, _: filmcraft_media::FrameRequest) -> filmcraft_media::Result<Arc<filmcraft_frame::VideoFrame>> {
filmcraft_media::pending::mark();
Err(filmcraft_media::MediaError::Decode("still decoding".into()))
}
fn audio(&self, _: i64, _: usize, _: u32) -> filmcraft_media::Result<filmcraft_frame::AudioBuffer> {
filmcraft_media::pending::mark();
Err(filmcraft_media::MediaError::Decode("still loading".into()))
}
}
#[test]
fn a_frame_still_loading_is_reported_not_saved_blank() {
let (mut s, item) = session();
let generator = s.media.cached(item).unwrap();
s.media.insert(item, Arc::new(Loading(generator.info().clone())));
let dir = std::env::temp_dir().join(format!("fc-frame-loading-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
for target in ["program", "source"] {
let path = dir.join(format!("{target}.png"));
let err = s.execute("file.exportFrame", json!({"target":target,"path":path.to_string_lossy()})).unwrap_err();
assert!(err.to_string().contains("still loading"), "{target}: {err}");
assert!(!path.exists(), "{target}: nothing is written");
}
// once the picture is there, the same export succeeds
s.media.insert(item, generator);
for target in ["program", "source"] {
let path = dir.join(format!("{target}.png"));
s.execute("file.exportFrame", json!({"target":target,"path":path.to_string_lossy()})).unwrap();
assert!(path.exists(), "{target}");
}
drop(s);
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn an_earlier_deferred_read_does_not_fail_the_export() {
let (mut s, _) = session();
filmcraft_media::pending::mark();
let path = std::env::temp_dir().join(format!("fc-frame-stale-pending-{}.png", std::process::id()));
s.execute("file.exportFrame", json!({"target":"program","path":path.to_string_lossy()})).unwrap();
assert!(!filmcraft_media::pending::is_set());
std::fs::remove_file(path).unwrap();
}

174
crates/engine/src/gap_selection_tests.rs Normal file
View File

@ -0,0 +1,174 @@
//! Selecting a gap between clips and closing it with Delete / Backspace / Ripple Delete (#648, #668):
//! `timeline.selectGap` selects the empty time on one track, `edit.clear` and `edit.rippleDelete`
//! close it as one "Ripple Delete" undo step.
use filmcraft_project::ClipId;
use filmcraft_time::Tick;
use serde_json::json;
use crate::{GapSelection, Session};
fn range(s: &Session, c: ClipId) -> (Tick, Tick) {
s.active_sequence().unwrap().find_item(c).map(|(_, i)| (i.start, i.end())).unwrap()
}
/// Two 48-frame clips (pictures on V1, linked sounds on A1) placed at `frames`.
fn two_clips(s: &mut Session, frames: [i64; 2]) -> [(ClipId, ClipId); 2] {
s.execute("file.openDemoProject", json!({})).unwrap();
s.execute("file.newSequence", json!({"name": "Gaps", "video": 1, "audio": 1})).unwrap();
let rate = s.sequence_rate();
let item = s.project.items.values().find(|i| i.has_video() && i.has_audio() && i.as_media().is_some()).map(|i| i.id).unwrap();
for f in frames {
s.execute("timeline.place", json!({"item": item.0, "frame": f, "sourceIn": rate.tick_of(48).0, "duration": rate.tick_of(48).0})).unwrap();
}
s.execute("edit.deselectAll", json!({})).unwrap();
let q = s.active_sequence().unwrap();
[0, 1].map(|n| (q.video_tracks[0].items[n].id, q.audio_tracks[0].items[n].id))
}
#[test]
fn clicking_between_two_clips_selects_the_gap_and_delete_closes_it() {
let mut s = Session::default();
let [(picture1, _), (picture2, sound2)] = two_clips(&mut s, [0, 96]);
let rate = s.sequence_rate();
let f = |n: i64| rate.tick_of(n);
let (f48, f60, f96) = (f(48), f(60), f(96));
s.execute("timeline.select", json!({"clips": [picture1.0]})).unwrap();
let before = s.project.clone();
let r = s.execute("timeline.selectGap", json!({"track": "V1", "time": f60.0})).unwrap();
assert_eq!(r, json!({"track": s.active_sequence().unwrap().video_tracks[0].id.0, "start": f48.0, "end": f96.0}));
assert!(s.state.selection.is_empty(), "the gap replaces the clip selection");
assert!(s.is_enabled("edit.clear") && s.is_enabled("edit.rippleDelete"), "Delete and Ripple Delete apply to the gap");
// Delete / Backspace
s.execute("edit.clear", json!({})).unwrap();
assert_eq!(range(&s, picture2), (f48, f96), "the next clip moves up into the gap");
assert_eq!(range(&s, sound2), (f48, f96), "and its sound with it");
assert_eq!(range(&s, picture1).0, Tick::ZERO, "the clip before stays");
assert!(s.state.gap_selection.is_none(), "the gap is gone");
s.active_sequence().unwrap().check().unwrap();
assert_eq!(s.undo().as_deref(), Some("Ripple Delete"), "one undo step");
assert_eq!(*s.project, *before);
}
#[test]
fn ripple_delete_closes_a_selected_gap_too() {
let mut s = Session::default();
let [_, (picture2, _)] = two_clips(&mut s, [0, 96]);
let rate = s.sequence_rate();
let f = |n: i64| rate.tick_of(n);
s.execute("timeline.selectGap", json!({"track": "A1", "frame": 50})).unwrap();
s.execute("edit.rippleDelete", json!({})).unwrap();
assert_eq!(range(&s, picture2).0, f(48));
}
#[test]
fn the_empty_time_before_the_first_clip_is_a_gap() {
let mut s = Session::default();
let [(picture1, _), (picture2, _)] = two_clips(&mut s, [24, 72]);
let rate = s.sequence_rate();
let f = |n: i64| rate.tick_of(n);
let r = s.execute("timeline.selectGap", json!({"track": "V1", "frame": 5})).unwrap();
assert_eq!((r["start"].clone(), r["end"].clone()), (json!(0), json!(f(24).0)));
s.execute("edit.clear", json!({})).unwrap();
assert_eq!((range(&s, picture1).0, range(&s, picture2).0), (Tick::ZERO, f(48)));
}
#[test]
fn empty_time_after_the_last_clip_is_not_a_gap() {
let mut s = Session::default();
two_clips(&mut s, [0, 96]);
assert!(s.execute("timeline.selectGap", json!({"track": "V1", "frame": 400})).is_err());
assert!(s.execute("timeline.selectGap", json!({"track": "V1", "frame": 10})).is_err(), "inside a clip");
assert!(s.state.gap_selection.is_none());
assert!(!s.is_enabled("edit.clear"), "nothing to delete");
}
#[test]
fn selecting_a_clip_or_deselecting_all_drops_the_gap() {
let mut s = Session::default();
let [(picture1, _), _] = two_clips(&mut s, [0, 96]);
s.execute("timeline.selectGap", json!({"track": "V1", "frame": 60})).unwrap();
s.execute("timeline.select", json!({"clips": [picture1.0]})).unwrap();
assert!(s.state.gap_selection.is_none());
s.execute("timeline.selectGap", json!({"track": "V1", "frame": 60})).unwrap();
s.execute("edit.deselectAll", json!({})).unwrap();
assert!(s.state.gap_selection.is_none());
}
/// An edit that fills the gap leaves no gap selected; Delete then removes nothing it should not.
#[test]
fn an_edit_that_changes_the_gap_drops_it() {
let mut s = Session::default();
let [(picture1, _), (picture2, _)] = two_clips(&mut s, [0, 96]);
let rate = s.sequence_rate();
let f = |n: i64| rate.tick_of(n);
s.execute("timeline.selectGap", json!({"track": "V1", "frame": 60})).unwrap();
s.execute("timeline.trim", json!({"clip": picture1.0, "edge": "out", "mode": "regular", "deltaFrames": 12})).unwrap();
assert!(s.state.gap_selection.is_none(), "the gap changed");
assert!(s.execute("edit.clear", json!({})).is_err());
assert_eq!(range(&s, picture2).0, f(96), "nothing moved");
}
/// A stale selection (set from outside, as the control channel can) is refused, not acted on.
#[test]
fn a_stale_gap_is_refused() {
let mut s = Session::default();
let [_, (picture2, _)] = two_clips(&mut s, [0, 96]);
let rate = s.sequence_rate();
let f = |n: i64| rate.tick_of(n);
let track = s.active_sequence().unwrap().video_tracks[0].id;
s.state.gap_selection = Some(GapSelection { track, range: filmcraft_time::TimeRange::from_bounds(f(10), f(20)) });
assert!(s.execute("edit.clear", json!({})).is_err());
assert!(s.state.gap_selection.is_none());
assert_eq!(range(&s, picture2).0, f(96));
}
#[test]
fn a_gap_on_a_locked_track_cannot_be_selected_or_closed() {
let mut s = Session::default();
let [_, (picture2, _)] = two_clips(&mut s, [0, 96]);
let rate = s.sequence_rate();
let f = |n: i64| rate.tick_of(n);
s.execute("timeline.setTrack", json!({"track": "V1", "locked": true})).unwrap();
assert!(s.execute("timeline.selectGap", json!({"track": "V1", "frame": 60})).is_err());
// Sequence ▸ Close Gap moved the other tracks while the locked one stayed put
assert!(s.execute("sequence.closeGap", json!({"track": "V1", "frame": 60})).is_err());
assert_eq!(range(&s, picture2).0, f(96));
let sound2 = s.active_sequence().unwrap().audio_tracks[0].items[1].id;
assert_eq!(range(&s, sound2).0, f(96), "the unlocked sound stays in sync");
}
#[test]
fn hostile_select_gap_parameters_are_errors() {
let mut s = Session::default();
assert!(s.execute("timeline.selectGap", json!({"track": "V1", "frame": 1})).is_err(), "no sequence");
two_clips(&mut s, [0, 96]);
for p in [
json!({}),
json!({"track": "V1"}),
json!({"frame": 60}),
json!({"track": "V99", "frame": 60}),
json!({"track": 999_999, "frame": 60}),
json!({"track": "V1", "time": i64::MAX}),
json!({"track": "V1", "time": i64::MIN}),
json!({"track": "V1", "frame": i64::MAX}),
json!({"track": "V1", "seconds": f64::MAX}),
json!({"track": [], "frame": "x"}),
] {
let _ = s.execute("timeline.selectGap", p);
}
assert!(s.state.gap_selection.is_none());
}
#[test]
fn the_gap_selection_is_part_of_the_ui_state() {
let mut s = Session::default();
two_clips(&mut s, [0, 96]);
s.execute("timeline.selectGap", json!({"track": "V1", "frame": 60})).unwrap();
let v = serde_json::to_value(&s.state).unwrap();
let back: crate::EditorState = serde_json::from_value(v).unwrap();
assert_eq!(back.gap_selection, s.state.gap_selection);
}

11
crates/engine/src/image_sequence_tests.rs
View File

@ -11,6 +11,17 @@ use serde_json::{Value, json};
use crate::Session;
use crate::media_test_util::tmp_dir;
#[test]
fn import_accepts_uppercase_extensions() {
let dir = tmp_dir("import-uppercase");
let wav = dir.join("TONE.WAV");
std::fs::write(&wav, filmcraft_media::wav::write_wav16(&[0.1; 4800], 2, 48_000)).unwrap();
let mut s = Session::default();
let r = s.execute("file.import", json!({"paths": [wav.to_string_lossy()]})).unwrap();
assert_eq!(r["items"].as_array().unwrap().len(), 1);
assert!(s.project.items.values().any(|it| it.name == "TONE.WAV"));
}
#[test]
fn import_replace_starts_from_an_empty_project() {
let dir = tmp_dir("import-replace");

11
crates/engine/src/keyboard.rs
View File

@ -631,13 +631,20 @@ fn export_frame(s: &mut Session, p: &Value) -> Result<Value> {
}
};
let provider = s.media.full_res_provider(s.project.clone(), s.services.clone());
// On the web a frame can still be decoding (WebCodecs) or its bytes loading (Blob reads): the
// render then leaves that clip out, so saving it would pass a blank or partial picture off as
// the requested frame (#228). Report it instead and write nothing; a later call succeeds.
let _ = filmcraft_media::pending::take();
let img = if target.source {
filmcraft_render::render_item(&s.project, target.item, target.time, 1.0, &provider)
.ok_or_else(|| bad("file.exportFrame", "cannot decode the Source video frame"))?
} else {
let opts = filmcraft_render::RenderOptions { scale: 1.0, captions: true, ..Default::default() };
filmcraft_render::render_sequence(&s.project, target.item, target.time, opts, &provider)
Some(filmcraft_render::render_sequence(&s.project, target.item, target.time, opts, &provider))
};
if filmcraft_media::pending::take() {
return Err(EngineError::Other("the frame is still loading; export it again in a moment".into()));
}
let img = img.ok_or_else(|| bad("file.exportFrame", "cannot decode the Source video frame"))?;
let (w, h) = (
u32::try_from(img.w).map_err(|_| bad("file.exportFrame", "image width exceeds limits"))?,
u32::try_from(img.h).map_err(|_| bad("file.exportFrame", "image height exceeds limits"))?,

183
crates/engine/src/lib.rs
View File

@ -27,6 +27,7 @@ mod marker_export;
pub mod masks;
pub mod media_browser;
pub mod media_pool;
pub mod media_watch;
pub mod mixer;
pub mod multicam;
pub mod narration;
@ -58,7 +59,7 @@ use std::sync::Arc;
use filmcraft_edit::EditCtx;
use filmcraft_project::{ClipId, ItemId, Project, Sequence, TrackId, TrackKind};
use filmcraft_time::{FrameRate, Tick};
use filmcraft_time::{FrameRate, Tick, TimeRange};
use serde::{Deserialize, Serialize};
use serde_json::Value;
@ -191,6 +192,10 @@ impl Services for FsServices {
}
}
/// A point in the undo history (its top step), from [`Session::undo_mark`].
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct UndoMark(Option<usize>);
/// Undo history of whole-project snapshots.
#[derive(Clone, Default)]
pub struct History {
@ -222,6 +227,36 @@ pub struct Targeting {
pub audio_dest: Option<TrackId>,
}
/// A gap on one track of the active sequence (see [`EditorState::gap_selection`]).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct GapSelection {
pub track: TrackId,
pub range: TimeRange,
}
/// An effect of a clip (`effects.select`): its id and which of the clip's effects with that id it
/// is (0 = the first). Not its index, which applying an effect shifts (standard effects go first).
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EffectSel {
pub clip: ClipId,
pub effect: String,
pub instance: usize,
}
impl EffectSel {
/// The selection of `effects[index]` of `clip`.
pub fn of(clip: ClipId, effects: &[filmcraft_project::EffectInstance], index: usize) -> Option<Self> {
let id = &effects.get(index)?.effect;
let instance = effects.iter().take(index).filter(|e| &e.effect == id).count();
Some(Self { clip, effect: id.clone(), instance })
}
/// Where the selected effect is in the clip's `effects`.
pub fn index(&self, effects: &[filmcraft_project::EffectInstance]) -> Option<usize> {
effects.iter().enumerate().filter(|(_, e)| e.effect == self.effect).nth(self.instance).map(|(i, _)| i)
}
}
/// Editing state that commands depend on (not project data, but headless-relevant).
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct EditorState {
@ -231,6 +266,10 @@ pub struct EditorState {
/// Item loaded in the Source monitor and its playhead (media time).
pub source_item: Option<ItemId>,
pub source_playhead: Tick,
/// Clips opened in the Source monitor, most recent first: the Source panel's list of recent
/// clips (#313). Kept up to date after every command; items gone from the project drop out.
#[serde(default)]
pub source_history: Vec<ItemId>,
/// Selected timeline items.
pub selection: Vec<ClipId>,
/// Selected project panel items.
@ -267,12 +306,23 @@ pub struct EditorState {
/// them. Selecting clips clears it and selecting transitions clears the clip selection.
#[serde(default)]
pub transition_selection: Vec<filmcraft_project::TransitionId>,
/// The selected gap (#648, #668): empty time on one track, clicked in the Timeline, from the
/// end of a clip (or the sequence start) up to the next clip. Delete, Backspace and Ripple
/// Delete close it. Selecting clips, transitions or edit points clears it, and so does an edit
/// that changes the gap.
#[serde(default)]
pub gap_selection: Option<GapSelection>,
/// Selected layers (indices among the graphic layers, 0 = back) of the selected graphic clip.
#[serde(default)]
pub graphic_layers: Vec<usize>,
/// The mask being edited on the Program monitor (Effect Controls selection).
#[serde(default)]
pub selected_mask: Option<masks::MaskSel>,
/// The effect selected in Effect Controls (its header or one of its properties): Motion and
/// Transform show their handles on the Program monitor. Selecting a mask clears it, and
/// selecting an effect clears the mask selection.
#[serde(default)]
pub selected_effect: Option<EffectSel>,
/// Multi-Camera Audio Follows Video: switching a multi-camera clip's angle switches its linked
/// audio clips too.
#[serde(default)]
@ -339,6 +389,9 @@ pub struct Session {
pub journal: Vec<(String, Value)>,
/// Background jobs (exports, render previews…).
pub jobs: Vec<Job>,
/// Export job → the offline items it drew the slate for, filled when the export finishes
/// (#1054; `offlineMedia` in `file.exportMedia` and queue results).
pub export_offline: std::collections::HashMap<u64, Arc<std::sync::Mutex<Vec<media_pool::OfflineMedia>>>>,
/// User preferences (`prefs.*` commands) and where they persist (None = not persisted).
pub prefs: autosave::Preferences,
pub prefs_path: Option<std::path::PathBuf>,
@ -388,6 +441,11 @@ pub struct Session {
pub browser: media_browser::BrowserState,
/// Nesting depth of [`Session::execute`] (commands that run other commands).
exec_depth: u32,
/// Each media file's size and modification time when last seen (Settings ▸ Media ▸
/// Automatically refresh growing files, [`media_watch`]).
pub media_stamps: std::collections::HashMap<String, media_watch::Stamp>,
/// The stamps being taken on a worker thread ([`Session::start_media_scan`]).
pub media_scan: Option<media_watch::MediaScan>,
}
/// An export advanced a batch at a time on the host's thread ([`Session::pump_jobs`]).
@ -453,6 +511,7 @@ impl Session {
events: Vec::new(),
journal: Vec::new(),
jobs: Vec::new(),
export_offline: Default::default(),
prefs: Default::default(),
prefs_path: None,
persistence: None,
@ -477,6 +536,8 @@ impl Session {
log: Default::default(),
browser: Default::default(),
exec_depth: 0,
media_stamps: Default::default(),
media_scan: None,
}
}
@ -646,6 +707,9 @@ impl Session {
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (spec.run)(self, &params)))
.unwrap_or_else(|_| Err(EngineError::Other(format!("internal error in `{id}` (see the crash log); the command did not complete"))));
self.exec_depth -= 1;
if self.exec_depth == 0 {
source_monitor::track_history(self);
}
// source graphics: an edited instance updates the shared layers and the other instances
if r.is_ok() && spec.journal && id.starts_with("graphics.") && !self.project.source_graphics.is_empty() {
graphic_templates::sync_source_graphics(self);
@ -780,6 +844,7 @@ impl Session {
self.history.redo.clear();
self.history.merge_key = None;
self.state = st;
self.drop_stale_gap();
self.bump();
Ok(r)
}
@ -799,10 +864,82 @@ impl Session {
self.refuse_self_nesting(&p)?;
self.project = Arc::new(p);
self.state = st;
self.drop_stale_gap();
self.bump();
Ok(r)
}
/// Run several undoable edits as one undo step labelled `label`. If `f` fails, every edit it made
/// is rolled back and the history is as before.
pub fn edit_group<R>(&mut self, label: &str, f: impl FnOnce(&mut Session) -> Result<R>) -> Result<R> {
let (project, state, undo, redo, limit) =
(self.project.clone(), self.state.clone(), self.history.undo.len(), self.history.redo.clone(), self.history.limit);
// keep every step the group makes, however many, so the first one can be found afterwards
self.history.limit = usize::MAX;
// a panic inside the group must not leave the history unlimited: restore, then let it go on
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(self)));
self.history.limit = limit;
let first = self.history.undo.get(undo).map(|e| e.1.clone());
let rolled_back = |s: &mut Session| {
s.history.undo.truncate(undo);
s.history.redo = redo.clone();
s.history.merge_key = None;
s.project = project.clone();
s.state = state.clone();
s.bump();
};
match (r, first) {
(Ok(Ok(v)), Some(first)) => {
self.history.undo.truncate(undo);
self.history.undo.push((label.to_string(), first));
if self.history.undo.len() > limit {
self.history.undo.remove(0);
}
self.history.merge_key = None;
Ok(v)
}
(Ok(Ok(v)), None) => Ok(v),
(Ok(Err(e)), _) => {
rolled_back(self);
Err(e)
}
(Err(panic), _) => {
rolled_back(self);
std::panic::resume_unwind(panic)
}
}
}
/// Where the undo history stands now, for [`Session::revert_to`].
pub fn undo_mark(&self) -> UndoMark {
UndoMark(self.history.undo.last().map(|u| Arc::as_ptr(&u.1) as usize))
}
/// Take back every edit made since `mark` (a gesture cancelled with Escape, #580), merged or
/// not: the project returns to how it was at `mark`, and neither undo nor redo keeps them.
/// Nothing happens when `mark` has left the history. Returns how many steps were taken back.
pub fn revert_to(&mut self, mark: UndoMark) -> usize {
if let UndoMark(Some(p)) = mark
&& !self.history.undo.iter().any(|u| Arc::as_ptr(&u.1) as usize == p)
{
return 0;
}
let mut earliest = None;
let mut n = 0;
while self.undo_mark() != mark {
let Some((_, prev)) = self.history.undo.pop() else { break };
earliest = Some(prev);
n += 1;
}
if let Some(prev) = earliest {
self.project = prev;
self.history.merge_key = None;
self.fix_state();
self.bump();
}
n
}
/// Edit the active sequence with the edit-algebra context.
pub fn edit_sequence<R>(&mut self, label: &str, f: impl FnOnce(&mut Sequence, &mut EditCtx, &mut EditorState) -> Result<R>) -> Result<R> {
self.edit_sequence_as(label, None, f)
@ -872,6 +1009,11 @@ impl Session {
self.state.selected_mask = None;
}
}
if let Some(e) = &self.state.selected_effect
&& self.active_sequence().and_then(|q| q.find_item(e.clip)).and_then(|(_, it)| e.index(&it.effects)).is_none()
{
self.state.selected_effect = None;
}
if let Some(s) = self.state.active_sequence
&& p.sequence(s).is_none()
{
@ -884,6 +1026,7 @@ impl Session {
{
self.state.source_item = None;
}
source_monitor::track_history(self);
if let Some(seq) = self.state.active_sequence.and_then(|s| p.sequence(s)) {
self.state.selection.retain(|c| seq.find_item(*c).is_some());
self.state.caption_selection.retain(|c| seq.find_caption(*c).is_some());
@ -892,6 +1035,17 @@ impl Session {
self.state.caption_selection.clear();
}
self.state.project_selection.retain(|i| p.item(*i).is_some());
self.drop_stale_gap();
}
/// Forget the selected gap when it is no longer a gap of the active sequence (a clip moved
/// into it or next to it, its track went, another sequence became the active one).
fn drop_stale_gap(&mut self) {
if let Some(g) = self.state.gap_selection
&& !self.active_sequence().and_then(|q| q.track(g.track)).is_some_and(|tr| filmcraft_edit::through::track_gaps(tr).contains(&g.range))
{
self.state.gap_selection = None;
}
}
fn bump(&mut self) {
@ -982,11 +1136,22 @@ impl Session {
/// [`Session::render_program_at`], with the reason when nothing can be rendered.
pub fn try_render_program_at(&self, scale: f32, t: Tick) -> Result<filmcraft_render::Image> {
self.render_program_with(scale, t, &self.media.provider(self.project.clone(), self.services.clone()))
}
/// [`Session::try_render_program_at`], also naming the offline items the frame drew the slate
/// for (#1054).
pub fn try_render_program_with_offline_at(&self, scale: f32, t: Tick) -> Result<(filmcraft_render::Image, Vec<media_pool::OfflineMedia>)> {
let provider = self.media.provider(self.project.clone(), self.services.clone()).recording();
let img = self.render_program_with(scale, t, &provider)?;
Ok((img, provider.offline_used()))
}
fn render_program_with(&self, scale: f32, t: Tick, provider: &dyn filmcraft_render::SourceProvider) -> Result<filmcraft_render::Image> {
let seq = self.renderable_sequence(scale)?;
let t = self.sequence_rate().snap(t.max(Tick::ZERO));
let provider = self.media.provider(self.project.clone(), self.services.clone());
let opts = filmcraft_render::RenderOptions { scale, captions: true, ..Default::default() };
Ok(filmcraft_render::render_sequence(&self.project, seq, t, opts, &provider))
Ok(filmcraft_render::render_sequence(&self.project, seq, t, opts, provider))
}
/// The active sequence, if a frame of it at `scale` is within the image size limits.
@ -1050,6 +1215,8 @@ mod audio_placement_tests;
#[cfg(test)]
mod autosave_tests;
#[cfg(test)]
mod avi_import_tests;
#[cfg(test)]
mod clip_ops_tests;
#[cfg(test)]
mod color_tests;
@ -1062,6 +1229,8 @@ mod export_tests;
#[cfg(test)]
mod file_tests;
#[cfg(test)]
mod gap_selection_tests;
#[cfg(test)]
mod image_sequence_tests;
#[cfg(test)]
mod interchange_auto_points_tests;
@ -1070,12 +1239,16 @@ mod keyboard_tests;
#[cfg(test)]
mod linked_speed_tests;
#[cfg(test)]
mod locked_clip_edits_tests;
#[cfg(test)]
mod masks_tests;
#[cfg(test)]
mod media_browser_tests;
#[cfg(test)]
mod media_test_util;
#[cfg(test)]
mod media_watch_tests;
#[cfg(test)]
mod mixer_tests;
#[cfg(test)]
mod multicam_tests;
@ -1094,6 +1267,8 @@ mod par_tests;
#[cfg(test)]
mod paste_media_tests;
#[cfg(test)]
mod place_items_tests;
#[cfg(test)]
mod presets_tests;
#[cfg(test)]
mod previews_tests;
@ -1110,6 +1285,8 @@ mod remix_tests;
#[cfg(test)]
mod ripple_delete_tests;
#[cfg(test)]
mod roll_linked_tests;
#[cfg(test)]
mod scopes_tests;
#[cfg(test)]
mod sequence_inspect_tests;

84
crates/engine/src/locked_clip_edits_tests.rs Normal file
View File

@ -0,0 +1,84 @@
//! #911: a locked track refuses timing and source-range edits the way a locked destination
//! refuses `timeline.move`.
use super::*;
use filmcraft_project::ClipId;
use serde_json::json;
fn fixture() -> (Session, u64) {
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Lock check"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 0, "width": 16, "height": 16})).unwrap();
s.execute("file.newOfflineFile", json!({"name": "Media", "seconds": 4, "fps": 25, "video": true, "audio": false, "width": 16, "height": 16})).unwrap();
let item = s.project.items.values().find(|i| i.as_media().is_some()).map(|i| i.id.0).unwrap();
s.execute("timeline.place", json!({"item": item, "track": "V1", "time": 0, "sourceIn": 0, "duration": 1_016_064_000_000i64})).unwrap();
let clip = s.active_sequence().unwrap().video_tracks[0].items[0].id.0;
(s, clip)
}
fn v1_clip(s: &Session, id: u64) -> filmcraft_project::TrackItem {
s.active_sequence().unwrap().find_item(ClipId(id)).unwrap().1.clone()
}
/// Place a 100-frame clip, lock V1, trim Out by −20 frames: the command errors and duration stays 100.
#[test]
fn locked_track_refuses_trim_out_and_keeps_duration() {
let (mut s, clip) = fixture();
let before = v1_clip(&s, clip);
assert_eq!(s.sequence_rate().frame_at(before.duration), 100);
s.execute("timeline.setTrack", json!({"track": "V1", "locked": true})).unwrap();
let err = s.execute("timeline.trim", json!({"clip": clip, "edge": "out", "mode": "regular", "deltaFrames": -20})).unwrap_err();
assert!(err.to_string().contains("locked"), "{err}");
let after = v1_clip(&s, clip);
assert_eq!(after.duration, before.duration);
assert_eq!(after.start, before.start);
assert_eq!(after.source_in, before.source_in);
assert_eq!(after.speed, before.speed);
}
#[test]
fn unlocked_track_still_trims_out() {
let (mut s, clip) = fixture();
s.execute("timeline.trim", json!({"clip": clip, "edge": "out", "mode": "regular", "deltaFrames": -20})).unwrap();
assert_eq!(s.sequence_rate().frame_at(v1_clip(&s, clip).duration), 80);
}
#[test]
fn locked_track_refuses_move_on_the_same_track() {
let (mut s, clip) = fixture();
let before = v1_clip(&s, clip);
s.execute("timeline.setTrack", json!({"track": "V1", "locked": true})).unwrap();
let err = s.execute("timeline.move", json!({"moves": [{"clip": clip, "track": "V1", "time": 1_016_064_000_000i64}], "linked": false})).unwrap_err();
assert!(err.to_string().contains("locked"), "{err}");
assert_eq!(v1_clip(&s, clip).duration, before.duration);
}
#[test]
fn locked_track_refuses_roll_slip_slide_speed() {
let mut s = Session::default();
s.execute("file.newProject", json!({"name": "Lock check"})).unwrap();
s.execute("file.newSequence", json!({"name": "Sequence", "fps": 25, "video": 1, "audio": 0, "width": 16, "height": 16})).unwrap();
s.execute("file.newOfflineFile", json!({"name": "Media", "seconds": 4, "fps": 25, "video": true, "audio": false, "width": 16, "height": 16})).unwrap();
let item = s.project.items.values().find(|i| i.as_media().is_some()).map(|i| i.id.0).unwrap();
let frame = 10_160_640_000i64;
s.execute("timeline.place", json!({"item": item, "track": "V1", "time": 0, "sourceIn": frame * 10, "duration": frame * 60})).unwrap();
s.execute("timeline.place", json!({"item": item, "track": "V1", "time": frame * 60, "sourceIn": 0, "duration": frame * 100})).unwrap();
let (left, right) = {
let q = s.active_sequence().unwrap();
(q.video_tracks[0].items[0].id.0, q.video_tracks[0].items[1].id.0)
};
s.execute("timeline.setTrack", json!({"track": "V1", "locked": true})).unwrap();
let snap = s.project.clone();
for cmd in [
json!({"id": "timeline.roll", "p": {"left": left, "right": right, "deltaFrames": 10}}),
json!({"id": "timeline.slip", "p": {"clip": left, "deltaFrames": -10}}),
json!({"id": "timeline.slide", "p": {"clip": left, "deltaFrames": 10}}),
json!({"id": "timeline.rateStretch", "p": {"clip": left, "edge": "out", "deltaFrames": 20}}),
json!({"id": "clip.speedDuration", "p": {"clips": [left], "speed": 200}}),
json!({"id": "timeline.trim", "p": {"clip": left, "edge": "in", "mode": "regular", "deltaFrames": 20}}),
] {
let err = s.execute(cmd["id"].as_str().unwrap(), cmd["p"].clone()).unwrap_err();
assert!(err.to_string().contains("locked"), "{}: {err}", cmd["id"]);
assert_eq!(*s.project, *snap, "{} must leave the sequence unchanged", cmd["id"]);
}
}

55
crates/engine/src/masks.rs
View File

@ -38,7 +38,7 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
spec(
"masks.add",
"Create Mask",
r#"{"clip":id?,"effect":index|id?="opacity","shape":"ellipse"|"polygon"|"bezier","path":[[x,y]…]|{vertices}?,"center":[x,y]?,"size":[w,h]?}"#,
r#"{"clip":id?,"effect":index|id?="opacity","shape":"ellipse"|"polygon"|"rectangle"|"bezier","path":[[x,y]…]|{vertices}?,"center":[x,y]?,"size":[w,h]?,"cornerRadius":px?}"#,
add,
),
spec("masks.remove", "Delete Mask", r#"{"clip":id?,"effect":index|id?,"mask":n?}"#, remove),
@ -64,6 +64,7 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
track,
),
spec("masks.toggleVertexSmooth", "Convert Mask Vertex", r#"{"clip":id?,"effect":index|id?,"mask":n?,"vertex":n}"#, toggle_smooth),
spec("masks.roundCorners", "Round Mask Corners", r#"{"clip":id?,"effect":index|id?,"mask":n?,"radius":px,"vertices":[n]?}"#, round_corners),
CommandSpec {
id: "masks.select",
label: "Select Mask",
@ -90,9 +91,12 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
/// A mask path from JSON: the serialized [`MaskPath`] object, an array of vertices
/// (`{"p":[x,y],"in":[x,y]?,"out":[x,y]?}`), or an array of `[x, y]` / `{"x","y"}` points.
pub fn path_from_json(v: &Value) -> Option<MaskPath> {
if let Ok(p) = serde_json::from_value::<MaskPath>(v.clone()) {
return Some(p);
}
// The vertex forms first: serde also accepts `{"p","in","out"}` vertices (the `masks.list` /
// `masks.add` form) but silently drops the unknown `in` / `out` tangents.
vertex_path(v).or_else(|| serde_json::from_value::<MaskPath>(v.clone()).ok())
}
fn vertex_path(v: &Value) -> Option<MaskPath> {
let arr = v.as_array().or_else(|| v.get("vertices").and_then(Value::as_array))?;
let closed = v.get("closed").and_then(Value::as_bool).unwrap_or(true);
let mut vertices = Vec::new();
@ -212,7 +216,12 @@ fn add(s: &mut Session, p: &Value) -> Result<Value> {
None => match shape.as_str() {
"ellipse" => MaskPath::ellipse(center, size * 0.5),
"polygon" | "rectangle" | "rect" | "4-point" => {
MaskPath::rect(center.x - size.x / 2.0, center.y - size.y / 2.0, center.x + size.x / 2.0, center.y + size.y / 2.0)
let (x0, y0, x1, y1) = (center.x - size.x / 2.0, center.y - size.y / 2.0, center.x + size.x / 2.0, center.y + size.y / 2.0);
match f64_p(p, "cornerRadius") {
Some(r) if !r.is_finite() || r < 0.0 => return Err(bad("masks.add", "`cornerRadius` is pixels ≥ 0")),
Some(r) => MaskPath::rounded_rect(x0, y0, x1, y1, r),
None => MaskPath::rect(x0, y0, x1, y1),
}
}
_ => return Err(bad("masks.add", "a bezier mask needs `path` (pen points)")),
},
@ -238,6 +247,7 @@ fn add(s: &mut Session, p: &Value) -> Result<Value> {
e.masks.push(Mask::new(name.clone(), path));
let mi = e.masks.len() - 1;
st.selected_mask = Some(MaskSel { clip, effect: ei, mask: mi });
st.selected_effect = None;
Ok((ei, mi, name))
})?;
Ok(json!({"clip": clip.0, "effect": ei, "mask": mi, "name": name}))
@ -454,6 +464,40 @@ fn toggle_smooth(s: &mut Session, p: &Value) -> Result<Value> {
Ok(Value::Null)
}
/// Round the mask's sharp corners (Premiere 26's rounded rectangle masks): every corner between
/// two straight edges (or only the listed `vertices`) becomes a circular fillet of `radius`
/// pixels, shrunk to fit half of each edge. The same corners are rounded in every Mask Path
/// keyframe so an animated path keeps interpolating. Returns how many corners were rounded.
fn round_corners(s: &mut Session, p: &Value) -> Result<Value> {
const ID: &str = "masks.roundCorners";
let radius = f64_p(p, "radius").filter(|r| r.is_finite() && *r > 0.0).ok_or_else(|| bad(ID, "need `radius` (pixels, > 0)"))?;
let only: Option<Vec<usize>> = match p.get("vertices") {
None | Some(Value::Null) => None,
Some(v) => {
let list = v.as_array().ok_or_else(|| bad(ID, "`vertices` is a list of vertex indices"))?;
let mut out = list
.iter()
.map(|i| i.as_u64().and_then(|i| usize::try_from(i).ok()).ok_or_else(|| bad(ID, "`vertices` is a list of vertex indices")))
.collect::<Result<Vec<_>>>()?;
out.sort_unstable();
out.dedup();
Some(out)
}
};
let rounded = edit_mask(s, p, "Round Mask Corners", move |m, t| {
let mut corners = m.path_at(t).roundable_corners();
if let Some(only) = &only {
corners.retain(|i| only.binary_search(i).is_ok());
}
if corners.is_empty() {
return Err(bad(ID, "no sharp corner between straight edges to round"));
}
drop_all(m, t, |path| *path = path.with_rounded_corners(radius, &corners));
Ok(corners.len())
})?;
Ok(json!({"corners": rounded}))
}
fn select(s: &mut Session, p: &Value) -> Result<Value> {
if bool_p(p, "none") == Some(true) || p.is_null() || p.as_object().is_some_and(|o| o.is_empty()) {
s.state.selected_mask = None;
@ -468,6 +512,7 @@ fn select(s: &mut Session, p: &Value) -> Result<Value> {
return Err(bad("masks.select", format!("no mask {mi}")));
}
s.state.selected_mask = Some(MaskSel { clip, effect: ei, mask: mi });
s.state.selected_effect = None;
Ok(Value::Null)
}

82
crates/engine/src/masks_tests.rs
View File

@ -317,3 +317,85 @@ fn tracking_runs_in_the_background_and_can_be_cancelled() {
assert!(n < 48, "stopped early ({n} keyframes)");
assert!(s.execute("masks.track", json!({"direction": "backward"})).is_err(), "nothing before the first frame");
}
/// #813: a rectangle mask with rounded corners (created rounded, or rounded afterwards) cuts the
/// clip's corners away while the edges stay where they were; one undo step, every keyframe.
#[test]
fn rounded_rectangle_masks_cut_their_corners() {
let (mut s, clip) = demo();
// a 960×540 opacity mask in the middle of the 1920×1080 frame, rounded by 200 px
s.execute("masks.add", json!({"clip": clip.0, "shape": "rectangle", "center": [960, 540], "size": [960, 540], "cornerRadius": 200})).unwrap();
s.execute("masks.set", json!({"feather": 0})).unwrap();
let list = s.execute("masks.list", json!({})).unwrap();
assert_eq!(list["masks"][0]["path"]["vertices"].as_array().unwrap().len(), 8);
let img = s.render_program(0.25).unwrap();
// just inside the rectangle's corner: cut away; middle of an edge and the centre: kept
let corner = px(&img, 0.26, 0.26);
assert!(corner[0] + corner[1] + corner[2] < 1e-4, "rounded corner cut: {corner:?}");
assert!(px(&img, 0.5, 0.26)[3] > 0.99, "top edge kept");
assert!(px(&img, 0.5, 0.5)[3] > 0.99, "inside kept");
// round an existing 4-point mask: same picture, one undo step back to sharp corners
s.undo();
s.undo();
s.execute("masks.add", json!({"clip": clip.0, "shape": "polygon", "center": [960, 540], "size": [960, 540]})).unwrap();
s.execute("masks.set", json!({"feather": 0})).unwrap();
let sharp = s.render_program(0.25).unwrap();
assert!(px(&sharp, 0.26, 0.26)[3] > 0.99, "a sharp corner keeps the clip");
let r = s.execute("masks.roundCorners", json!({"radius": 200})).unwrap();
assert_eq!(r["corners"], 4);
assert_eq!(s.render_program(0.25).unwrap().px, img.px, "rounding afterwards = creating rounded");
// nothing left to round; bad radii are refused without changing the mask
assert!(s.execute("masks.roundCorners", json!({"radius": 50})).is_err());
for radius in [json!(0), json!(-5), json!("big"), json!(null)] {
assert!(s.execute("masks.roundCorners", json!({"radius": radius})).is_err(), "{radius}");
}
s.undo();
assert_eq!(s.render_program(0.25).unwrap().px, sharp.px, "one undo step restores the sharp corners");
}
#[test]
fn rounding_an_animated_mask_rounds_every_keyframe_and_only_the_chosen_corners() {
let (mut s, clip) = demo();
let it = s.active_sequence().unwrap().find_item(clip).unwrap().1.clone();
s.set_playhead(it.start);
s.execute("masks.add", json!({"effect": "opacity", "shape": "polygon", "center": [400, 400], "size": [200, 200]})).unwrap();
s.execute("effects.toggleAnimation", json!({"clip": clip.0, "effect": "opacity", "mask": 0, "param": "path"})).unwrap();
let rate = s.active_sequence().unwrap().settings.frame_rate;
s.set_playhead(it.start + rate.tick_of(10));
s.execute("masks.translate", json!({"delta": [300, 0]})).unwrap();
// only vertices 0 and 2 (top-left, bottom-right), out-of-range indices ignored
let r = s.execute("masks.roundCorners", json!({"radius": 30, "vertices": [0, 2, 99]})).unwrap();
assert_eq!(r["corners"], 2);
let m = &s.active_sequence().unwrap().find_item(clip).unwrap().1.effects.iter().find(|e| e.effect == "opacity").unwrap().masks[0];
assert_eq!(m.path.keyframes.len(), 2);
for k in &m.path.keyframes {
let filmcraft_project::ParamValue::Path(p) = &k.value else { panic!("path keyframe") };
assert_eq!(p.len(), 6, "both keyframes rounded the same corners");
assert_eq!(p.vertices.iter().filter(|v| v.is_corner()).count(), 2, "vertices 1 and 3 stay sharp");
}
// halfway the path still interpolates (equal vertex counts)
let l = s.execute("masks.list", json!({"time": (it.start + rate.tick_of(5)).0})).unwrap();
let v = l["masks"][0]["path"]["vertices"].as_array().unwrap();
assert_eq!(v.len(), 6);
assert!((v[0]["p"][0].as_f64().unwrap() - 450.0).abs() < 1e-6, "{}", v[0]);
}
/// `path_to_json` → `path_from_json` keeps the Bézier tangents (they used to come back as zero, so
/// a pen mask's dragged-out curves were committed as corners; #853).
#[test]
fn path_json_round_trip_keeps_tangents() {
use filmcraft_geom::Vec2;
use filmcraft_project::mask::{MaskPath, MaskVertex};
let path = MaskPath {
vertices: vec![
MaskVertex { p: Vec2::new(10.0, 20.0), t_in: Vec2::new(-5.0, 0.0), t_out: Vec2::new(5.0, 0.0) },
MaskVertex::corner(Vec2::new(80.0, 20.0)),
MaskVertex { p: Vec2::new(40.0, 90.0), t_in: Vec2::new(3.0, -4.0), t_out: Vec2::new(-3.0, 4.0) },
],
closed: true,
};
assert_eq!(crate::masks::path_from_json(&crate::masks::path_to_json(&path)), Some(path.clone()));
// the serialized project form still parses
assert_eq!(crate::masks::path_from_json(&serde_json::to_value(&path).unwrap()), Some(path));
}

5
crates/engine/src/media_browser_tests.rs
View File

@ -29,6 +29,7 @@ impl FakeFs {
"/home/me/Movies/edit.edl",
"/home/me/Movies/subs.srt",
"/home/me/Movies/clip.mov",
"/home/me/Movies/CLIP.MOV",
] {
files.insert(f.to_string(), vec![0; 16]);
}
@ -100,7 +101,7 @@ fn listing_puts_folders_first_and_hides_unsupported_files() {
assert_eq!(l["dir"], "/home/me");
assert_eq!(names(&l), ["Movies"]);
let l = s.execute("mediaBrowser.list", json!({"path": "/home/me/Movies"})).unwrap();
assert_eq!(names(&l), ["Selects", "A-interview.wav", "b-roll.wav", "clip.mov", "edit.edl", "shot_0001.png", "shot_0002.png", "subs.srt"]);
assert_eq!(names(&l), ["Selects", "A-interview.wav", "b-roll.wav", "CLIP.MOV", "clip.mov", "edit.edl", "shot_0001.png", "shot_0002.png", "subs.srt"]);
let e = &l["entries"][0];
assert_eq!((e["isDir"].as_bool(), e["kind"].as_str()), (Some(true), Some("folder")));
let shot = l["entries"].as_array().unwrap().iter().find(|e| e["name"] == "shot_0001.png").unwrap();
@ -117,7 +118,7 @@ fn file_type_filter() {
let l = s.execute("mediaBrowser.list", json!({"path": "/home/me/Movies", "fileTypes": "image"})).unwrap();
assert_eq!(names(&l), ["Selects", "shot_0001.png", "shot_0002.png"]);
let l = s.execute("mediaBrowser.list", json!({"path": "/home/me/Movies", "fileTypes": "mov"})).unwrap();
assert_eq!(names(&l), ["Selects", "clip.mov"]);
assert_eq!(names(&l), ["Selects", "CLIP.MOV", "clip.mov"]);
// the setting persists and applies by default
s.execute("mediaBrowser.settings", json!({"fileTypes": "project"})).unwrap();
let l = s.execute("mediaBrowser.list", json!({"path": "/home/me/Movies"})).unwrap();

362
crates/engine/src/media_pool.rs
View File

@ -4,13 +4,15 @@
//! - Sources are cached per item **and per reference** (the file path, the offline flag), so a
//! relink, Make Offline or their undo picks up the right file on the next frame.
//! - Media that can't be opened (missing, unreadable) or was made offline renders the offline
//! slate ([`filmcraft_render::offline`]) instead of failing; the reason is kept for the UI.
//! slate ([`filmcraft_render::offline`]) instead of failing; the reason is kept for the UI. So
//! do the frames of a file that opens but whose picture FilmCraft cannot decode (a codec or
//! profile without a decoder).
//! - With proxies enabled ([`MediaPool::set_use_proxies`]) items with an attached proxy are read
//! from it. Proxy frames are smaller; the source still reports the full-resolution size, and the
//! compositor derives its pixel scale from the frame it gets, so effects and Motion render the
//! same picture at lower resolution. Export asks for [`MediaPool::full_res_provider`].
use std::collections::HashMap;
use std::collections::{BTreeSet, HashMap};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock};
@ -30,12 +32,25 @@ pub struct OfflineStatus {
pub error: String,
}
/// An offline item an export or a rendered frame drew the slate for (#1054). `status` is the
/// word `media.status` uses: `missing`, `offline` (made offline) or `unreadable`.
#[derive(Clone, Debug, PartialEq, serde::Serialize)]
pub struct OfflineMedia {
pub item: u64,
pub name: String,
pub path: String,
pub status: &'static str,
pub error: String,
}
pub struct MediaPool {
/// item → (reference key, source)
sources: RwLock<HashMap<ItemId, (String, SharedSource)>>,
proxies: RwLock<HashMap<ItemId, (String, SharedSource)>>,
offline: RwLock<HashMap<ItemId, OfflineStatus>>,
offline: Arc<RwLock<HashMap<ItemId, OfflineStatus>>>,
use_proxies: AtomicBool,
/// Per item: video frames that failed in a row in the monitors (see [`Self::monitor_source`]).
failures: Arc<Mutex<HashMap<ItemId, u32>>>,
/// Openers tried before the built-in ones (MP4/MOV + codecs register here).
pub openers: RwLock<Vec<Opener>>,
}
@ -46,15 +61,16 @@ impl Default for MediaPool {
Self {
sources: RwLock::new(HashMap::new()),
proxies: RwLock::new(HashMap::new()),
offline: RwLock::new(HashMap::new()),
offline: Arc::new(RwLock::new(HashMap::new())),
use_proxies: AtomicBool::new(false),
failures: Arc::default(),
openers: RwLock::new(filmcraft_codecs::openers()),
}
}
}
/// Largest file [`MediaPool::probe_file`] reads whole when no streaming reader takes it (WAV,
/// MP3 and other byte-opened formats, or an unsupported container such as AVI).
/// MP3 and other byte-opened formats, or an unsupported container).
pub const PROBE_WHOLE_FILE_MAX: u64 = 64 * 1024 * 1024;
/// Cache key of a media clip's full-resolution reference ("" matches anything: sources inserted
@ -139,8 +155,16 @@ impl MediaPool {
/// Cache a source opened from `path` for an item.
pub fn insert_file(&self, item: ItemId, path: &str, src: SharedSource) {
self.sources.write().unwrap_or_else(|e| e.into_inner()).insert(item, (format!("file:{path}"), src));
self.offline.write().unwrap_or_else(|e| e.into_inner()).remove(&item);
let src = self.unsupported_as_slate(item, path, src);
self.sources.write().unwrap_or_else(|e| e.into_inner()).insert(item, (format!("file:{path}"), src));
}
/// `src` (opened from `path` for `item`) showing the unreadable slate for the frames it
/// cannot decode.
fn unsupported_as_slate(&self, item: ItemId, path: &str, src: SharedSource) -> SharedSource {
let slate = SlateSource::new(src.info().clone(), &file_name(path), OfflineReason::Unreadable);
Arc::new(UnsupportedAsSlate { inner: src, slate, item, path: path.to_string(), offline: self.offline.clone() })
}
pub fn remove(&self, item: ItemId) {
@ -173,6 +197,19 @@ impl MediaPool {
self.use_proxies.store(on, Ordering::Relaxed);
}
/// `src` (item `item`) as the monitors show it: once [`DAMAGED_AFTER`] of its video frames in a
/// row fail to decode or read (a damaged file), its failing frames show the unreadable slate
/// instead of nothing, until a frame decodes again (#626, follow-up to #762). Only for preview:
/// exports and other renders use the pool's sources directly, so they never contain the slate.
pub fn monitor_source(&self, item: ItemId, src: SharedSource) -> SharedSource {
Arc::new(DamagedAsSlate {
slate: SlateSource::new(src.info().clone(), &src.info().name, OfflineReason::Unreadable),
inner: src,
item,
failures: self.failures.clone(),
})
}
/// Why an item last rendered offline (None = online or not opened yet).
pub fn offline_status(&self, item: ItemId) -> Option<OfflineStatus> {
self.offline.read().unwrap_or_else(|e| e.into_inner()).get(&item).cloned()
@ -249,7 +286,7 @@ impl MediaPool {
} {
Ok(s) => {
self.offline.write().unwrap_or_else(|e| e.into_inner()).remove(&item);
s
self.unsupported_as_slate(item, path, s)
}
Err(_) if filmcraft_media::pending::is_set() => {
// the host is still fetching the file's bytes: show the slate for now, retry later
@ -306,12 +343,12 @@ impl MediaPool {
/// A render-side provider bound to a project snapshot (proxies when enabled).
pub fn provider(self: &Arc<Self>, project: Arc<Project>, services: Arc<dyn Services>) -> PoolProvider {
PoolProvider { pool: self.clone(), project, services, full_res: false }
PoolProvider { pool: self.clone(), project, services, full_res: false, used: None }
}
/// A provider that always reads full-resolution media (export).
pub fn full_res_provider(self: &Arc<Self>, project: Arc<Project>, services: Arc<dyn Services>) -> PoolProvider {
PoolProvider { pool: self.clone(), project, services, full_res: true }
PoolProvider { pool: self.clone(), project, services, full_res: true, used: None }
}
}
@ -322,10 +359,45 @@ pub struct PoolProvider {
pub services: Arc<dyn Services>,
/// Ignore proxies (export always renders full resolution).
pub full_res: bool,
/// Items asked for so far, when recording ([`PoolProvider::recording`]).
used: Option<Mutex<BTreeSet<ItemId>>>,
}
impl PoolProvider {
/// This provider, remembering the items it is asked for so [`PoolProvider::offline_used`] can
/// tell which of them rendered offline.
pub fn recording(mut self) -> Self {
self.used = Some(Mutex::new(BTreeSet::new()));
self
}
/// The offline items among those asked for since [`PoolProvider::recording`], once each, in
/// item order (subclips count as their media). Empty when not recording.
pub fn offline_used(&self) -> Vec<OfflineMedia> {
let Some(used) = &self.used else { return Vec::new() };
let used = used.lock().unwrap_or_else(|e| e.into_inner()).clone();
let media: BTreeSet<ItemId> = used.into_iter().filter_map(|i| self.project.resolve_media(i).map(|(id, _, _)| id)).collect();
media
.into_iter()
.filter_map(|id| {
let st = self.pool.offline_status(id)?;
let status = match st.reason {
OfflineReason::Missing => "missing",
OfflineReason::MadeOffline => "offline",
OfflineReason::Unreadable => "unreadable",
};
let name = self.project.item(id).map(|i| i.name.clone()).unwrap_or_default();
Some(OfflineMedia { item: id.0, name, path: st.path, status, error: st.error })
})
.collect()
}
}
impl filmcraft_render::SourceProvider for PoolProvider {
fn source(&self, item: ItemId) -> Option<SharedSource> {
if let Some(used) = &self.used {
used.lock().unwrap_or_else(|e| e.into_inner()).insert(item);
}
if self.full_res { self.pool.full_res_source(&self.project, item, &*self.services) } else { self.pool.source_for(&self.project, item, &*self.services) }
}
}
@ -373,6 +445,90 @@ impl MediaSource for SlateSource {
}
}
/// A file's source whose picture shows the unreadable slate for each frame its decoder reports as
/// unsupported (H.264 High 4:2:2, a codec without a decoder...), instead of no picture at all,
/// and marks the item unreadable with the reason, so the monitors and the Project panel say why
/// (#626). Its sound plays as before.
struct UnsupportedAsSlate {
inner: SharedSource,
slate: SlateSource,
item: ItemId,
path: String,
offline: Arc<RwLock<HashMap<ItemId, OfflineStatus>>>,
}
impl MediaSource for UnsupportedAsSlate {
fn info(&self) -> &MediaInfo {
self.inner.info()
}
fn video_frame(&self, req: FrameRequest) -> filmcraft_media::Result<Arc<VideoFrame>> {
match self.inner.video_frame(req) {
Err(MediaError::Unsupported(why)) => {
self.offline.write().unwrap_or_else(|e| e.into_inner()).entry(self.item).or_insert_with(|| {
log::warn!("media unreadable: {}: {why}", self.path);
OfflineStatus { reason: OfflineReason::Unreadable, path: self.path.clone(), error: why }
});
self.slate.video_frame(req)
}
r => r,
}
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
self.inner.audio(start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
self.inner.audio_stream(stream, start, frames, sample_rate)
}
}
/// Consecutive failed video frames after which a monitor treats a source as damaged. A single
/// failure (one bad slice, a read hiccup) and the decoder's retry of it stay as they were; three
/// in a row (an eighth of a second at 24 fps) mean the file itself is damaged.
pub const DAMAGED_AFTER: u32 = 3;
/// See [`MediaPool::monitor_source`]. Counts `Decode` / `Io` failures per item; a successful frame
/// resets the count. Data still on its way (the web build's pending reads, see
/// [`filmcraft_media::pending`]) neither counts nor resets, and cancellation, absent video and
/// unsupported frames (already the slate, #762) pass through.
struct DamagedAsSlate {
inner: SharedSource,
slate: SlateSource,
item: ItemId,
failures: Arc<Mutex<HashMap<ItemId, u32>>>,
}
impl MediaSource for DamagedAsSlate {
fn info(&self) -> &MediaInfo {
self.inner.info()
}
fn video_frame(&self, req: FrameRequest) -> filmcraft_media::Result<Arc<VideoFrame>> {
let r = self.inner.video_frame(req);
let mut failures = self.failures.lock().unwrap_or_else(|e| e.into_inner());
match r {
Ok(f) => {
failures.remove(&self.item);
Ok(f)
}
Err(MediaError::Decode(_) | MediaError::Io(_)) if !filmcraft_media::pending::is_set() => {
let n = failures.entry(self.item).or_insert(0);
*n = n.saturating_add(1);
if *n >= DAMAGED_AFTER {
drop(failures);
return self.slate.video_frame(req);
}
r
}
r => r,
}
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
self.inner.audio(start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
self.inner.audio_stream(stream, start, frames, sample_rate)
}
}
/// A proxy standing in for its full-resolution media: reports the full-resolution info and hands
/// out the proxy's (smaller) frames. A proxy whose aspect ratio differs from the original is
/// resampled to the original's aspect so it lines up exactly.
@ -501,4 +657,192 @@ mod tests {
assert!(matches!(MediaPool::default().open_file(&missing, &crate::FsServices), Err(MediaError::Offline(_))));
std::fs::remove_dir_all(&dir).unwrap();
}
/// A file whose first frame decodes and whose later frames need a decoder FilmCraft lacks.
struct PartlyDecodable(MediaInfo);
impl MediaSource for PartlyDecodable {
fn info(&self) -> &MediaInfo {
&self.0
}
fn video_frame(&self, req: FrameRequest) -> filmcraft_media::Result<Arc<VideoFrame>> {
match req.time.0 {
0 => Ok(Arc::new(VideoFrame::rgba8(2, 2, vec![7; 16]))),
1 => Err(MediaError::Decode("damaged slice".into())),
_ => Err(MediaError::Unsupported("H.264: field pictures".into())),
}
}
fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
Ok(AudioBuffer::silence(sample_rate, 3, frames))
}
}
/// #626: frames the decoder does not support show the unreadable slate (not nothing) and the
/// item says why; decodable frames, damaged ones and the sound are left alone.
#[test]
fn undecodable_frames_show_the_unreadable_slate() {
let info = GeneratorSource::new(filmcraft_media::Generator::ColorMatte { color: [0.0; 4] }, 16, 8, Default::default(), filmcraft_time::Tick(100))
.info()
.clone();
let pool = MediaPool::default();
let item = ItemId(1);
pool.insert_file(item, "/media/cam/a.mov", Arc::new(PartlyDecodable(info)));
let src = pool.cached(item).unwrap();
let at = |t| src.video_frame(FrameRequest { time: filmcraft_time::Tick(t), scale: 1.0 });
assert_eq!(at(0).unwrap().to_rgba8(), vec![7; 16]);
assert!(matches!(at(1), Err(MediaError::Decode(_))));
assert!(pool.offline_status(item).is_none(), "nothing unsupported yet");
let f = at(2).unwrap();
assert_eq!((f.width, f.height), (16, 8));
assert_eq!(f.to_rgba8(), filmcraft_render::offline::slate_rgba8(16, 8, "a.mov", OfflineReason::Unreadable));
let st = pool.offline_status(item).unwrap();
assert_eq!((st.reason, st.path.as_str(), st.error.as_str()), (OfflineReason::Unreadable, "/media/cam/a.mov", "H.264: field pictures"));
assert_eq!(src.audio(0, 4, 48_000).unwrap().channels.len(), 3);
// linking the item to another file forgets the reason
pool.insert_file(item, "/media/cam/b.mov", src.clone());
assert!(pool.offline_status(item).is_none());
}
/// A source whose video decodes (`OK`) or fails as its mode says, switchable at run time.
struct Flaky(MediaInfo, std::sync::atomic::AtomicU8);
const OK: u8 = 0;
const DAMAGED: u8 = 1;
const IO: u8 = 2;
const LOADING: u8 = 3;
const CANCELLED: u8 = 4;
const NO_VIDEO: u8 = 5;
impl Flaky {
fn set(&self, mode: u8) {
self.1.store(mode, Ordering::Relaxed);
}
}
impl MediaSource for Flaky {
fn info(&self) -> &MediaInfo {
&self.0
}
fn video_frame(&self, _: FrameRequest) -> filmcraft_media::Result<Arc<VideoFrame>> {
Err(match self.1.load(Ordering::Relaxed) {
OK => return Ok(Arc::new(VideoFrame::rgba8(2, 2, vec![7; 16]))),
DAMAGED => MediaError::Decode("damaged slice".into()),
IO => MediaError::Io("short read".into()),
LOADING => {
// as the web build's WebCodecs / Blob sources do
filmcraft_media::pending::mark();
MediaError::Decode("still decoding".into())
}
CANCELLED => MediaError::Cancelled,
_ => MediaError::NoStream("video"),
})
}
fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
Ok(AudioBuffer::silence(sample_rate, 1, frames))
}
}
fn flaky() -> (MediaPool, ItemId, Arc<Flaky>) {
let info = GeneratorSource::new(filmcraft_media::Generator::ColorMatte { color: [0.0; 4] }, 16, 8, Default::default(), filmcraft_time::Tick(100))
.info()
.clone();
(MediaPool::default(), ItemId(1), Arc::new(Flaky(info, Default::default())))
}
fn frame(src: &SharedSource) -> filmcraft_media::Result<Arc<VideoFrame>> {
src.video_frame(FrameRequest { time: filmcraft_time::Tick(0), scale: 1.0 })
}
fn is_slate(r: &filmcraft_media::Result<Arc<VideoFrame>>, name: &str) -> bool {
r.as_ref().is_ok_and(|f| f.to_rgba8() == filmcraft_render::offline::slate_rgba8(16, 8, name, OfflineReason::Unreadable))
}
/// #626 follow-up to #762: a damaged file (frames failing again and again) shows the
/// unreadable slate in the monitors; one failure does not, and a decoded frame clears it.
#[test]
fn monitors_show_the_slate_for_persistently_failing_frames_only() {
let (pool, item, src) = flaky();
let name = src.info().name.clone();
let mon = pool.monitor_source(item, src.clone());
for mode in [DAMAGED, IO] {
src.set(mode);
for _ in 1..DAMAGED_AFTER {
assert!(frame(&mon).is_err(), "a transient failure stays as it was");
}
assert!(is_slate(&frame(&mon), &name), "persistent failure shows the slate");
assert!(is_slate(&frame(&mon), &name));
// recovery clears it, and one failure after that is transient again
src.set(OK);
assert_eq!(frame(&mon).unwrap().to_rgba8(), vec![7; 16]);
src.set(DAMAGED);
assert!(frame(&mon).is_err());
src.set(OK);
assert!(frame(&mon).is_ok());
}
// cancellation and absent video never become the slate
for mode in [CANCELLED, NO_VIDEO] {
src.set(mode);
for _ in 0..DAMAGED_AFTER * 2 {
assert!(frame(&mon).is_err());
}
}
}
/// Pictures still on their way (the web build's pending reads) never count as damage.
#[test]
fn still_decoding_frames_never_show_the_slate() {
let (pool, item, src) = flaky();
let mon = pool.monitor_source(item, src.clone());
src.set(LOADING);
for _ in 0..DAMAGED_AFTER * 4 {
assert!(frame(&mon).is_err());
assert!(filmcraft_media::pending::take());
}
// and they do not interrupt a run of real failures either way: a following damaged frame
// is still only the first
src.set(DAMAGED);
assert!(frame(&mon).is_err());
}
/// Exports read the pool's own sources: a damaged file never burns the slate into them,
/// however often the monitors have shown it.
#[test]
fn export_sources_never_show_the_damaged_slate() {
let (pool, item, src) = flaky();
let name = src.info().name.clone();
pool.insert(item, src.clone());
let mon = pool.monitor_source(item, pool.cached(item).unwrap());
src.set(DAMAGED);
for _ in 0..DAMAGED_AFTER {
let _ = frame(&mon);
}
assert!(is_slate(&frame(&mon), &name));
let export = pool.cached(item).unwrap();
for _ in 0..DAMAGED_AFTER * 2 {
assert!(matches!(frame(&export), Err(MediaError::Decode(_))));
}
}
/// #626 end to end: an H.264 profile FilmCraft does not decode yet (4:4:4) imports and shows
/// the unreadable slate with the decoder's reason, instead of a black picture.
#[test]
fn ffmpeg_unsupported_h264_profile_shows_the_unreadable_slate() {
let Some(ffmpeg) = filmcraft_testkit::oracle::ffmpeg_or_skip("media_pool") else { return };
let path = filmcraft_testkit::fixtures_dir("engine/media_pool").join("h264_444_64x32.mp4");
let Some(path) = filmcraft_testkit::fixtures::generate(&path, |tmp| {
std::process::Command::new(&ffmpeg)
.args(["-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=64x32:rate=24:duration=0.5"])
.args(["-c:v", "libx264", "-preset", "fast", "-pix_fmt", "yuv444p"])
.arg(tmp)
.status()
.is_ok_and(|s| s.success())
}) else {
return;
};
let mut s = crate::Session::default();
let r = s.execute("file.import", serde_json::json!({"paths": [path.to_string_lossy()]})).unwrap();
let item = ItemId(r["items"][0].as_u64().unwrap());
let src = s.media.source_for(&s.project, item, &*s.services).unwrap();
let f = src.video_frame(FrameRequest { time: filmcraft_time::Tick::ZERO, scale: 1.0 }).unwrap();
assert_eq!(f.to_rgba8(), filmcraft_render::offline::slate_rgba8(64, 32, "h264_444_64x32.mp4", OfflineReason::Unreadable));
let st = s.media.offline_status(item).unwrap();
assert_eq!(st.reason, OfflineReason::Unreadable);
assert!(st.error.contains("H.264") && st.error.contains("chroma_format_idc 3"), "{}", st.error);
}
}

165
crates/engine/src/media_watch.rs Normal file
View File

@ -0,0 +1,165 @@
//! Settings ▸ Media ▸ Automatically refresh growing files: media files that change on disk while
//! the project is open are read again, whether they are still being written (a recording, an
//! ingest) or were replaced (a render from EffectCraft or After Effects written over the file a
//! clip uses).
//!
//! Each file media item's file is compared by size and modification time with what it was when
//! last seen. The desktop app takes the stamps every *Refresh growing files every* seconds and as
//! soon as its window comes back to the front, on a worker thread so a file on a slow network share
//! never holds up the interface ([`Session::start_media_scan`], [`Session::poll_media_scan`]);
//! `media.refreshChanged` takes them on the spot (CLI, agents).
//!
//! Refreshing opens the file again: the item gets the file's new [`MediaInfo`] (a longer
//! duration, another frame size) and identity (so relinking compares against the new file), the
//! media pool swaps in the new source (the old one still indexes the file as it was), and the
//! view is refreshed without an undo step or modifying a clean project ([`Session::bump_view`]):
//! the file changed, not the edit. A file seen for the first time is only remembered; one that
//! can't be opened (still being written) is tried again once it changes again, not on every look.
//!
//! [`MediaInfo`]: filmcraft_media::MediaInfo
use std::collections::{BTreeSet, HashMap};
use std::sync::{Arc, Mutex, PoisonError};
use filmcraft_project::{ItemId, ItemKind, MediaRef, Project};
use serde_json::{Value, json};
use crate::{Result, Session};
/// A file's size and modification time (nanoseconds since 1970).
pub type Stamp = (u64, u128);
/// Stamps of the project's media files taken on a worker thread.
pub type MediaScan = Arc<Mutex<Option<Vec<(String, Option<Stamp>)>>>>;
#[cfg(not(target_arch = "wasm32"))]
fn stamp(path: &str) -> Option<Stamp> {
let meta = std::fs::metadata(path).ok()?;
let modified = meta.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?;
Some((meta.len(), modified.as_nanos()))
}
#[cfg(target_arch = "wasm32")]
fn stamp(_: &str) -> Option<Stamp> {
None
}
/// The files of `p`'s file media items (made-offline ones aside).
pub fn media_files(p: &Project) -> BTreeSet<String> {
p.items
.values()
.filter_map(|i| match &i.kind {
ItemKind::Media(m) if !m.offline => match &m.media {
MediaRef::File { path } => Some(path.clone()),
_ => None,
},
_ => None,
})
.collect()
}
/// `media.refreshChanged`: stamp the project's media files now and read the changed ones again.
pub(crate) fn refresh_changed(s: &mut Session, _: &Value) -> Result<Value> {
let now = media_files(&s.project).into_iter().map(|p| (p.clone(), stamp(&p))).collect();
s.apply_media_stamps(now)
}
impl Session {
/// Start stamping the project's media files on a worker thread, unless a look is still
/// running (or the build has no files to look at: the web app).
pub fn start_media_scan(&mut self) {
if self.media_scan.is_some() || cfg!(target_arch = "wasm32") {
return;
}
let files = media_files(&self.project);
if files.is_empty() {
return;
}
let slot: MediaScan = Arc::default();
let out = slot.clone();
let spawned = std::thread::Builder::new().name("media-watch".into()).spawn(move || {
let stamps = files.into_iter().map(|p| (p.clone(), stamp(&p))).collect();
*out.lock().unwrap_or_else(PoisonError::into_inner) = Some(stamps);
});
if spawned.is_ok() {
self.media_scan = Some(slot);
}
}
/// Apply a finished look (`None` while there is none, or it is still running).
pub fn poll_media_scan(&mut self) -> Option<Result<Value>> {
let stamps = self.media_scan.as_ref()?.lock().unwrap_or_else(PoisonError::into_inner).take()?;
self.media_scan = None;
Some(self.apply_media_stamps(stamps))
}
/// Compare the media files' stamps with the ones last seen and read the changed files
/// again. Returns `{refreshed: [item ids]}`.
fn apply_media_stamps(&mut self, now: Vec<(String, Option<Stamp>)>) -> Result<Value> {
// Only the files the project uses now are remembered.
let before: HashMap<String, Stamp> = std::mem::take(&mut self.media_stamps);
let mut changed = BTreeSet::new();
for (path, stamp) in now {
match (before.get(&path), stamp) {
// Not there right now (an app saving by delete and rename): keep the last stamp.
(Some(old), None) => {
self.media_stamps.insert(path, *old);
}
(old, Some(new)) => {
if old.is_some_and(|o| *o != new) {
changed.insert(path.clone());
}
self.media_stamps.insert(path, new);
}
(None, None) => {}
}
}
if changed.is_empty() {
return Ok(json!({"refreshed": []}));
}
let mut fresh = Vec::new();
let mut failed = BTreeSet::new();
for (id, it) in &self.project.items {
let Some(m) = it.as_media().filter(|m| !m.offline) else { continue };
let MediaRef::File { path } = &m.media else { continue };
if !changed.contains(path) || failed.contains(path) {
continue;
}
// Opened the way import opens it, so the info is what a new import would get.
match self.media.open_file(path, &*self.services) {
Ok(src) => {
let mut info = src.info().clone();
if info.kind == filmcraft_media::MediaKind::Still
&& let Some(v) = info.video.as_mut()
{
v.frame_rate = crate::settings::timebase_rate(&self.prefs.media.indeterminate_timebase);
}
let identity = crate::relink::identity_of(&*self.services, path).ok();
fresh.push((*id, path.clone(), info, identity, src));
}
Err(_) => {
failed.insert(path.clone());
}
}
}
// A file that can't be opened (still being written, or not media FilmCraft reads) keeps its
// new stamp: it is tried again once it changes again, not on every look.
if fresh.is_empty() {
return Ok(json!({"refreshed": []}));
}
let p = Arc::make_mut(&mut self.project);
let mut refreshed: Vec<ItemId> = Vec::new();
for (id, path, info, identity, src) in fresh {
if let Some(m) = p.item_mut(id).and_then(|i| i.as_media_mut()) {
m.info = info;
if identity.is_some() {
m.identity = identity;
}
}
self.media.insert_file(id, &path, src);
refreshed.push(id);
}
self.bump_view();
Ok(json!({"refreshed": refreshed.iter().map(|i| i.0).collect::<Vec<_>>()}))
}
}

100
crates/engine/src/media_watch_tests.rs Normal file
View File

@ -0,0 +1,100 @@
//! Media files changing on disk while the project is open (Settings ▸ Media ▸ Automatically
//! refresh growing files, `media_watch`): a re-rendered movie is read again (new length, new
//! picture, new identity) without an undo step, a file still being written is tried again, and
//! the background look does the same.
use filmcraft_media::DemoScene;
use serde_json::json;
use crate::Session;
use crate::media_test_util::{frame_rgba, make_movie, psnr, session_with, tmp_dir};
const W: u32 = 160;
const H: u32 = 90;
/// Write a movie over `path` so its modification time moves even on coarse file systems.
fn rerender(path: &std::path::Path, scene: DemoScene, frames: i64) {
std::thread::sleep(std::time::Duration::from_millis(30));
make_movie(path, scene, W, H, frames);
}
fn refresh(s: &mut Session) -> serde_json::Value {
s.execute("media.refreshChanged", json!({})).unwrap()
}
/// What frame 3 of a fresh import of `scene` looks like.
fn reference(name: &str, scene: DemoScene, frames: i64) -> Vec<u8> {
let root = tmp_dir(name);
let f = root.join("ref.mov");
make_movie(&f, scene, W, H, frames);
let (mut r, _, _) = session_with(&[&f]);
frame_rgba(&mut r, 3, 1.0).2
}
#[test]
fn a_rerendered_movie_is_read_again_without_an_undo_step() {
let root = tmp_dir("watch-rerender");
let a = root.join("a.mov");
make_movie(&a, DemoScene::Aurora, W, H, 12);
let (mut s, items, _) = session_with(&[&a]);
let item = items[0];
let before = frame_rgba(&mut s, 3, 1.0).2;
let length = s.project.item(item).unwrap().duration();
s.saved_revision = s.revision;
let undo = s.history.undo.len();
// First look: remembered, nothing to read.
assert_eq!(refresh(&mut s), json!({"refreshed": []}));
rerender(&a, DemoScene::CityNight, 24);
assert_eq!(refresh(&mut s), json!({"refreshed": [item.0]}));
let m = s.project.item(item).unwrap().as_media().unwrap().clone();
assert_eq!(m.info.duration.0, length.0 * 2, "the new length");
assert_eq!(m.identity, crate::relink::identity_of(&*s.services, &a.to_string_lossy()).ok(), "relinking compares with the new file");
let after = frame_rgba(&mut s, 3, 1.0).2;
assert!(psnr(&after, &reference("watch-rerender-ref", DemoScene::CityNight, 24)) > 40.0, "the new picture");
assert!(psnr(&after, &before) < 30.0, "not the old one");
assert_eq!(s.history.undo.len(), undo, "the file changed, not the edit");
assert!(!s.is_dirty());
assert_eq!(refresh(&mut s), json!({"refreshed": []}), "acted on once");
}
#[test]
fn a_file_still_being_written_is_tried_again() {
let root = tmp_dir("watch-partial");
let a = root.join("a.mov");
make_movie(&a, DemoScene::Aurora, W, H, 12);
let (mut s, items, _) = session_with(&[&a]);
refresh(&mut s);
// The other app has only written the start of the file so far.
let full = root.join("full.mov");
make_movie(&full, DemoScene::CityNight, W, H, 24);
let bytes = std::fs::read(&full).unwrap();
std::thread::sleep(std::time::Duration::from_millis(30));
std::fs::write(&a, &bytes[..64]).unwrap();
assert_eq!(refresh(&mut s), json!({"refreshed": []}));
std::thread::sleep(std::time::Duration::from_millis(30));
std::fs::write(&a, &bytes).unwrap();
assert_eq!(refresh(&mut s), json!({"refreshed": [items[0].0]}));
}
#[test]
fn a_background_look_reads_changed_media_again() {
let root = tmp_dir("watch-scan");
let a = root.join("a.mov");
make_movie(&a, DemoScene::Aurora, W, H, 12);
let (mut s, items, _) = session_with(&[&a]);
let look = |s: &mut Session| {
s.start_media_scan();
let t0 = std::time::Instant::now();
loop {
if let Some(r) = s.poll_media_scan() {
return r.unwrap();
}
assert!(t0.elapsed().as_secs() < 10, "the look never finished");
std::thread::yield_now();
}
};
assert_eq!(look(&mut s), json!({"refreshed": []}));
rerender(&a, DemoScene::CityNight, 24);
assert_eq!(look(&mut s), json!({"refreshed": [items[0].0]}));
assert!(s.media_scan.is_none());
}

Some files were not shown because too many files have changed in this diff Show More