shifathosn
/
effectcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #827 from pravdomil/claude/import-exif-orientation 

Import: stills and movies respect their orientation

# Conflicts:
#	crates/media/src/pool.rs
This commit is contained in:
Brandon Thomas 2026-10-12 03:23:08 -04:00
parent 30f7ce8ae5 c03b0a7aa7
commit c11740f461
8 changed files with 241 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -1430,6 +1430,7 @@ dependencies = [
"filmcraft-codecs",
"filmcraft-color",
"filmcraft-frame",
"filmcraft-isobmff",
"filmcraft-media",
"filmcraft-time",
"gif",

1
crates/media/Cargo.toml
View File

@ -20,6 +20,7 @@ effectcraft-pdf = { workspace = true }
effectcraft-model = { workspace = true }
filmcraft-codecs = { workspace = true }
filmcraft-media = { workspace = true }
filmcraft-isobmff = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-color = { workspace = true }
filmcraft-time = { workspace = true }

26
crates/media/src/convert.rs
View File

@ -114,6 +114,32 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec<Px>) -
img
}
/// A movie frame turned the way its file says it is shown ([`crate::probe::video_turn`]:
/// clockwise quarter turns; anything else leaves it as decoded).
pub(crate) fn turned(img: Image, turn: image::metadata::Orientation) -> Image {
use image::metadata::Orientation;
let (w, h) = (img.width as usize, img.height as usize);
if img.data.len() != w * h || w == 0 || h == 0 {
return img;
}
// The source pixel shown at (x, y) of the turned picture.
let (tw, th, from): (usize, usize, fn(usize, usize, usize, usize) -> usize) = match turn {
Orientation::Rotate90 => (h, w, |x, y, w, h| (h - 1 - x) * w + y),
Orientation::Rotate180 => (w, h, |x, y, w, h| (h - 1 - y) * w + (w - 1 - x)),
Orientation::Rotate270 => (h, w, |x, y, w, _| x * w + (w - 1 - y)),
_ => return img,
};
let mut data = vec![[0.0; 4]; tw * th];
data.par_chunks_mut(tw).enumerate().for_each(|(y, row)| {
for (x, px) in row.iter_mut().enumerate() {
if let Some(s) = img.data.get(from(x, y, w, h)) {
*px = *s;
}
}
});
Image { width: tw as u32, height: th as u32, data }
}
/// A FilmCraft frame for a render that shows it at `w` × `h`, at most half its size each way:
/// what `resample(&frame_to_image(f, op), w, h)` makes, converting a quarter of the pixels.
/// Opaque 4:2:0 Y'CbCr is box-filtered to half size before conversion (each 2×2 block of luma,

2
crates/media/src/layered.rs
View File

@ -87,7 +87,7 @@ pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) -
let inner = effectcraft_psd::Psd::parse(data.to_vec()).map_err(|e| MediaError::Decode(format!("{path} (smart object): {e}")))?;
inner.composite().map_err(|e| MediaError::Decode(format!("{path} (smart object): {e}")))?
} else {
let img = image::load_from_memory(data).map_err(|e| MediaError::Decode(format!("{path} (smart object): {e}")))?;
let img = crate::probe::load_oriented(data).map_err(|e| MediaError::Decode(format!("{path} (smart object): {e}")))?;
let rgba = img.to_rgba32f();
effectcraft_psd::Pixels { width: rgba.width(), height: rgba.height(), data: rgba.pixels().map(|p| p.0).collect() }
};

31
crates/media/src/pool.rs
View File

@ -9,8 +9,9 @@ use effectcraft_raster::{Image, Px};
use effectcraft_render::{FloatCurve, FootageSource};
use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick};
use filmcraft_media::{FrameRequest, SharedSource};
use image::metadata::Orientation;
use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled};
use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled, turned};
use crate::gif_anim::{self, Anim};
use crate::{MediaError, Result};
@ -170,9 +171,10 @@ struct CacheState {
last: HashMap<Arc<str>, i64>,
}
/// A movie/audio source by path, opened by the first caller while the others wait for it
/// A movie/audio source by path and the turn its picture is shown with
/// ([`crate::probe::video_turn`]), opened by the first caller while the others wait for it
/// (`None`: failed to open; not retried until `forget`).
type SourceSlot = Arc<OnceLock<Option<SharedSource>>>;
type SourceSlot = Arc<OnceLock<Option<(SharedSource, Orientation)>>>;
/// An animated GIF by path, as [`SourceSlot`] (the error message when it failed to load).
type GifSlot = Arc<OnceLock<std::result::Result<Arc<Anim>, String>>>;
@ -576,6 +578,11 @@ impl Inner {
/// it again: a preview's frame workers, audio feeder, conform and waveform threads each
/// decoded a long FLAC in full at once (#408).
fn source(&self, path: &Arc<str>) -> Option<SharedSource> {
self.opened(path).map(|(src, _)| src)
}
/// [`Self::source`] and the turn the movie's picture is shown with.
fn opened(&self, path: &Arc<str>) -> Option<(SharedSource, Orientation)> {
let slot = lock(&self.sources).entry(path.clone()).or_default().clone();
slot.get_or_init(|| {
self.opened.fetch_add(1, Ordering::Relaxed);
@ -586,7 +593,8 @@ impl Inner {
let opened = in_place.unwrap_or_else(|| {
self.read(path).and_then(|bytes| {
let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from)
let turn = crate::probe::video_turn(&bytes);
filmcraft_codecs::open_bytes(&name, bytes).map(|src| (src, turn)).map_err(MediaError::from)
})
});
match opened {
@ -759,7 +767,7 @@ impl Inner {
}
let loaded = match crate::still_format(path, &bytes) {
Some(fmt) => image::load_from_memory_with_format(&bytes, fmt),
None => image::load_from_memory(&bytes),
None => crate::probe::load_oriented(&bytes),
};
let img = match loaded {
Ok(img) => img,
@ -774,19 +782,22 @@ impl Inner {
Ok(dynamic_to_image(frame, op))
}
Some(mt) => {
let src = self.source(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?;
let (src, turn) = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?;
let vf = src.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))).map_err(MediaError::from)?;
// A movie shown turned a quarter (a phone held upright): the footage's size and
// the render's are the turned picture's, the decoded frame's are the file's.
let side = |(w, h): (u32, u32)| if crate::probe::swaps_sides(turn) { (h, w) } else { (w, h) };
// A reduced-resolution render's frame, made at its size (a frame of the footage's
// own size only, as the renderer resamples only those).
let (w, h) = loc.key.size;
let (w, h) = side(loc.key.size);
if w > 0
&& (vf.width, vf.height) == (footage.width, footage.height)
&& (vf.width, vf.height) == side((footage.width, footage.height))
&& let Some(img) = frame_to_image_scaled(&vf, w, h)
{
return Ok(img);
return Ok(turned(img, turn));
}
let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize);
Ok(frame_to_image_in(&vf, op, buf))
Ok(turned(frame_to_image_in(&vf, op, buf), turn))
}
}
}

55
crates/media/src/probe.rs
View File

@ -5,6 +5,7 @@ use std::sync::Arc;
use effectcraft_project::{AlphaMode, Footage, FootageKind};
use effectcraft_time::{FrameRate, Tick};
use image::metadata::Orientation;
use crate::{MediaError, Result, ext_of, rate_from_fc};
@ -90,8 +91,17 @@ pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result<Footage> {
return probe_still_bytes(path, &bytes, fmt);
}
let name = Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string());
let turn = video_turn(&bytes);
let src = filmcraft_codecs::open_bytes(&name, bytes)?;
Ok(footage_from_info(path, src.info()))
let mut f = footage_from_info(path, src.info());
// The size as displayed: a movie shot with the camera on its side trades width and height.
if swaps_sides(turn) {
std::mem::swap(&mut f.width, &mut f.height);
if f.pixel_aspect > 0.0 {
f.pixel_aspect = 1.0 / f.pixel_aspect;
}
}
Ok(f)
}
pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) -> Footage {
@ -181,7 +191,12 @@ fn probe_still_bytes(path: &str, bytes: &[u8], fmt: image::ImageFormat) -> Resul
}
let reader = image::ImageReader::with_format(std::io::Cursor::new(bytes), fmt);
let (w, h, has_alpha) = match reader.into_decoder() {
Ok(dec) => (dec.dimensions().0, dec.dimensions().1, dec.color_type().has_alpha()),
Ok(mut dec) => {
// The size as displayed: an Exif orientation of a quarter turn swaps the sides.
let (w, h) = dec.dimensions();
let (w, h) = if swaps_sides(dec.orientation().unwrap_or(Orientation::NoTransforms)) { (h, w) } else { (w, h) };
(w, h, dec.color_type().has_alpha())
}
Err(e) => {
// A multi-layer OpenEXR file without an unnamed RGB layer.
let img = (fmt == image::ImageFormat::OpenExr).then(|| crate::exr_channels::layered_image(bytes)).flatten();
@ -191,3 +206,39 @@ fn probe_still_bytes(path: &str, bytes: &[u8], fmt: image::ImageFormat) -> Resul
};
Ok(still_footage(path, w, h, fmt, has_alpha))
}
/// Whether an Exif orientation turns the picture a quarter turn (its width and height trade places).
pub(crate) fn swaps_sides(o: Orientation) -> bool {
matches!(o, Orientation::Rotate90 | Orientation::Rotate270 | Orientation::Rotate90FlipH | Orientation::Rotate270FlipH)
}
/// Decode a still the way it is meant to be seen: its Exif orientation (a camera held sideways, a
/// mirrored scan) is applied to the pixels, as image viewers do. A file without
/// the tag, or with a broken one, decodes as stored.
pub(crate) fn load_oriented(bytes: &[u8]) -> image::ImageResult<image::DynamicImage> {
use image::ImageDecoder;
let mut dec = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format()?.into_decoder()?;
let orientation = dec.orientation().unwrap_or(Orientation::NoTransforms);
let mut img = image::DynamicImage::from_decoder(dec)?;
img.apply_orientation(orientation);
Ok(img)
}
/// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and
/// says so in its video track's header): the quarter turn nearest the track's rotation. Only the
/// rotation counts; any mirroring, scaling or offset in the header is ignored, and other
/// containers have none.
pub(crate) fn video_turn(bytes: &[u8]) -> Orientation {
let Ok(file) = filmcraft_isobmff::open(bytes) else { return Orientation::NoTransforms };
let Some(m) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).map(|t| t.matrix) else {
return Orientation::NoTransforms;
};
// [a b u; c d v; x y w]: (a, b) is where the picture's x axis points.
let (a, b) = (m[0] as i64, m[1] as i64);
match (a.abs() >= b.abs(), a >= 0, b >= 0) {
(true, true, _) => Orientation::NoTransforms,
(true, false, _) => Orientation::Rotate180,
(false, _, true) => Orientation::Rotate90,
(false, _, false) => Orientation::Rotate270,
}
}

84
crates/media/tests/exif_orientation.rs Normal file
View File

@ -0,0 +1,84 @@
//! A still's Exif orientation (a camera held sideways, a mirrored scan) is applied on import: the
//! footage has the size the picture is seen at, and its frames are the picture the right way up.
use std::sync::Arc;
use effectcraft_media::{MediaPool, probe_bytes};
use effectcraft_project::FootageKind;
use effectcraft_time::Tick;
const RED: [f32; 3] = [1.0, 0.0, 0.0];
const GREEN: [f32; 3] = [0.0, 1.0, 0.0];
const BLUE: [f32; 3] = [0.0, 0.0, 1.0];
const WHITE: [f32; 3] = [1.0, 1.0, 1.0];
/// A 32×16 JPEG with red, green / blue, white quarters, tagged with Exif orientation `tag`
/// (`None`: no Exif at all).
fn quartered_jpeg(tag: Option<u8>) -> Vec<u8> {
let img = image::RgbImage::from_fn(32, 16, |x, y| {
let c = match (x < 16, y < 8) {
(true, true) => RED,
(false, true) => GREEN,
(true, false) => BLUE,
(false, false) => WHITE,
};
image::Rgb(c.map(|v| (v * 255.0) as u8))
});
let mut jpeg = Vec::new();
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 100).encode_image(&img).expect("encode");
let Some(tag) = tag else { return jpeg };
// APP1 "Exif": a big-endian TIFF header and one IFD holding Orientation (0x0112, SHORT).
let mut app1 = vec![0xFF, 0xE1, 0x00, 0x22];
app1.extend_from_slice(b"Exif\0\0MM\0\x2A\0\0\0\x08");
app1.extend_from_slice(&[0x00, 0x01, 0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, tag, 0x00, 0x00]);
app1.extend_from_slice(&[0x00; 4]);
// (Straight after the SOI marker.)
jpeg.splice(2..2, app1);
jpeg
}
/// The footage size and the colours of its four corners (top-left, top-right, bottom-left,
/// bottom-right), as imported.
fn imported(tag: Option<u8>) -> ((u32, u32), [[f32; 3]; 4]) {
let name = format!("oriented-{tag:?}.jpg");
let bytes: Arc<[u8]> = quartered_jpeg(tag).into();
let f = probe_bytes(&name, bytes.clone()).expect("probe");
assert_eq!(f.kind, FootageKind::Still);
let pool = MediaPool::new();
pool.add_bytes(&name, bytes);
let img = pool.frame_at(&f, Tick::ZERO).expect("frame");
assert_eq!((img.width, img.height), (f.width, f.height), "orientation {tag:?}: the frame has the probed size");
let (w, h) = (img.width as usize, img.height as usize);
let at = |x: usize, y: usize| {
let p = img.data[y * w + x];
[p[0], p[1], p[2]]
};
((f.width, f.height), [at(2, 2), at(w - 3, 2), at(2, h - 3), at(w - 3, h - 3)])
}
#[test]
fn exif_orientation_is_applied_to_stills() {
let (landscape, portrait) = ((32, 16), (16, 32));
let cases = [
(None, landscape, [RED, GREEN, BLUE, WHITE]),
(Some(1), landscape, [RED, GREEN, BLUE, WHITE]),
(Some(2), landscape, [GREEN, RED, WHITE, BLUE]),
(Some(3), landscape, [WHITE, BLUE, GREEN, RED]),
(Some(4), landscape, [BLUE, WHITE, RED, GREEN]),
(Some(5), portrait, [RED, BLUE, GREEN, WHITE]),
(Some(6), portrait, [BLUE, RED, WHITE, GREEN]),
(Some(7), portrait, [WHITE, GREEN, BLUE, RED]),
(Some(8), portrait, [GREEN, WHITE, RED, BLUE]),
// (Not an orientation: the picture as stored.)
(Some(0), landscape, [RED, GREEN, BLUE, WHITE]),
(Some(9), landscape, [RED, GREEN, BLUE, WHITE]),
];
for (tag, size, corners) in cases {
let (got_size, got) = imported(tag);
assert_eq!(got_size, size, "orientation {tag:?}: size");
for (i, (g, want)) in got.iter().zip(corners).enumerate() {
let near = g.iter().zip(want).all(|(a, b)| (a - b).abs() < 0.2);
assert!(near, "orientation {tag:?}: corner {i} is {g:?}, expected {want:?}");
}
}
}

55
crates/media/tests/footage.rs
View File

@ -9,7 +9,7 @@ use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
use effectcraft_media::{MediaPool, probe};
use effectcraft_media::{MediaPool, probe, probe_bytes};
use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemId};
use effectcraft_raster::Image;
use effectcraft_render::FootageSource;
@ -336,6 +336,59 @@ fn resampled_audio_read_in_frame_pieces_matches_ffmpeg() {
}
}
/// The H.264 fixture with the rotation part of its video track header's matrix (`[a b; c d]`,
/// 16.16 fixed point) replaced: what a phone writes for a movie shot with the camera turned.
fn turned_mp4(p: &Path, abcd: [i32; 4]) -> Arc<[u8]> {
let mut bytes = std::fs::read(p).expect("read");
// (The video track comes first; a version 0 `tkhd` has its matrix 44 bytes after the type.)
let at = bytes.windows(4).position(|w| w == b"tkhd").expect("tkhd");
assert_eq!(bytes[at + 4], 0, "a version 0 tkhd");
for (slot, v) in [0, 1, 3, 4].into_iter().zip(abcd) {
let o = at + 44 + slot * 4;
bytes[o..o + 4].copy_from_slice(&v.to_be_bytes());
}
bytes.into()
}
/// A movie whose track header says it is shown rotated (a phone held upright) imports the way it
/// is shown: the footage has the turned size and its frames are turned, at full and reduced size.
#[test]
fn rotated_mp4_imports_upright() {
let Some(p) = h264_mp4() else { return };
let plain_footage = probe(&p).expect("probe");
let pool = MediaPool::new();
let t = t_of(plain_footage.frame_rate, 3);
let plain = pool.frame(ItemId(1), &plain_footage, t).expect("frame");
let (w, h) = (plain.width as usize, plain.height as usize);
let one = 0x1_0000;
// The matrix, the turned size and the plain frame's pixel shown at (x, y).
type From = fn(usize, usize, usize, usize) -> (usize, usize);
let cases: [(&str, [i32; 4], (usize, usize), From); 3] = [
("90", [0, one, -one, 0], (h, w), |x, y, _, h| (y, h - 1 - x)),
("180", [-one, 0, 0, -one], (w, h), |x, y, w, h| (w - 1 - x, h - 1 - y)),
("270", [0, -one, one, 0], (h, w), |x, y, w, _| (w - 1 - y, x)),
];
for (name, matrix, (tw, th), from) in cases {
let path = format!("turned-{name}.mp4");
let bytes = turned_mp4(&p, matrix);
let f = probe_bytes(&path, bytes.clone()).expect("probe");
assert_eq!((f.width as usize, f.height as usize), (tw, th), "{name}°: size");
pool.add_bytes(&path, bytes);
let img = pool.frame(ItemId(2), &f, t).expect("frame");
assert_eq!((img.width as usize, img.height as usize), (tw, th), "{name}°: frame size");
for (x, y) in [(0, 0), (tw - 1, 0), (0, th - 1), (tw - 1, th - 1), (tw / 3, th / 5), (tw / 2, th - 7)] {
let (sx, sy) = from(x, y, w, h);
assert_eq!(img.data[y * tw + x], plain.data[sy * w + sx], "{name}°: pixel ({x}, {y})");
}
// (A reduced-resolution render asks for the turned size.)
let (hw, hh) = (f.width / 2, f.height / 2);
let sized = pool.frame_at_size(ItemId(2), &f, t, hw, hh).expect("frame");
assert_eq!((sized.width, sized.height), (hw, hh), "{name}°: reduced size");
let db = psnr(&sized, &effectcraft_raster::resample(&img, hw, hh));
assert!(db > 45.0, "{name}° at half size: {db:.2} dB");
}
}
#[test]
fn prores_4444_with_alpha() {
let Some(p) = prores_mov() else { return };