diff --git a/Cargo.lock b/Cargo.lock index 3606369f..b1f42080 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2892,6 +2892,7 @@ dependencies = [ "lightcraft-raw", "lightcraft-scenes", "lightcraft-segment", + "lightcraft-tiff", "log", "miniz_oxide 0.8.9", "rayon", diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index b92742b1..d419eaea 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -40,3 +40,4 @@ sam = ["dep:lightcraft-segment"] [dev-dependencies] libc = { workspace = true } +lightcraft-tiff = { workspace = true } diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index 91be84b8..90056249 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -182,11 +182,15 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result { /// Why a raw that failed to decode should show its embedded preview instead (`Ok`), or the error /// to report (`Err`). Variants we can't decode yet always fall back. So does any CR3 error: the CRX /// decoder is verified on few bodies, and every CR3 opened from its embedded JPEG before it existed. +/// So does any other failure of a recognised raw (damaged or oversized raw data, unreadable TIFF +/// structure): its preview may still be intact. Only a file that is not a raw reports an error. fn preview_reason(bytes: &[u8], e: lightcraft_raw::RawError) -> Result { + use lightcraft_raw::RawError; match e { - lightcraft_raw::RawError::Unsupported(why) => Ok(why), + RawError::Unsupported(why) => Ok(why), e if lightcraft_raw::probe(bytes) == Some(lightcraft_raw::RawFormat::Cr3) => Ok(format!("CR3 {e}")), - e => Err(e.to_string()), + RawError::NotRaw => Err(RawError::NotRaw.to_string()), + e => Ok(e.to_string()), } } @@ -329,10 +333,24 @@ fn preview_orientation(raw_bytes: &[u8], jpeg_orientation: u16) -> Orientation { lightcraft_meta::extract(raw_bytes).orientation.unwrap_or(Orientation::Normal) } +/// What stands in for a raw file we can't decode: its largest embedded JPEG, else, for a TIFF-based +/// raw, the file's own first image when the TIFF reader can show it (a reduced RGB copy: the only +/// preview some containers carry). Returns image bytes for [`lightcraft_codecs::decode`]. +fn stand_in_image(bytes: &[u8]) -> Option> { + if let Some(jpeg) = lightcraft_raw::embedded_preview(bytes) { + return Some(std::borrow::Cow::Owned(jpeg)); + } + let undecodable = lightcraft_raw::probe(bytes).is_some_and(|f| !f.is_supported()); + (undecodable && lightcraft_codecs::sniff(bytes).is_some_and(|f| f == lightcraft_codecs::Format::Tiff)) + .then_some(std::borrow::Cow::Borrowed(bytes)) +} + /// Shared colour interpretation for quick previews, unsupported RAW fallback and camera-look fitting. pub(crate) fn decode_raw_preview(bytes: &[u8], opts: lightcraft_codecs::DecodeOptions) -> Option { - let jpeg = lightcraft_raw::embedded_preview(bytes)?; - match lightcraft_raw::embedded_preview_color_space(bytes) { + let jpeg = stand_in_image(bytes)?; + // the colour space is a property of the JPEG; a TIFF stand-in is read by the TIFF reader + let is_jpeg = matches!(jpeg, std::borrow::Cow::Owned(_)); + match lightcraft_raw::embedded_preview_color_space(bytes).filter(|_| is_jpeg) { Some(space) => { let space = match space { lightcraft_raw::PreviewColorSpace::Srgb => lightcraft_codecs::NamedSpace::Srgb, @@ -346,7 +364,7 @@ pub(crate) fn decode_raw_preview(bytes: &[u8], opts: lightcraft_codecs::DecodeOp /// Oriented size of the embedded preview of a raw file. fn embedded_preview_size(bytes: &[u8]) -> Option<(u32, u32)> { - let jpeg = lightcraft_raw::embedded_preview(bytes)?; + let jpeg = stand_in_image(bytes)?; let d = lightcraft_codecs::decode(&jpeg, lightcraft_codecs::DecodeOptions::fit(64, 64)).ok()?; let (mut w, mut h) = (d.source_width, d.source_height); if preview_orientation(bytes, d.orientation).swaps_axes() { @@ -704,6 +722,84 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// A raw whose TIFF shell holds only a small RGB image (IFD0) next to a private block, as Exif + /// declares a picture 25x its size. + fn tiff_shell(w: u32, h: u32, shell: bool) -> Vec { + use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value, tags as t}; + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + ifd0.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + ifd0.set(t::BITS_PER_SAMPLE, Value::Short(vec![8, 8, 8])); + ifd0.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3])); + ifd0.set(t::PHOTOMETRIC, Value::Short(vec![2])); + ifd0.set(t::COMPRESSION, Value::Short(vec![1])); + let px: Vec = (0..w * h).flat_map(|i| [(i % 251) as u8, 128, 200]).collect(); + ifd0.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![px] }); + ifd0.set_child( + t::EXIF_IFD, + IfdBuilder::new().with(t::PIXEL_X_DIMENSION, Value::Long(vec![w * 25])).with(t::PIXEL_Y_DIMENSION, Value::Long(vec![h * 25])), + ); + let mut b = TiffWriter::default().write(&[ifd0]).unwrap(); + if shell { + b.extend(std::iter::repeat_n(0x5au8, (w * h * 625 / 8) as usize + 100)); + } + b + } + + /// A TIFF raw whose only image is a small thumbnail used to open as a plain image of that size + /// ("a 150 MP file renders 296x220"), with nothing saying so. Now it is recognised as a raw we can't + /// decode: preview only, with the reason, still showing that image. + #[test] + fn thumbnail_only_tiff_raw_is_preview_only_not_a_plain_image() { + let b = tiff_shell(24, 16, true); + let p = probe_bytes("IMG_0001.IIQ", &b).unwrap(); + assert_eq!((p.kind, p.format.as_str(), p.width, p.height), (MediaKind::Raw, "IIQ", 24, 16)); + let why = p.preview_only.expect("marked preview only"); + assert!(why.contains("600x400") && why.contains("24x16"), "{why}"); + let (img, src) = load_bytes(&b, 12).unwrap(); + assert_eq!((img.width, img.height, src.raw), (12, 8, false)); + assert!(embedded_preview_srgb(&b, 12).is_some()); + // the same pixels without the private block are a small ordinary image + let plain = probe_bytes("small.tif", &tiff_shell(24, 16, false)).unwrap(); + assert_eq!((plain.kind, plain.preview_only, plain.width, plain.height), (MediaKind::Image, None, 24, 16)); + assert!(lightcraft_raw::probe(&tiff_shell(24, 16, false)).is_none()); + } + + /// Recognised raw containers we don't decode (Minolta MRW here: the preview's first byte is + /// overwritten in the file) import as preview only with the JPEG's size, instead of failing. + #[test] + fn undecodable_container_with_a_preview_imports_as_preview_only() { + let px: Vec = (0..40 * 30).flat_map(|i| [(i % 251) as u8, 128, 200]).collect(); + let mut jpeg = encode_jpeg(&EncodeImage::new(40, 30, 3, Samples::U8(&px)), 90, ChromaSubsampling::S444, &EncodeMeta::default()).unwrap(); + jpeg[0] = 0x02; + let mut f = b"\0MRM\0\x01\0\0".to_vec(); + f.extend(std::iter::repeat_n(0x11u8, 300)); + f.extend_from_slice(&jpeg); + f.extend(std::iter::repeat_n(0x22u8, 300)); + let p = probe_bytes("A.MRW", &f).unwrap(); + assert_eq!((p.kind, p.format.as_str(), p.width, p.height), (MediaKind::Raw, "MRW", 40, 30)); + assert!(p.preview_only.is_some_and(|w| w.contains("Mrw"))); + let (img, src) = load_bytes(&f, 20).unwrap(); + assert_eq!((img.width, img.height, src.raw), (20, 15, false)); + // without a JPEG: a clear error + let e = probe_bytes("A.MRW", b"\0MRM\0\x01\0\0 nothing here").unwrap_err(); + assert!(e.contains("without an embedded preview"), "{e}"); + } + + /// Any failure of a recognised raw (not only an unsupported variant) can fall back to its preview; + /// CR3 keeps its own wording. + #[test] + fn decode_failures_of_recognised_raws_may_fall_back_to_the_preview() { + use lightcraft_raw::RawError; + let cr3 = b"\0\0\0\x18ftypcrx \0\0\0\x01"; + assert_eq!(preview_reason(b"x", RawError::Unsupported("x".into())), Ok("x".to_string())); + assert!(preview_reason(b"x", RawError::Corrupt("bad strip".into())).is_ok_and(|w| w.contains("bad strip"))); + assert!(preview_reason(b"x", RawError::Limit("too big")).is_ok()); + assert!(preview_reason(b"x", RawError::Tiff(lightcraft_tiff::TiffError::MissingTag(256))).is_ok()); + assert!(preview_reason(b"x", RawError::NotRaw).is_err()); + assert!(preview_reason(cr3, RawError::Corrupt("bad".into())).is_ok_and(|w| w.starts_with("CR3 "))); + } + #[test] fn quick_jobs_show_embedded_previews_then_cached_renders() { use crate::media::QuickSource; diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 33d48b1e..282d93b7 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -33,7 +33,8 @@ use crate::media::ProbeInfo; /// File extensions LightCraft imports (lower case). pub const EXTENSIONS: &[&str] = &[ "jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "psd", "jxl", - "gif", "bmp", "heic", "avif", + "gif", "bmp", "heic", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG) + "iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", ]; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index beb55883..5d0c5d82 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 13; +pub const RENDER_CACHE_VERSION: u64 = 14; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/engine/src/tests_import.rs b/crates/engine/src/tests_import.rs index 69d75ec2..4a4cad53 100644 --- a/crates/engine/src/tests_import.rs +++ b/crates/engine/src/tests_import.rs @@ -1062,3 +1062,46 @@ fn expand_stops_at_the_walk_limits() { assert!(cut && few.len() < 8, "{few:?}"); let _ = std::fs::remove_dir_all(&d); } + +/// Raw containers LightCraft cannot decode but shows by their embedded JPEG are picked up from a folder +/// (the extensions are in `import::EXTENSIONS`; the files differ so none is a duplicate) and import as preview only. +#[test] +fn folder_import_picks_up_undecodable_raw_containers_as_preview_only() { + use lightcraft_codecs::{ChromaSubsampling, EncodeImage, EncodeMeta, Samples, encode_jpeg}; + let src = temp_dir("preview-only-exts"); + let px: Vec = (0..40 * 30).flat_map(|i| [(i % 251) as u8, 128, 200]).collect(); + let jpeg = encode_jpeg(&EncodeImage::new(40, 30, 3, Samples::U8(&px)), 90, ChromaSubsampling::S444, &EncodeMeta::default()).unwrap(); + // CRW / MRW / X3F: their magic, then the JPEG. IIQ / KDC / MOS / ERF: a TIFF whose IFD0 only points at the JPEG. + let tiff = { + let off = 8 + 2 + 24 + 4; + let mut f = b"II*\0\x08\0\0\0\x02\0".to_vec(); + for (tag, v) in [(513u16, off as u32), (514, jpeg.len() as u32)] { + f.extend_from_slice(&tag.to_le_bytes()); + f.extend_from_slice(&[4, 0, 1, 0, 0, 0]); + f.extend_from_slice(&v.to_le_bytes()); + } + f.extend_from_slice(&[0, 0, 0, 0]); + f.extend_from_slice(&jpeg); + f + }; + let with_magic = |magic: &[u8]| [magic, &[0x11u8; 64][..], &jpeg].concat(); + let files: [(&str, Vec); 7] = [ + ("a.CRW", with_magic(b"II\x1a\0\0\0HEAPCCDR")), + ("b.mrw", with_magic(b"\0MRM\0\x01\0\0")), + ("c.x3f", with_magic(b"FOVb\x02\0\x02\0")), + ("d.iiq", [&tiff[..], b"d"].concat()), + ("e.kdc", [&tiff[..], b"e"].concat()), + ("f.mos", [&tiff[..], b"f"].concat()), + ("g.erf", [&tiff[..], b"g"].concat()), + ]; + for (name, bytes) in &files { + std::fs::write(src.join(name), bytes).unwrap(); + } + let mut s = Session::new().with_fs(); + let r = s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap(); + assert_eq!(r["scanned"], 7, "{r}"); + assert_eq!(ids(&r, "imported"), 7, "{r}"); + assert_eq!(ids(&r, "failed"), 0, "{r}"); + assert!(s.catalog.photos().all(|p| p.preview_only.is_some() && p.kind == lightcraft_catalog::MediaKind::Raw && (p.width, p.height) == (40, 30))); + let _ = std::fs::remove_dir_all(&src); +} diff --git a/crates/mcp/src/headless.rs b/crates/mcp/src/headless.rs index e423e3f6..bdebec60 100644 --- a/crates/mcp/src/headless.rs +++ b/crates/mcp/src/headless.rs @@ -13,7 +13,8 @@ use crate::backend::Backend; /// File extensions recognised as photos when expanding folders. pub const PHOTO_EXTENSIONS: &[&str] = &[ "jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "psd", "jxl", "gif", - "bmp", "avif", + "bmp", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG) + "iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", ]; /// Headless backend: a [`Session`] with filesystem hooks. diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index 105690ae..d63e5254 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -83,6 +83,12 @@ pub enum RawFormat { Rw2, Pef, Srw, + /// Canon CRW (CIFF heap), the bodies before the CR2 era. + Crw, + /// Minolta MRW (`\0MRM` block container). + Mrw, + /// Sigma / Foveon X3F (`FOVb`). + X3f, /// Another TIFF-based raw (3FR, IIQ, ERF, KDC, DCR, MOS, …). OtherTiff, } @@ -119,6 +125,15 @@ pub fn probe(bytes: &[u8]) -> Option { if bytes.starts_with(b"IIU\0") { return Some(RawFormat::Rw2); } + if bytes.starts_with(b"\0MRM") { + return Some(RawFormat::Mrw); + } + if bytes.starts_with(b"FOVb") { + return Some(RawFormat::X3f); + } + if bytes.len() >= 14 && bytes.starts_with(b"II") && &bytes[6..14] == b"HEAPCCDR" { + return Some(RawFormat::Crw); + } if bytes.len() >= 10 && bytes.starts_with(b"II*\0") && &bytes[8..10] == b"CR" { return Some(RawFormat::Cr2); } @@ -130,10 +145,7 @@ pub fn probe(bytes: &[u8]) -> Option { return Some(RawFormat::Dng); } let make = t.find(lightcraft_tiff::tags::MAKE).and_then(|e| e.value.as_str()).unwrap_or_default().to_ascii_uppercase(); - let has_cfa = t.all_ifds().iter().any(|i| { - i.u16(lightcraft_tiff::tags::PHOTOMETRIC) == Some(lightcraft_tiff::tags::photometric::CFA) - || i.u16(lightcraft_tiff::tags::COMPRESSION).is_some_and(|c| c == 34713 || c == 32767 || c == 32769 || c == 32770) - }); + let has_cfa = has_raw_ifd(&t); if make.starts_with("CANON") && t.ifds.len() >= 4 && t.ifds[3].u16(lightcraft_tiff::tags::COMPRESSION) == Some(6) { return Some(RawFormat::Cr2); } @@ -149,12 +161,103 @@ pub fn probe(bytes: &[u8]) -> Option { if make.starts_with("SAMSUNG") { return Some(RawFormat::Srw); } - if has_cfa { + if has_cfa || thumbnail_shell(&t, bytes.len()).is_some() || is_preview_container(ifd0) { return Some(RawFormat::OtherTiff); } None } +/// Whether some IFD is marked as raw: CFA photometric, or a raw-only compression value (99 is not a +/// registered TIFF compression; Leaf MOS files use it for their tiled 16-bit lossless-JPEG raw). +fn has_raw_ifd(t: &Tiff) -> bool { + t.all_ifds().iter().any(|i| { + i.u16(lightcraft_tiff::tags::PHOTOMETRIC) == Some(lightcraft_tiff::tags::photometric::CFA) + || i.u16(lightcraft_tiff::tags::COMPRESSION).is_some_and(|c| c == 34713 || c == 32767 || c == 32769 || c == 32770 || c == 99) + }) +} + +/// IFD0 that is not an image at all: no `ImageWidth` / `ImageLength`, but a JPEG preview pointer or +/// `SubIFDs` (the layout of Kodak KDC files, whose pictures sit in private tags). A plain TIFF reader +/// rejects such a file as malformed even though it carries a preview. +fn is_preview_container(ifd0: &lightcraft_tiff::Ifd) -> bool { + use lightcraft_tiff::tags::{IMAGE_LENGTH, IMAGE_WIDTH, JPEG_INTERCHANGE_FORMAT, SUB_IFDS}; + !ifd0.contains(IMAGE_WIDTH) && !ifd0.contains(IMAGE_LENGTH) && (ifd0.contains(JPEG_INTERCHANGE_FORMAT) || ifd0.contains(SUB_IFDS)) +} + +/// A TIFF whose first image is only a reduced copy of a picture stored somewhere else. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ThumbnailShell { + /// Size of the first image (IFD0). + pub thumb: (u32, u32), + /// Size of the picture it stands in for. + pub full: (u32, u32), +} + +/// Some raw files are a TIFF "shell" around a private block: IFD0 holds a small RGB preview and nothing +/// in the TIFF structure marks the real image as raw (no CFA photometric, no raw compression). The +/// structure still gives them away, with a generic rule that names no maker or model: the file +/// declares a picture at least 4x larger than IFD0 in both directions, and either +/// - a `SubIFDs` child of IFD0 is that bigger image (an 8-bit grey mosaic in one such file), or +/// - the Exif `PixelXDimension` / `PixelYDimension` say so *and* the file holds enough bytes outside +/// every IFD's strips and tiles to carry that many pixels at one bit each (a downsized copy that kept +/// its old Exif size has no such block). +/// +/// A multi-page TIFF is not caught (its pages are the IFD chain, not `SubIFDs`), nor a pyramid whose +/// IFD0 is the full image. +fn thumbnail_shell(t: &Tiff, len: usize) -> Option { + use lightcraft_tiff::tags::{IMAGE_LENGTH, IMAGE_WIDTH, PIXEL_X_DIMENSION, PIXEL_Y_DIMENSION}; + let ifd0 = t.ifds.first()?; + let (w0, h0) = (ifd0.u32(IMAGE_WIDTH)?, ifd0.u32(IMAGE_LENGTH)?); + if w0 == 0 || h0 == 0 { + return None; + } + let much_bigger = |(w, h): (u32, u32)| u64::from(w) >= 4 * u64::from(w0) && u64::from(h) >= 4 * u64::from(h0); + let shell = |full| Some(ThumbnailShell { thumb: (w0, h0), full }); + let mut best: Option<(u32, u32)> = None; + for s in &ifd0.sub_ifds { + if s.image().is_ok() + && let (Some(w), Some(h)) = (s.u32(IMAGE_WIDTH), s.u32(IMAGE_LENGTH)) + && much_bigger((w, h)) + && best.is_none_or(|(bw, bh)| u64::from(w) * u64::from(h) > u64::from(bw) * u64::from(bh)) + { + best = Some((w, h)); + } + } + if let Some(full) = best { + return shell(full); + } + let exif = t.exif()?; + let full = (exif.u32(PIXEL_X_DIMENSION)?, exif.u32(PIXEL_Y_DIMENSION)?); + if !much_bigger(full) { + return None; + } + // bytes covered by image data (strips and tiles shared between IFDs, e.g. IFD0 and its thumbnail, count once) + let mut seen = std::collections::BTreeMap::new(); + for ifd in t.all_ifds() { + if let Ok(info) = ifd.image() { + for c in info.chunks(len as u64) { + seen.insert(c.offset, c.len); + } + } + } + let covered = seen.values().fold(0u64, |a, &l| a.saturating_add(l)); + let spare = (len as u64).saturating_sub(covered); + (spare.saturating_mul(8) >= u64::from(full.0) * u64::from(full.1)).then_some(())?; + shell(full) +} + +/// Why [`decode`] gives up on a file [`probe`] called [`RawFormat::OtherTiff`]. +fn other_tiff_reason(bytes: &[u8]) -> String { + let t = Tiff::parse_with(bytes, &lightcraft_tiff::ParseOptions { max_ifds: 256, ..Default::default() }).ok(); + match t.as_ref().filter(|t| !has_raw_ifd(t)).and_then(|t| thumbnail_shell(t, bytes.len())) { + Some(s) => format!( + "{}x{} raw image in a private block, not decoded yet (the file's first image is a {}x{} reduced copy)", + s.full.0, s.full.1, s.thumb.0, s.thumb.1 + ), + None => "OtherTiff files are not decoded yet".to_string(), + } +} + /// Decode a raw file. pub fn decode(bytes: &[u8]) -> Result { decode_with(bytes, Mode::Full) @@ -195,6 +298,7 @@ fn decode_with(bytes: &[u8], mode: Mode) -> Result { RawFormat::Rw2 => vendor::rw2::decode(bytes, mode), RawFormat::Pef => vendor::pef::decode(bytes, mode), RawFormat::Orf => vendor::orf::decode(bytes, mode), + RawFormat::OtherTiff => Err(RawError::Unsupported(other_tiff_reason(bytes))), other => Err(RawError::Unsupported(format!("{other:?} files are not decoded yet"))), } } @@ -623,6 +727,136 @@ mod tests { assert_eq!(BlackLevel::uniform(7.0).at(9, 9, 0, 3), 7.0); } + // --- TIFF shells around a private raw block (thumbnail-only IFD0) --- + + use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value, tags as t}; + + /// An uncompressed 8-bit RGB image IFD. + fn rgb_ifd(w: u32, h: u32) -> IfdBuilder { + let mut ifd = IfdBuilder::new(); + ifd.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + ifd.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![8, 8, 8])); + ifd.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3])); + ifd.set(t::PHOTOMETRIC, Value::Short(vec![2])); + ifd.set(t::COMPRESSION, Value::Short(vec![1])); + ifd.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![vec![90u8; (w * h * 3) as usize]] }); + ifd + } + + fn exif_size(w: u32, h: u32) -> IfdBuilder { + IfdBuilder::new().with(t::PIXEL_X_DIMENSION, Value::Long(vec![w])).with(t::PIXEL_Y_DIMENSION, Value::Long(vec![h])) + } + + fn write(chain: &[IfdBuilder]) -> Vec { + TiffWriter::default().write(chain).unwrap() + } + + const SHELL_REASON: &str = "raw image in a private block"; + + /// A bigger image in a `SubIFDs` child of a small IFD0 (the layout of an 8-bit-mosaic TIFF raw). + #[test] + fn thumbnail_with_a_bigger_sub_ifd_is_an_undecodable_raw() { + let mut ifd0 = rgb_ifd(16, 12); + ifd0.add_sub_ifd(rgb_ifd(64, 48)); + let bytes = write(&[ifd0]); + assert_eq!(probe(&bytes), Some(RawFormat::OtherTiff)); + let Err(RawError::Unsupported(why)) = decode(&bytes) else { panic!("expected Unsupported") }; + assert!(why.contains("64x48") && why.contains("16x12") && why.contains(SHELL_REASON), "{why}"); + // a sub-IFD that is not much bigger (a pyramid level) does not count + let mut ifd0 = rgb_ifd(16, 12); + ifd0.add_sub_ifd(rgb_ifd(32, 24)); + assert_eq!(probe(&write(&[ifd0])), None); + } + + /// Exif says the picture is far bigger than IFD0 and the file holds a block that could carry it. + #[test] + fn thumbnail_with_a_private_block_behind_the_exif_size_is_an_undecodable_raw() { + let mut ifd0 = rgb_ifd(16, 12); + ifd0.set_child(t::EXIF_IFD, exif_size(400, 300)); + let mut bytes = write(&[ifd0.clone()]); + // no private block: a downsized copy that kept its old Exif size is an ordinary image + assert_eq!(probe(&bytes), None); + // 400 x 300 pixels at one bit each is 15000 bytes: 14000 is not enough, 16000 is + bytes.extend(std::iter::repeat_n(7u8, 14_000)); + assert_eq!(probe(&bytes), None); + bytes.extend(std::iter::repeat_n(7u8, 2_000)); + assert_eq!(probe(&bytes), Some(RawFormat::OtherTiff)); + let Err(RawError::Unsupported(why)) = decode(&bytes) else { panic!("expected Unsupported") }; + assert!(why.contains("400x300") && why.contains("16x12") && why.contains(SHELL_REASON), "{why}"); + // truncating it never panics + for n in (0..bytes.len()).step_by(37) { + let _ = probe(&bytes[..n]); + let _ = decode(&bytes[..n]); + } + } + + /// Ordinary TIFFs stay images: the full picture first, equal Exif size, or later pages that are bigger. + #[test] + fn ordinary_tiffs_are_not_shells() { + let padded = |chain: &[IfdBuilder]| { + let mut b = write(chain); + b.extend(std::iter::repeat_n(0u8, 100_000)); + b + }; + let mut full = rgb_ifd(400, 300); + full.set_child(t::EXIF_IFD, exif_size(400, 300)); + assert_eq!(probe(&padded(&[full])), None); + // the Exif size is much bigger than IFD0, with spare bytes, but IFD0 is not tiny next to it + let mut near = rgb_ifd(120, 90); + near.set_child(t::EXIF_IFD, exif_size(400, 300)); + assert_eq!(probe(&padded(&[near])), None); + // a multi-page TIFF whose first page is small + assert_eq!(probe(&padded(&[rgb_ifd(16, 12), rgb_ifd(400, 300)])), None); + // no dimensions at all + assert_eq!(probe(&padded(&[IfdBuilder::new().with(t::MAKE, Value::Ascii("X".into()))])), None); + } + + // --- containers that are recognised but not decoded, with a preview --- + + #[test] + fn private_containers_are_recognised_by_their_magic() { + assert_eq!(probe(b"II\x1a\0\0\0HEAPCCDR\0\0"), Some(RawFormat::Crw)); + assert_eq!(probe(b"\0MRM\0\x01\0\0"), Some(RawFormat::Mrw)); + assert_eq!(probe(b"FOVb\x02\0\x02\0"), Some(RawFormat::X3f)); + for f in [RawFormat::Crw, RawFormat::Mrw, RawFormat::X3f] { + assert!(!f.is_supported()); + } + assert!(matches!(decode(b"\0MRM\0\x01\0\0"), Err(RawError::Unsupported(w)) if w.contains("Mrw"))); + // too short for the CIFF signature + assert_eq!(probe(b"II\x1a\0\0\0HEAP"), None); + } + + /// An IFD0 with a JPEG pointer but no `ImageWidth` / `ImageLength` (the Kodak KDC layout) is a raw + /// container with a preview; a TIFF reader would call it malformed. + #[test] + fn ifd0_without_dimensions_but_with_a_jpeg_is_a_preview_container() { + let jpeg: Vec = [&[0xffu8, 0xd8, 0xff, 0xc0, 0, 0x0b, 8, 0, 1, 0, 1, 1, 1, 0x11, 0][..], &[0x55; 40], &[0xff, 0xd9]].concat(); + let off = 8 + 2 + 24 + 4; + let mut f = b"II*\0\x08\0\0\0\x02\0".to_vec(); + for (tag, v) in [(513u16, off as u32), (514, jpeg.len() as u32)] { + f.extend_from_slice(&tag.to_le_bytes()); + f.extend_from_slice(&[4, 0, 1, 0, 0, 0]); + f.extend_from_slice(&v.to_le_bytes()); + } + f.extend_from_slice(&[0, 0, 0, 0]); + f.extend_from_slice(&jpeg); + assert_eq!(probe(&f), Some(RawFormat::OtherTiff)); + assert!(matches!(decode(&f), Err(RawError::Unsupported(_)))); + assert_eq!(embedded_preview(&f), Some(jpeg)); + // no pointer and no sub-IFDs: just a broken TIFF + let g = write(&[IfdBuilder::new().with(t::MAKE, Value::Ascii("KODAK".into()))]); + assert_eq!(probe(&g), None); + } + + /// Compression 99 (not a registered TIFF value; Leaf MOS tiles) marks a raw even without a CFA tag. + #[test] + fn private_compression_99_is_a_raw() { + let mut ifd = rgb_ifd(16, 12); + ifd.set(t::COMPRESSION, Value::Short(vec![99])); + assert_eq!(probe(&write(&[ifd])), Some(RawFormat::OtherTiff)); + } + #[test] fn probe_rejects_junk() { assert_eq!(probe(b""), None); diff --git a/crates/raw/src/preview.rs b/crates/raw/src/preview.rs index 42b5d70b..6ec326ea 100644 --- a/crates/raw/src/preview.rs +++ b/crates/raw/src/preview.rs @@ -91,9 +91,13 @@ pub fn embedded_preview(bytes: &[u8]) -> Option> { let j = crate::vendor::raf::header(bytes).ok()?.jpeg?; return is_dct_jpeg(j).then(|| trim_eoi(j).to_vec()); } - if crate::probe(bytes) == Some(crate::RawFormat::Cr3) { + let format = crate::probe(bytes); + if format == Some(crate::RawFormat::Cr3) { return cr3_preview(bytes).map(|j| trim_eoi(j).to_vec()); } + if matches!(format, Some(crate::RawFormat::Crw | crate::RawFormat::Mrw | crate::RawFormat::X3f)) { + return scan_for_jpeg(bytes); + } let tiff = Tiff::parse(bytes).ok()?; let mut found: Vec<&[u8]> = Vec::new(); for ifd in tiff.all_ifds() { @@ -117,7 +121,112 @@ pub fn embedded_preview(bytes: &[u8]) -> Option> { if let Some(p) = crate::vendor::orf::preview(bytes) { found.push(p); } - found.into_iter().filter(|s| is_dct_jpeg(s)).max_by_key(|s| s.len()).map(|s| trim_eoi(s).to_vec()) + let best = found.into_iter().filter(|s| is_dct_jpeg(s)).max_by_key(|s| s.len()).map(|s| trim_eoi(s).to_vec()); + // a TIFF raw we can't decode whose preview no tag points to (Leaf MOS, Epson ERF) + best.or_else(|| format.filter(|f| !f.is_supported()).and_then(|_| scan_for_jpeg(bytes))) +} + +/// The end of the DCT JPEG whose SOI is at `soi` (exclusive index after its EOI) and its component count, +/// walking the marker segments and the entropy-coded data. `None` when it is not a complete, displayable +/// (baseline or progressive) JPEG, so a stray `D8 FF` in other data is rejected. `*reach` is left at +/// the furthest byte looked at (what the walk cost; see [`scan_for_jpeg`]). +fn jpeg_extent(b: &[u8], soi: usize, reach: &mut usize) -> Option<(usize, u8)> { + let mut i = soi + 2; + let r = walk_jpeg(b, &mut i); + *reach = i; + r +} + +fn walk_jpeg(b: &[u8], i: &mut usize) -> Option<(usize, u8)> { + let mut components = None; + for _ in 0..4096 { + if *b.get(*i)? != 0xff { + return None; + } + while *b.get(*i)? == 0xff { + *i += 1; + } + let m = *b.get(*i)?; + *i += 1; + match m { + 0xd9 => return components.map(|c| (*i, c)), + 0x01 | 0xd0..=0xd8 => continue, + _ => {} + } + let len = usize::from(u16::from_be_bytes([*b.get(*i)?, *b.get(*i + 1)?])); + if len < 2 { + return None; + } + match m { + 0xc0..=0xc2 => { + // precision, height, width, component count + let h = u16::from_be_bytes([*b.get(*i + 3)?, *b.get(*i + 4)?]); + let w = u16::from_be_bytes([*b.get(*i + 5)?, *b.get(*i + 6)?]); + if h == 0 || w == 0 || components.is_some() { + return None; + } + components = Some(*b.get(*i + 7)?); + } + 0xc3 | 0xc5..=0xc7 | 0xc9..=0xcb | 0xcd..=0xcf => return None, // lossless, arithmetic, hierarchical + _ => {} + } + *i += len; + if m == 0xda { + // entropy-coded data up to the next marker that is not a stuffed 0xff or a restart + loop { + *i += b.get(*i..)?.iter().position(|&x| x == 0xff)?; + match *b.get(*i + 1)? { + 0x00 | 0xd0..=0xd7 => *i += 2, + 0xff => *i += 1, + _ => break, + } + } + } + } + None +} + +/// Last resort for the containers whose structure is not walked (Canon CRW, Minolta MRW, Sigma X3F, TIFF raws +/// with private blocks such as Leaf MOS or Epson ERF): the largest complete colour DCT JPEG stored as a plain +/// byte run in the file. A one-component JPEG is skipped (a Canon PowerShot CRW stores its sensor mosaic as one). +/// +/// Evidence: in the Minolta and Epson files the first byte of the stored preview is not `ff` (it is `00`, `02` or +/// `ee`) while the rest is a regular JPEG, so a run is found by its `d8 ff` and starts one byte earlier, +/// with that byte restored to `ff`. +/// +/// A crafted file can hold millions of JPEG-like starts whose walks each run to its end: the walks +/// share a budget of eight times the file's size, so the scan stays linear (a real file's candidates +/// fail within a few bytes, or are the preview itself). +fn scan_for_jpeg(data: &[u8]) -> Option> { + let mut best: Option<(usize, usize)> = None; + let mut at = 1; + let mut budget = data.len().saturating_mul(8).max(1 << 20); + while let Some(off) = data.get(at..).and_then(|s| s.iter().position(|&b| b == 0xd8)) { + let d8 = at + off; + at = d8 + 1; + if data.get(d8 + 1) != Some(&0xff) || !matches!(data.get(d8 + 2), Some(0xc0..=0xc4 | 0xdb | 0xe0..=0xef | 0xfe)) { + continue; + } + let start = d8 - 1; // `at` starts at 1, so d8 >= 1 + let mut reach = start; + let found = jpeg_extent(data, start, &mut reach); + budget = budget.saturating_sub(reach.saturating_sub(start)); + if budget == 0 { + break; + } + if let Some((end, 3)) = found { + if best.is_none_or(|(s, e)| end - start > e - s) { + best = Some((start, end)); + } + at = end; + } + } + let (start, end) = best?; + let mut jpeg = data.get(start..end)?.to_vec(); + if let Some(first) = jpeg.first_mut() { + *first = 0xff; + } + Some(jpeg) } /// Canon CR3: the full-size JPEG track (see [`lightcraft_meta::cr3`]), else the `PRVW` / `THMB` boxes. @@ -242,4 +351,66 @@ mod tests { file.truncate(at as usize + 100); assert_eq!(embedded_preview(&file).unwrap(), small); } + + /// A structurally complete DCT JPEG (no real image data): `nc` components, `n` entropy bytes. + fn scan_jpeg(nc: u8, n: usize) -> Vec { + let mut j = vec![0xff, 0xd8, 0xff, 0xdb, 0x00, 0x43, 0x00]; + j.extend(std::iter::repeat_n(8u8, 64)); + j.extend_from_slice(&[0xff, 0xc0, 0x00, 8 + 3 * nc, 8, 0, 16, 0, 24, nc]); + for c in 0..nc { + j.extend_from_slice(&[c + 1, 0x11, 0]); + } + j.extend_from_slice(&[0xff, 0xda, 0x00, 6 + 2 * nc, nc]); + for c in 0..nc { + j.extend_from_slice(&[c + 1, 0]); + } + j.extend_from_slice(&[0, 63, 0]); + j.extend(std::iter::repeat_n(0x55u8, n)); + j.extend_from_slice(&[0xff, 0x00, 0x12, 0xff, 0xd0, 0x34, 0xff, 0xd9]); + j + } + + /// A crafted container full of JPEG-like starts (each walk running on through the ones after it) + /// is scanned in linear time: the walks share a budget (without it this 40 MB file takes about a minute). + #[test] + fn a_scan_through_many_fake_jpegs_stays_linear() { + let mut unit = vec![0u8, 0xd8, 0xff, 0xdb, 0x00, 0x02, 0xff, 0xda, 0x00, 0x02]; + unit.extend(std::iter::repeat_n(0x55u8, 190)); + let mut f = b"\0MRM\0\x01\0\0".to_vec(); + for _ in 0..200_000 { + f.extend_from_slice(&unit); + } + let t = std::time::Instant::now(); + assert_eq!(scan_for_jpeg(&f), None); + assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed()); + } + + /// Containers whose structure we do not walk give up their largest colour JPEG, found by scanning. + #[test] + fn scanned_containers_give_the_largest_colour_jpeg() { + let (small, big, mosaic) = (scan_jpeg(3, 100), scan_jpeg(3, 900), scan_jpeg(1, 5000)); + let heads: [&[u8]; 3] = [b"II\x1a\0\0\0HEAPCCDR", b"\0MRM\0\x01\0\0", b"FOVb\x02\0\x02\0"]; + for head in heads { + let mut f = head.to_vec(); + for part in [&small, &mosaic, &big] { + f.extend_from_slice(&[0xff, 0xd8, 0xff, 0x00, 0x00]); // a stray SOI-like run: not a JPEG + f.extend_from_slice(part); + f.extend_from_slice(&[0u8; 7]); + } + assert!(crate::probe(&f).is_some_and(|p| !p.is_supported()), "{head:?}"); + assert_eq!(embedded_preview(&f).as_deref(), Some(&big[..]), "{head:?}"); + for n in (0..f.len()).step_by(11) { + let _ = embedded_preview(&f[..n]); + } + } + // a preview whose first byte was overwritten (Minolta and Epson files store 00, 02 or ee there) is restored + let mut f = b"\0MRM\0\x01\0\0".to_vec(); + f.push(0x02); + f.extend_from_slice(&big[1..]); + assert_eq!(embedded_preview(&f).as_deref(), Some(&big[..])); + // only a one-component JPEG, or none: nothing + let mut f = b"FOVb\x02\0\x02\0".to_vec(); + f.extend_from_slice(&mosaic); + assert_eq!(embedded_preview(&f), None); + } } diff --git a/crates/raw/tests/corpus.rs b/crates/raw/tests/corpus.rs index 42cca03a..b3c0ade9 100644 --- a/crates/raw/tests/corpus.rs +++ b/crates/raw/tests/corpus.rs @@ -534,3 +534,38 @@ fn corpus_fujifilm_compressed_samples() { } eprintln!("verified {seen} Fujifilm compressed sensor arrays"); } + +/// The container rule for TIFF "shells" (a small IFD0 next to a private raw block) must not touch real +/// raws: every corpus file keeps the format its maker implies, and none is described as a private block. +#[test] +fn corpus_raws_keep_their_container_and_are_not_thumbnail_shells() { + let dir = corpus_root().join("raw"); + let Ok(rd) = std::fs::read_dir(&dir) else { + eprintln!("skip: {} absent", dir.display()); + return; + }; + let mut checked = 0; + for p in rd.flatten().map(|e| e.path()).filter(|p| p.is_file()) { + let name = p.file_name().unwrap().to_string_lossy().to_lowercase(); + let expected = match name.split(['-', '.']).next().unwrap_or_default() { + "arw" => RawFormat::Arw, + "cr2" => RawFormat::Cr2, + "cr3" => RawFormat::Cr3, + "dng" => RawFormat::Dng, + "nef" => RawFormat::Nef, + "nrw" => RawFormat::Nef, // probed as Nef when the file has several IFDs + "orf" => RawFormat::Orf, + "pef" => RawFormat::Pef, + "raf" => RawFormat::Raf, + "rw2" | "rwl" | "raw" => RawFormat::Rw2, + _ => continue, + }; + let bytes = std::fs::read(&p).unwrap(); + assert_eq!(probe(&bytes), Some(expected), "{name}"); + if let Err(RawError::Unsupported(why)) = probe_info(&bytes) { + assert!(!why.contains("private block"), "{name}: {why}"); + } + checked += 1; + } + eprintln!("{checked} corpus raws keep their container"); +} diff --git a/docs/parity.md b/docs/parity.md index ef1da34f..b9c5eed5 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -135,7 +135,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported; profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported | | LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) | | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars | -| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop; full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode, palette (indexed-colour) TIFF (refused by the `tiff` crate) | +| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop; full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS TIFFs: IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan), and any decode failure of a recognised raw, not only an unsupported variant; ones with no JPEG still fail with "without an embedded preview", HEIC/AVIF decode, palette (indexed-colour) TIFF (refused by the `tiff` crate) | | LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files plus 6 exact-sensor CR3 cases (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; private Z 8 crop verified. | | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic | | LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving | diff --git a/docs/whats-new.md b/docs/whats-new.md index e1b74676..99ed3157 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -22,6 +22,12 @@ are unchanged; photos imported before this change get it when imported again. - Canon CR3 raws now develop from their sensor data: lossless RAW and C-RAW, checked sample for sample on the EOS M50, R100 and R8. CR3 files the decoder can't read yet still open from their embedded JPEG, as before. +- Canon CRW, Minolta MRW, Sigma X3F, Kodak KDC, Leaf MOS and Epson ERF files that LightCraft can't decode yet + now import as "preview only" with their embedded JPEG instead of failing. A raw whose data is damaged but whose + preview is intact does the same. +- Raw files whose raw data sits in a private block of a TIFF (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS + TIFFs) are no longer opened as a thumbnail-sized ordinary image. They are recognised as raws LightCraft can't decode + yet and import as "preview only", with the reason. - Sony ILCE-7M4 downsized lossless ARWs now decode subsampled YCbCr tiles into linear RGB, preserving RAW editing & full-resolution export instead of using embedded JPEG previews. - Sony A7R II (and other) raws whose camera JPEG is lens-corrected no longer open grey and too dark (issue #232): the