Merge PR #727

# Conflicts:
#	crates/raw/src/vendor/nef.rs
#	docs/parity-checklist.md

crates/raw/src/lib.rs

@@ -325,10 +325,13 @@ pub fn decode(bytes: &[u8]) -> Result<RawImage> {
 /// [`RawImage::info`] of the decoded image. Black and white levels are not included (some formats
 /// measure them from the samples).
 ///
-/// A few uncompressed vendor formats derive part of this from the samples themselves (Nikon
-/// NEF: optically masked trailing columns; Olympus ORF: the bit depth of 16-bit files, and the CFA
-/// phase of files without an Exif `CFAPattern`; Pentax PEF without crop tags: dark borders); for
-/// those the samples are read (unpacked, nothing to decompress) and dropped.
+/// A few vendor formats derive part of this from the samples themselves (Nikon NEF: optically
+/// masked trailing columns; Olympus ORF: the bit depth of 16-bit files, and the CFA phase of files
+/// without an Exif `CFAPattern`; Pentax PEF without crop tags: dark borders). Those read the
+/// samples they need and drop them: ORF and PEF unpack their uncompressed data; NEF reads only the
+/// first 128 rows (`vendor::nef::MASK_ROWS`), which both the probe and the full decode judge the masked
+/// columns from, so a compressed NEF probes in a few milliseconds instead of decoding its whole
+/// Huffman stream (issue #708).
 pub fn probe_info(bytes: &[u8]) -> Result<RawInfo> {
     decode_with(bytes, Mode::Header).map(RawImage::into_info)
 }
@@ -348,7 +351,7 @@ fn decode_with(bytes: &[u8], mode: Mode) -> Result<RawImage> {
         RawFormat::Dng => dng::decode(bytes, mode).map(lift_clipped),
         RawFormat::Cr2 => vendor::cr2::decode(bytes, mode),
         RawFormat::Cr3 => vendor::cr3::decode(bytes, mode),
-        RawFormat::Nef | RawFormat::Nrw => vendor::nef::decode(bytes),
+        RawFormat::Nef | RawFormat::Nrw => vendor::nef::decode(bytes, mode),
         RawFormat::Arw => vendor::arw::decode(bytes, mode),
         RawFormat::Raf => vendor::raf::decode(bytes, mode),
         RawFormat::Rw2 => vendor::rw2::decode(bytes, mode),

crates/raw/src/vendor/nef.rs

@@ -9,8 +9,8 @@
 //! the JPEG XS markers SOC + CAP (ISO/IEC 21122-1, `FF10 FF50`) and have no `0x0096` table (issue #193).
 
 use super::{nefc, white_from_data};
-use crate::tiffraw::{Packing, read_image};
-use crate::{BlackLevel, Cfa, ColorData, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
+use crate::tiffraw::{Packing, check_image, read_image};
+use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
 use lightcraft_geom::Orientation;
 use lightcraft_tiff::image::ImageInfo;
 use lightcraft_tiff::tags::{self as t, photometric};
@@ -88,16 +88,27 @@ fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
         .max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0).saturating_mul(i.u64(t::IMAGE_LENGTH).unwrap_or(0)))
 }
 
-/// Width without the optically masked columns some bodies append on the right: trailing columns (at most 64)
-/// whose mean is below 1% of the white level while the image interior is brighter. Kept even for CFA phase.
-pub(crate) fn trailing_masked_columns(d: &[u16], w: usize, h: usize, white: f32) -> usize {
-    if w < 128 || h == 0 {
+/// Rows the masked-column test reads, from the top of the frame. The header probe ([`Mode::Header`]) decodes
+/// only these (a compressed strip is one sequential Huffman stream, so it can stop here; an uncompressed one reads
+/// just its first strips), and the full decode judges the active width from the same rows, so
+/// [`crate::probe_info`] equals [`RawImage::info`] of the decoded image (issue #708). On every corpus NEF the
+/// answer is the same for 8 to 512 rows as for the whole frame: the masked columns decode to 0 (compressed) or
+/// ~19 (uncompressed) while live columns sit at the black level (~150 at 12 bit, ~600 at 14 bit) or above.
+pub(crate) const MASK_ROWS: usize = 128;
+
+/// Width without the optically masked columns some bodies append on the right (the D5100 and D7000 write 8, with
+/// no maker-note tag that says so, in compressed and uncompressed files alike): trailing columns (at most 64) whose
+/// mean over the first [`MASK_ROWS`] rows of `d` is below 1% of the nominal white level and a tenth of the image
+/// interior. Kept even for CFA phase.
+pub(crate) fn trailing_masked_columns(d: &[u16], w: usize, bits: u32) -> usize {
+    let rows = d.len().checked_div(w).unwrap_or(0).min(MASK_ROWS);
+    if w < 128 || rows == 0 {
         return w;
     }
-    let step = (h / 256).max(1);
-    let col_mean = |x: usize| (0..h).step_by(step).map(|y| d[y * w + x] as f64).sum::<f64>() / h.div_ceil(step) as f64;
+    let col_mean = |x: usize| (0..rows).map(|y| d.get(y * w + x).copied().unwrap_or(0) as f64).sum::<f64>() / rows as f64;
     let interior = (w / 4..w * 3 / 4).step_by(w / 64).map(col_mean).sum::<f64>() / 32.0;
-    let dark = (white as f64 * 0.01).min(interior * 0.1);
+    let white = ((1u32 << bits.clamp(1, 16)) - 1) as f64;
+    let dark = (white * 0.01).min(interior * 0.1);
     let mut aw = w;
     while aw > w - 64 && col_mean(aw - 1) < dark {
         aw -= 1;
@@ -105,7 +116,7 @@ pub(crate) fn trailing_masked_columns(d: &[u16], w: usize, h: usize, white: f32)
     aw & !1
 }
 
-pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
+pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
     let tiff = Tiff::parse(bytes)?;
     let ifd0 = &tiff.ifds[0];
     let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Unsupported("NEF without a CFA image IFD".into()))?;
@@ -115,10 +126,15 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
     let make = ifd0.string(t::MAKE).unwrap_or_default();
     let mn =
         tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make));
+    // the header probe reads only the rows the masked-column test needs
+    let rows = match mode {
+        Mode::Full => h,
+        Mode::Header => h.min(MASK_ROWS),
+    };
     let data = if info.compression == t::compression::NIKON {
-        compressed(bytes, &info, mn.as_ref())?
+        compressed(bytes, &info, mn.as_ref(), rows)?
     } else {
-        uncompressed(bytes, &info, tiff.order, mn.as_ref())?
+        uncompressed(bytes, &info, tiff.order, mn.as_ref(), rows)?
     };
     let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float NEF".into())) };
 
@@ -145,8 +161,14 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
         _ => Cfa::bayer_static("RGGB"),
     };
     let body = measured_body(&ifd0.string(t::MODEL).unwrap_or_default(), w, h, bits);
-    let white = body.map_or_else(|| white_from_data(samples, bits), |b| b.white);
-    let active_w = trailing_masked_columns(samples, w, h, white);
+    let active_w = trailing_masked_columns(samples, w, bits);
+    // the white level is measured from the whole image (or known for a measured body); the probe (no samples in its
+    // result) takes the nominal one
+    let white = match (body, mode) {
+        (Some(b), _) => b.white,
+        (None, Mode::Full) => white_from_data(samples, bits),
+        (None, Mode::Header) => ((1u32 << bits.clamp(1, 16)) - 1) as f32,
+    };
     let crop = match body {
         Some(&MeasuredBody { crop: (x, y, cw, ch), .. }) if x + cw <= active_w && y + ch <= h => Rect::new(x, y, cw, ch),
         _ => default_crop(mn.as_ref(), active_w, h),
@@ -159,7 +181,10 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
         width: w,
         height: h,
         cpp: 1,
-        data,
+        data: match mode {
+            Mode::Full => data,
+            Mode::Header => RawData::U16(Vec::new()),
+        },
         cfa: Some(cfa),
         bits,
         black,
@@ -173,14 +198,14 @@ pub(crate) fn decode(bytes: &[u8]) -> Result<RawImage> {
         opcodes: OpcodeLists::default(),
         metadata,
     };
-    img.validate()?;
+    img.validate_for(mode)?;
     Ok(img)
 }
 
 /// Uncompressed strips: 16-bit words or 12-bit packed, told apart by the strip size. Packed 12-bit data is an MSB-first
 /// byte stream, except in files with an NRW data block (Coolpix), where it is MSB-first inside little-endian 32-bit
-/// words (the rows are whole words).
-fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder, mn: Option<&makernote::MakerNote>) -> Result<RawData> {
+/// words (the rows are whole words). Reads the first `rows` rows (the strips that hold them) of the `info.height`.
+fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder, mn: Option<&makernote::MakerNote>, rows: usize) -> Result<RawData> {
     let (w, h) = (info.width as usize, info.height as usize);
     let bits = info.bits() as u32;
     let row_samples = w * info.samples_per_pixel as usize;
@@ -196,11 +221,18 @@ fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder, mn: Option<&ma
     } else {
         Packing::Msb
     };
-    read_image(bytes, info, order, packing)
+    if rows >= h {
+        return read_image(bytes, info, order, packing);
+    }
+    // the whole image's limits apply either way; then only the strips holding the first `rows` rows are unpacked
+    check_image(bytes, info)?;
+    let top = ImageInfo { height: rows as u32, ..info.clone() };
+    read_image(bytes, &top, order, packing)
 }
 
-/// Nikon Huffman-compressed strip (see [`super::nefc`]); the decode table is maker note `0x0096`.
-fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>) -> Result<RawData> {
+/// Nikon Huffman-compressed strip (see [`super::nefc`]); the decode table is maker note `0x0096`. Decodes the first
+/// `rows` rows of the `info.height`.
+fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, rows: usize) -> Result<RawData> {
     let starts_with = |magic: &[u8]| {
         let start = info.chunks(bytes.len() as u64).first().and_then(|c| usize::try_from(c.offset).ok());
         start.and_then(|o| bytes.get(o..o.checked_add(magic.len())?)) == Some(magic)
@@ -223,7 +255,7 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>)
     let end = last.offset.saturating_add(last.len).min(bytes.len() as u64);
     let src = usize::try_from(first.offset).ok().zip(usize::try_from(end).ok()).and_then(|(a, b)| bytes.get(a..b));
     let src = src.ok_or_else(|| RawError::Corrupt("NEF: image data outside the file".into()))?;
-    Ok(RawData::U16(nefc::decode(src, info.width as usize, info.height as usize, bits, &table)?))
+    Ok(RawData::U16(nefc::decode_rows(src, info.width as usize, info.height as usize, rows, bits, &table)?))
 }
 
 #[cfg(test)]
@@ -366,6 +398,34 @@ mod tests {
         }
     }
 
+    /// Issue #708: the active width is judged from the first [`MASK_ROWS`] rows in both modes, so the header probe
+    /// (which reads only those rows' strips) and the full decode agree whether the masked columns are there or not.
+    #[test]
+    fn masked_columns_are_judged_from_the_top_rows_in_both_modes() {
+        let (w, h, rps) = (256usize, MASK_ROWS + 64, 8u32);
+        let file = |masked_rows: std::ops::Range<usize>| {
+            let words: Vec<u8> = (0..w * h)
+                .map(|i| if masked_rows.contains(&(i / w)) && i % w >= w - 8 { 0u16 } else { 1000 + (i % 7) as u16 })
+                .flat_map(|v| v.to_be_bytes())
+                .collect();
+            nef(1, 14, words.chunks(w * 2 * rps as usize).map(|c| c.to_vec()).collect(), w as u32, h as u32, rps)
+        };
+        for (masked_rows, expected_width) in [(0..h, w - 8), (0..MASK_ROWS, w - 8), (MASK_ROWS..h, w), (0..0, w)] {
+            let bytes = file(masked_rows.clone());
+            let full = crate::decode(&bytes).unwrap();
+            assert_eq!(full.active_area.width, expected_width, "masked rows {masked_rows:?}");
+            assert_eq!(full.crop.width, expected_width);
+            assert_eq!(crate::probe_info(&bytes).unwrap(), full.info(), "masked rows {masked_rows:?}");
+        }
+        // the shared rule on its own: nominal white, trailing columns at most 64, even width
+        let d: Vec<u16> = (0..w * 4).map(|i| if i % w >= w - 9 { 0 } else { 600 }).collect();
+        assert_eq!(trailing_masked_columns(&d, w, 14), w - 10, "nine dark columns trim to an even width");
+        assert_eq!(trailing_masked_columns(&d, w, 12), w - 10);
+        assert_eq!(trailing_masked_columns(&d[..w - 1], w, 14), w, "less than a row of samples leaves the width alone");
+        assert_eq!(trailing_masked_columns(&[0; 64], 64, 14), 64, "narrow images are left alone");
+        assert_eq!(trailing_masked_columns(&[], 0, 14), 0);
+    }
+
     #[test]
     fn black_level_tag_is_in_14_bit_units() {
         let (w, h) = (16usize, 4usize);

crates/raw/src/vendor/nefc.rs

@@ -225,17 +225,28 @@ fn fixed_word_table(bits: u32) -> Vec<Option<i32>> {
     lut
 }
 
-/// Decode a `w × h` Nikon Huffman-compressed strip `src` with `bits`-bit samples.
+/// Decode a whole `w × h` Nikon Huffman-compressed strip `src` with `bits`-bit samples.
+#[cfg(test)]
 pub(crate) fn decode(src: &[u8], w: usize, h: usize, bits: u32, table: &DecodeTable) -> Result<Vec<u16>> {
+    decode_rows(src, w, h, h, bits, table)
+}
+
+/// The first `rows` rows (`w × rows` samples) of a `w × h` strip. The stream is sequential, so a header-only probe
+/// decodes just the rows its masked-column test looks at (issue #708) instead of the whole image. The size and
+/// truncation checks still cover the whole image; the "lossy after split" end-of-strip rule (and corruption
+/// further down) can only be seen when the whole strip is decoded.
+pub(crate) fn decode_rows(src: &[u8], w: usize, h: usize, rows: usize, bits: u32, table: &DecodeTable) -> Result<Vec<u16>> {
     if bits != 12 && bits != 14 {
         return Err(RawError::Unsupported(format!("Nikon compressed NEF with {bits}-bit samples")));
     }
-    let n = w.checked_mul(h).filter(|&n| n > 0 && n <= MAX_SAMPLES).ok_or(RawError::Limit("NEF image size"))?;
+    let whole = w.checked_mul(h).filter(|&n| n > 0 && n <= MAX_SAMPLES).ok_or(RawError::Limit("NEF image size"))?;
     // every code word is at least 2 bits long: a strip that can't hold the image is truncated (also bounds the
     // allocation by the file size)
-    if n / 4 > src.len() {
+    if whole / 4 > src.len() {
         return Err(RawError::Corrupt(format!("NEF: {} bytes of compressed data for {w}x{h} pixels", src.len())));
     }
+    let rows = rows.min(h);
+    let n = w.saturating_mul(rows);
     let huff = Huffman::new(
         &match (&table.encoding, bits) {
             (Encoding::Lossless, 12) => LOSSLESS_12,
@@ -286,7 +297,7 @@ pub(crate) fn decode(src: &[u8], w: usize, h: usize, bits: u32, table: &DecodeTa
             return Err(RawError::Corrupt(format!("NEF: compressed data ends at row {y} of {h}")));
         }
     }
-    if table.split != 0 {
+    if table.split != 0 && rows == h {
         // the fixed-rate section must fill the rest of the strip: whole rows of w words and 0..7 bits of padding.
         // A strip may hold a few more rows than the image (one body stores 4022 rows for 4020); anything else means
         // the file does not follow the rule, so refuse it instead of decoding it wrongly.
@@ -676,6 +687,47 @@ mod tests {
         assert!(e.is_err(), "{e:?}");
     }
 
+    /// Issue #708: the header probe of a compressed NEF decodes only the first [`super::super::nef::MASK_ROWS`]
+    /// rows (the ones the masked-column test reads) instead of the whole Huffman stream, and still finds the masked
+    /// columns the full decode finds. A strip cut off after those rows proves it: the probe succeeds with the same
+    /// description while the full decode reports the truncation.
+    #[test]
+    fn header_probe_stops_after_the_masked_column_rows() {
+        use super::super::nef::MASK_ROWS;
+        let (w, h) = (160usize, MASK_ROWS * 3);
+        let mut img = image(w, h, 16383, 7);
+        // eight optically masked columns at 0, as the D5100 / D7000 write them
+        for row in img.chunks_mut(w) {
+            row[w - 8..].fill(0);
+        }
+        let t = lossless(14);
+        let table = {
+            let mut v = vec![0x46, 0x30];
+            for s in t.seeds.iter().flatten() {
+                v.extend_from_slice(&(*s as u16).to_be_bytes());
+            }
+            v.resize(46, 0);
+            v
+        };
+        let src = encode(&img, w, &LOSSLESS_14, t.seeds);
+        let whole = nef_file(src.clone(), w as u32, h as u32, 14, table.clone(), ByteOrder::Big);
+        let full = crate::decode(&whole).unwrap();
+        assert_eq!(full.active_area, crate::Rect::new(0, 0, w - 8, h), "the masked columns are found in a compressed file");
+        assert_eq!(crate::probe_info(&whole).unwrap(), full.info());
+        // the same stream cut off after two thirds of the rows (the first rows' codes are a prefix of the whole)
+        let cut = encode(&img[..w * (h * 2 / 3)], w, &LOSSLESS_14, t.seeds).len();
+        assert!(cut > w * h / 4, "enough bytes to pass the size check");
+        let truncated = nef_file(src[..cut].to_vec(), w as u32, h as u32, 14, table, ByteOrder::Big);
+        assert!(matches!(crate::decode(&truncated), Err(RawError::Corrupt(_))), "the full decode sees the truncation");
+        assert_eq!(crate::probe_info(&truncated).unwrap(), full.info(), "the header probe never reads that far");
+        // decode_rows itself: a prefix of the whole decode, and the whole image's size check still applies
+        let top = decode_rows(&src, w, h, MASK_ROWS, 14, &t).unwrap();
+        assert_eq!(top.len(), w * MASK_ROWS);
+        assert_eq!(top[..], decode(&src, w, h, 14, &t).unwrap()[..w * MASK_ROWS]);
+        assert_eq!(decode_rows(&src, w, h, h + 5, 14, &t).unwrap().len(), w * h, "rows are clamped to the height");
+        assert!(decode_rows(&src[..w * h / 4 - 1], w, h, 1, 14, &t).is_err(), "a strip too short for the image is corrupt in any mode");
+    }
+
     #[test]
     fn truncated_and_corrupted_streams_error_never_panic() {
         let (w, h) = (64, 16);

docs/gpu-pipeline.md

@@ -266,7 +266,9 @@ A view's stages keep an uploaded source only up to 96 MB (a Preview-level source
   `/usr/bin/time -l lightcraft-cli snapshot <files> --script steps.jsonl -o out.png` → "maximum
   resident set size" and "peak memory footprint". Run it several times: the high-water mark is
   noisy (allocator caching, scheduling). On Apple silicon GPU buffers count in the footprint.
-- Imports read raw headers only (`lightcraft_raw::probe_info`): no pixel data is decompressed.
+- Imports read raw headers only (`lightcraft_raw::probe_info`): no pixel data is decompressed,
+  except the first rows a few vendor formats need for facts their headers lack (Nikon NEF: the
+  first 128 rows for the masked trailing columns, issue #708).
 - One budget (`memory::budget`, default min(25 % of RAM, 1.5 GiB); `LIGHTCRAFT_MEMORY_MB` or
   `app.memoryBudget {mb}`): half for the engine caches (decoded thumbnail / preview / full
   sources and rendered previews, evicted least recently used *across* them; the photo on screen