Merge PR #1223

# Conflicts:
#	crates/catalog/src/lock.rs
#	docs/parity-checklist.md

apps/lightcraft/src/editors.rs

@@ -270,6 +270,8 @@ mod tests {
         assert_eq!(reg_value("ERROR: The system was unable to find the specified registry key or value."), None);
     }
 
+    /// (Unix: `Path::is_absolute` decides by the platform's rules, and desktop entries run there)
+    #[cfg(unix)]
     #[test]
     fn desktop_entries_give_their_program() {
         let appimage = "[Desktop Entry]\nName=PhotoCraft\nExec=\"/home/simon/Apps/photocraft-1.2-linux-x86_64.AppImage\" %F\n";

contributors/people.toml

@@ -27,3 +27,6 @@ sync_github_name = true
 
 [people.sanyamsingh001]
 display_name = "sanyamsingh001"
+
+[people.mdlmarkham]
+sync_github_name = true

crates/engine/src/media.rs

@@ -1466,7 +1466,7 @@ impl crate::Session {
     /// says and what a smart preview renders with. Only for raws edited in Kelvin.
     fn learn_as_shot_wb(&mut self, id: PhotoId, info: &SourceInfo) {
         let Some(p) = self.catalog.photo(id) else { return };
-        if !(info.raw && !info.relative_wb && p.develops_raw() && !p.relative_wb()) {
+        if !info.raw || info.relative_wb || !p.develops_raw() || p.relative_wb() {
             return;
         }
         let read = (info.as_shot_temp, info.as_shot_tint);

crates/pipeline/src/local.rs

@@ -147,7 +147,7 @@ pub fn legacy_to_kelvin(temp: f64, tint: f64, as_shot: (f64, f64)) -> (f64, f64)
 /// edited on a rendered photo or a raw without matrices never pass for Kelvin when pasted onto one that has them.
 pub fn normalize_wb(d: &mut DevelopSettings, info: &SourceInfo) {
     use lightcraft_develop::{WbMode, WbScale};
-    if !(info.raw && !info.relative_wb) {
+    if !info.raw || info.relative_wb {
         d.wb.scale = WbScale::Legacy;
         return;
     }

crates/raw/src/sparse.rs

@@ -12,8 +12,10 @@
 //! start of the first one, where the coding headers a header probe validates sit), a CR3's image
 //! tracks (keeping the raw track's coding headers and quantization maps, and the timed-metadata
 //! track where Canon puts the as-shot white balance), a RAF's CFA strip (keeping its compression
-//! header) and a Panasonic RW2's raw data. Formats whose header probe reads samples (NEF: masked
-//! columns; ORF: bit depth and CFA phase; PEF: dark borders) and anything else get
+//! header) and a Panasonic RW2's raw data. A Sony ARW6's strip is not a prefix but the scattered
+//! bytes its header probe validates (the tile table, the stream headers and index tables, the
+//! residual stream's first half: see [`arw6_keep`]). Formats whose header probe reads samples (NEF:
+//! masked columns; ORF: bit depth and CFA phase; PEF: dark borders) and anything else get
 //! [`HeaderPlan::Whole`], as does a structure that is incomplete, inconsistent or keeps metadata
 //! inside its sample data: the plan never guesses.
 
@@ -160,7 +162,7 @@ pub fn preview_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {
 fn plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
     let mut w = Want::new(bytes, loaded);
     if !w.want(0, 16) {
-        return w.into_plan().unwrap_or(HeaderPlan::Whole("file too short"));
+        return w.pending_plan().unwrap_or(HeaderPlan::Whole("file too short"));
     }
     if is_cr3(bytes) {
         cr3_plan(bytes, loaded, preview)
@@ -200,9 +202,9 @@ impl<'a> Want<'a> {
         false
     }
 
-    /// `Need` when something is missing.
-    fn into_plan(self) -> Option<HeaderPlan> {
-        (!self.need.is_empty()).then_some(HeaderPlan::Need(self.need))
+    /// `Need` when something is missing (the asks are taken).
+    fn pending_plan(&mut self) -> Option<HeaderPlan> {
+        (!self.need.is_empty()).then(|| HeaderPlan::Need(std::mem::take(&mut self.need)))
     }
 }
 
@@ -223,7 +225,7 @@ fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
         Ok(meta) => meta,
         Err(why) => return HeaderPlan::Whole(why),
     };
-    if let Some(plan) = w.into_plan() {
+    if let Some(plan) = w.pending_plan() {
         return plan;
     }
     // (the Make is usually in the first kilobytes: a NEF, PEF or ORF is known before the rest is read)
@@ -233,20 +235,60 @@ fn tiff_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
     let Ok(tiff) = Tiff::parse(bytes) else { return HeaderPlan::Whole("TIFF structure") };
     let mut holes = Ranges::new();
     for ifd in tiff.all_ifds() {
-        let chunks: Vec<(u64, u64)> = match ifd.image() {
+        let info = ifd.image().ok();
+        // A JPEG-interchange image (compression 6/7: tags 0x0201/0x0202 are `JPEGInterchangeFormat` and
+        // `JPEGInterchangeFormatLength`, not strip tags, which is why `image()` declines; e.g. a Sony
+        // ARW's full-size JPEG): the header probe never reads it and `embedded_preview` wants it whole,
+        // so under [`header_plan`] its bulk is holed like any other image's.
+        let jif = (|| {
+            if preview || !matches!(ifd.u16(t::COMPRESSION), Some(6 | 7)) {
+                return None;
+            }
+            let off = ifd.u64(t::JPEG_INTERCHANGE_FORMAT)?;
+            let jif_len = ifd.u64(t::JPEG_INTERCHANGE_FORMAT_LENGTH)?;
+            (jif_len >= BULK).then_some((off, jif_len))
+        })();
+        let chunks: Vec<(u64, u64)> = match &info {
             // a preview's strips are read when the preview is wanted: only the raw image is skipped
-            Ok(info) if preview && !matches!(info.photometric, t::photometric::CFA | t::photometric::LINEAR_RAW) => continue,
-            Ok(info) => info.chunks(len).iter().map(|c| (c.offset, c.len)).collect(),
+            Some(info) if preview && !matches!(info.photometric, t::photometric::CFA | t::photometric::LINEAR_RAW) => continue,
+            Some(info) => info.chunks(len).iter().map(|c| (c.offset, c.len)).collect(),
             // a raw IFD without the tags a TIFF image needs (an older CR2's) still lists its strips
-            Err(_) => match (ifd.u64s(t::STRIP_OFFSETS), ifd.u64s(t::STRIP_BYTE_COUNTS)) {
+            None => match (ifd.u64s(t::STRIP_OFFSETS), ifd.u64s(t::STRIP_BYTE_COUNTS)) {
                 (Some(offsets), Some(counts)) if offsets.len() == counts.len() => offsets.into_iter().zip(counts).collect(),
-                _ => continue,
+                // nothing to hole but for a JPEG-interchange image (see `jif`)
+                _ => {
+                    if let Some((off, jif_len)) = jif {
+                        holes.insert(off.saturating_add(HEAD), off.saturating_add(jif_len).min(len));
+                    }
+                    continue;
+                }
             },
         };
         let total = chunks.iter().fold(0u64, |n, c| n.saturating_add(c.1));
         if total < BULK {
             continue;
         }
+        // A compression-32766 (ARW6) strip: its header probe reads scattered bytes (the tile table, stream
+        // headers, index tables, the residual's first half), not a prefix — keep exactly those and hole the
+        // rest ([`arw6_keep`]). Multiple strips or a layout it cannot walk fail the probe identically on the
+        // whole file, so they stay here for the generic prefix hole or fall to `Whole`.
+        if let Some(info) = &info
+            && info.compression == crate::vendor::arw6::COMPRESSION
+            && let Some(&(chunk_offset, chunk_size)) = chunks.first()
+        {
+            match arw6_keep(&mut w, bytes, (chunk_offset, chunk_size), info.width as usize, info.height as usize) {
+                None => return w.pending_plan().unwrap_or(HeaderPlan::Whole("ARW6 strip")),
+                Some(Ok(keep)) => {
+                    let mut full = Ranges::new();
+                    full.insert(chunk_offset, chunk_offset.saturating_add(chunk_size).min(len));
+                    for &(a, b) in full.minus(&keep).spans() {
+                        holes.insert(a, b);
+                    }
+                    continue;
+                }
+                Some(Err(_)) => return HeaderPlan::Whole("ARW6 strip"),
+            }
+        }
         for (i, &(offset, size)) in chunks.iter().enumerate() {
             let start = if i == 0 { offset.saturating_add(HEAD) } else { offset };
             holes.insert(start, offset.saturating_add(size).min(len));
@@ -408,7 +450,7 @@ fn cr3_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
         }
         at = end;
     }
-    if let Some(plan) = w.into_plan() {
+    if let Some(plan) = w.pending_plan() {
         return plan;
     }
     let mut w = Want::new(bytes, loaded);
@@ -434,7 +476,7 @@ fn cr3_plan(bytes: &[u8], loaded: &Ranges, preview: bool) -> HeaderPlan {
             }
         }
     }
-    w.into_plan().unwrap_or(HeaderPlan::Ready)
+    w.pending_plan().unwrap_or(HeaderPlan::Ready)
 }
 
 /// A RAF is read whole but for its CFA strip: the header, the embedded JPEG (its metadata), the
@@ -443,7 +485,7 @@ fn raf_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {
     let len = bytes.len() as u64;
     let mut w = Want::new(bytes, loaded);
     if !w.want(0, 160) {
-        return w.into_plan().unwrap_or(HeaderPlan::Whole("RAF header"));
+        return w.pending_plan().unwrap_or(HeaderPlan::Whole("RAF header"));
     }
     let (Some(start), Some(size)) = (be32(bytes, 100), be32(bytes, 104)) else { return HeaderPlan::Whole("RAF header") };
     let Some(end) = start.checked_add(size).filter(|&e| start > 0 && size > 0 && e <= len) else { return HeaderPlan::Whole("RAF raw container") };
@@ -452,7 +494,7 @@ fn raf_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {
         Ok(meta) => meta,
         Err(why) => return HeaderPlan::Whole(why),
     };
-    if let Some(plan) = w.into_plan() {
+    if let Some(plan) = w.pending_plan() {
         return plan;
     }
     let Ok(ifd) = crate::vendor::raf::raw_ifd(container) else { return HeaderPlan::Whole("RAF raw IFD") };
@@ -465,6 +507,105 @@ fn raf_plan(bytes: &[u8], loaded: &Ranges) -> HeaderPlan {
     finish(len, loaded, &holes, &meta, || HeaderPlan::Ready)
 }
 
+/// Ask for several ranges in one go, keeping the loaded ones; false when any is still missing.
+fn load_all(w: &mut Want, keep: &mut Ranges, spans: &[(u64, u64)]) -> bool {
+    let mut all = true;
+    for &(start, end) in spans {
+        let end = end.min(w.len);
+        if w.want(start, end) {
+            keep.insert(start, end);
+        } else {
+            all = false;
+        }
+    }
+    all
+}
+
+/// A compression-32766 (ARW6) strip's byte ranges that [`crate::vendor::arw6::decode`] reads in header mode — *not*
+/// a prefix like the other codecs (`docs/arw6-compression.md`): the tile table, and per tile the picture and group
+/// tables, the 13 stream header words, every stream's index table and the residual stream's first half, whose
+/// lines fix the first-half shift. The later ranges' positions are only known once earlier ones are loaded, so the
+/// walk runs in stages that ask for what they can in one round (`None` while something is still missing; its asks
+/// wait in `w`); `Err` when the layout cannot be walked at all — the probe then stops at the same validation of
+/// loaded bytes on the whole file, so the caller reads the file whole.
+fn arw6_keep(w: &mut Want, bytes: &[u8], strip: (u64, u64), width: usize, height: usize) -> Option<Result<Ranges, &'static str>> {
+    use crate::vendor::arw6;
+    let (offset, size) = strip;
+    let strip_end = offset.saturating_add(size).min(bytes.len() as u64);
+    // the tile table: a count and at most sixteen 24-byte entries
+    if !w.want(offset, offset.saturating_add(8 + 24 * arw6::MAX_TILES as u64).min(strip_end)) {
+        return None;
+    }
+    let (at, end) = match (usize::try_from(offset), usize::try_from(strip_end)) {
+        (Ok(at), Ok(end)) if at <= end => (at, end),
+        _ => return Some(Err("ARW6 strip")),
+    };
+    let Some(strip) = bytes.get(at..end) else { return Some(Err("ARW6 strip")) };
+    let Ok(tiles) = arw6::parse_tile_table(strip, width, height) else { return Some(Err("ARW6 tile table")) };
+    // each tile's bytes run to the next tile's offset (the last: the strip end), as `tile_bytes` does: the
+    // `end_rel(i)` are positions in the strip and `abs` makes one absolute
+    let end_rel = |i: usize| -> usize {
+        match tiles.get(i + 1) {
+            Some(next) => next.offset,
+            None => strip.len(),
+        }
+    };
+    let abs = |rel: usize| offset.saturating_add(rel as u64);
+    // stage A: every tile's picture header (word 0) and group tables (words 3..8)
+    let mut keep = Ranges::new();
+    keep.insert(offset, offset.saturating_add(8 + 24 * tiles.len() as u64).min(strip_end));
+    let tables: Vec<(u64, u64)> = tiles.iter().enumerate().map(|(i, entry)| (abs(entry.offset), abs((entry.offset + 128).min(end_rel(i))))).collect();
+    if !load_all(w, &mut keep, &tables) {
+        return None;
+    }
+    // stage B: the 13 stream header words, at positions the group tables alone imply
+    let mut spans = Vec::with_capacity(tiles.len() * 13);
+    for (i, entry) in tiles.iter().enumerate() {
+        let Some(found) = strip.get(entry.offset..(entry.offset + 128).min(end_rel(i))).and_then(arw6::stream_header_positions) else {
+            return Some(Err("ARW6 tile header"));
+        };
+        let start = abs(entry.offset);
+        for p in found {
+            let at = start.saturating_add(p as u64 * 16);
+            spans.push((at, at.saturating_add(16)));
+        }
+    }
+    if !load_all(w, &mut keep, &spans) {
+        return None;
+    }
+    // stage C: with the header words loaded, the probe's own parse places each stream's index table
+    let mut headers = Vec::with_capacity(tiles.len());
+    let mut spans = Vec::with_capacity(tiles.len() * 13);
+    for (i, entry) in tiles.iter().enumerate() {
+        let Some(tile) = strip.get(entry.offset..end_rel(i)) else { return Some(Err("ARW6 tile header")) };
+        let Ok(header) = arw6::parse_tile_header(tile, entry.width, entry.height) else { return Some(Err("ARW6 tile header")) };
+        let start = abs(entry.offset);
+        for st in &header.streams {
+            let from = start.saturating_add((st.word as u64 + 1) * 16);
+            spans.push((from, from.saturating_add(st.index_words as u64 * 16)));
+        }
+        headers.push(header);
+    }
+    if !load_all(w, &mut keep, &spans) {
+        return None;
+    }
+    // stage D: each residual stream's first half — its first index entry (loaded with the index table) gives
+    // the half's byte length, and [`arw6::first_half_shift`] decodes its lines for the anchor offset
+    let mut spans = Vec::with_capacity(tiles.len());
+    for (i, entry) in tiles.iter().enumerate() {
+        let Some(tile) = strip.get(entry.offset..end_rel(i)) else { return Some(Err("ARW6 tile header")) };
+        let Some(header) = headers.get(i) else { return Some(Err("ARW6 tile header")) };
+        let Some(g4) = header.streams.get(12) else { return Some(Err("ARW6 tile header")) };
+        let Some(a) = arw6::u16_be(tile, (g4.word + 1) * 16) else { return Some(Err("ARW6 residual half")) };
+        let data = abs(entry.offset).saturating_add((g4.word as u64 + 1) * 16).saturating_add(g4.index_words as u64 * 16);
+        spans.push((data, data.saturating_add(a as u64).min(abs(end_rel(i)))));
+    }
+    if !load_all(w, &mut keep, &spans) {
+        return None;
+    }
+    Some(Ok(keep))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -550,4 +691,35 @@ mod tests {
         raf[104..108].copy_from_slice(&1000u32.to_be_bytes());
         assert_eq!(run(&raf, 160, 0).0, HeaderPlan::Whole("RAF raw container"));
     }
+
+    /// An ARW6 strip's probe reads are not a prefix: the residual stream — whose first half fixes the first-half
+    /// shift — starts behind other streams' data (the corpus's `arw6-sony-ilce7rm6-apsc-land-hq.arw` used to be
+    /// described from zeros there and failed with "first-half row count outside 2..=7"). The plan keeps those
+    /// bytes, wherever in the strip they sit.
+    #[test]
+    fn arw6_plan_keeps_the_scattered_probe_reads() {
+        use crate::vendor::arw6_testenc::{Qis, arw6_file, one_tile_quantised};
+        for file in [
+            // one tile, like the ILCE-7RM6's APS-C crop
+            arw6_file(&[(0, 0, 1024, 1024, one_tile_quantised(1024, 1024, 3, Qis::REAL).0)], 1024, 1024),
+            // several tiles, like the full frame (each tile's reads are kept)
+            arw6_file(
+                &[
+                    (0, 0, 1024, 1024, one_tile_quantised(1024, 1024, 0, Qis::REAL).0),
+                    (1024, 0, 1024, 1024, one_tile_quantised(1024, 1024, 0, Qis::REAL).0),
+                ],
+                2048,
+                1024,
+            ),
+        ] {
+            for fill in [0u8, 0xff] {
+                let (plan, buf, loaded) = run(&file, 512 * 1024, fill);
+                assert!(matches!(plan, HeaderPlan::Ready), "{plan:?} (fill {fill})");
+                // the plan asked for bytes past the prefix keep: the probe's reads off a prefix (zeros or 0xFF)
+                // would break the first-half shift, the regression above
+                assert!(loaded.spans().iter().any(|&(a, _)| a >= 512 * 1024), "nothing read past the prefix (fill {fill})");
+                assert_eq!(format!("{:?}", crate::probe_info(&buf)), format!("{:?}", crate::probe_info(&file)), "fill {fill}");
+            }
+        }
+    }
 }

crates/raw/src/vendor/arw6.rs

@@ -20,12 +20,12 @@ fn unsupported(why: &str) -> RawError {
     RawError::Unsupported(format!("ARW6 {why}"))
 }
 
-fn u16_be(b: &[u8], at: usize) -> Option<usize> {
+pub(crate) fn u16_be(b: &[u8], at: usize) -> Option<usize> {
     let s = b.get(at..at.checked_add(2)?)?;
     Some(usize::from(u16::from_be_bytes([*s.first()?, *s.get(1)?])))
 }
 
-fn u24_be(b: &[u8], at: usize) -> Option<usize> {
+pub(crate) fn u24_be(b: &[u8], at: usize) -> Option<usize> {
     let s = b.get(at..at.checked_add(3)?)?;
     Some(s.iter().fold(0usize, |a, &v| (a << 8) | usize::from(v)))
 }
@@ -46,7 +46,10 @@ pub(crate) struct TileEntry {
 }
 
 /// Most tiles a strip may hold (real files have 1 or 4).
-const MAX_TILES: usize = 16;
+pub(crate) const MAX_TILES: usize = 16;
+
+/// TIFF `Compression` of ARW6: Sony's "Compressed RAW 2" (SMPTE RDD 34), written by the ILCE-7RM6 and ILCE-7M5.
+pub(crate) const COMPRESSION: u16 = 32766;
 
 /// Parse the tile table at the start of the strip (file structure, *Container*): `u32` count, `u32` 0, then
 /// 24-byte entries `u64` offset (relative to the strip), `u32` x, y, width, height, all little-endian.
@@ -171,6 +174,26 @@ pub(crate) fn parse_tile_header(tile: &[u8], tw: usize, th: usize) -> Result<Til
     Ok(TileHeader { hs, vs, streams })
 }
 
+/// Word positions of a tile's 13 stream header words, from the group tables alone (bytes 48..128: per group a count
+/// byte then `u24` stream sizes). [`crate::sparse`] loads those words before running [`parse_tile_header`] on a
+/// partly loaded buffer; the two walks must agree.
+pub(crate) fn stream_header_positions(tables: &[u8]) -> Option<[usize; 13]> {
+    let mut out = [0usize; 13];
+    let (mut word, mut i) = (8usize, 0usize);
+    for (group, &n) in GROUP_STREAMS.iter().enumerate() {
+        let base = (3 + group) * 16;
+        if usize::from(*tables.get(base)?) != n {
+            return None;
+        }
+        for comp in 0..n {
+            out[i] = word;
+            i += 1;
+            word = word.checked_add(u24_be(tables, base + 1 + 3 * comp)?)?;
+        }
+    }
+    Some(out)
+}
+
 /// One half-TU of a stream: `len` bytes at byte `start` of the tile, quantiser index per band
 /// (g0/g4: the single nibble repeated).
 #[derive(Clone, Debug, PartialEq, Eq)]

docs/arw6-compression.md

@@ -12,6 +12,7 @@ The codec is Sony's use of SMPTE RDD 34 (LLVC, Low Latency Video Codec). `crates
 
 - Raw SubIFD: Compression 32766, 14 bits per sample, BlackLevel 512 x4 (`0x7310`), WB `0x7313`, crop `0x74c7/0x74c8` (full frame: origin 12,8, size 9984x6656; APS-C: origin 16,10, size 6528x4352), CFA RGGB. The Sony tag-based white level (15360) is not read; the DNG `WhiteLevel` is (16383).
 - The single strip starts with a tile table: `u32` count, `u32` 0, then per tile a `u64` offset (relative to the strip) and `u32` x, y, width, height. Seen: full frame 4 tiles of 5008x3336 (HQ) or 5024 / 4992 wide (Compressed); APS-C (6592x4372) one tile. Bytes up to offset 512 follow the table (480 on APS-C, 408 on full frame; not interpreted, not needed).
+- Header mode (the import probe) reads the tile table and, per tile, the picture and group tables, the 13 stream header words, every stream's index table and the residual stream's first half — a few tens of kilobytes scattered through a 30-63 MB strip, which `lightcraft_raw::sparse::header_plan` keeps (the rest of the strip is skipped; the full-size JPEG the file also carries as a `JPEGInterchangeFormat` image is skipped too). The equivalence over `corpus/raw/**` is asserted by `crates/raw/tests/sparse_corpus.rs` and, without the corpus, `crates/raw/src/sparse.rs`'s synthetic single- and two-tile tests.
 - A tile is a sequence of 16-byte words. Word 0 is RDD 34 `Picture_info` (HS = tile width, VS = tile height / 2, BBD = 16, NC = 3, NW = 3, NP = `1000000000b`; other values are refused as unsupported). Word 1 holds five `u24` group byte-sums, word 2 is zero, words 3-7 describe the groups (a count byte, then `u24` stream sizes in words). Groups: g0 = LL3 (3 streams, one per component), g1 / g2 / g3 = level-3 / 2 / 1 bands (3 streams each), g4 = green residual (1 stream). Streams follow in that order from word 8.
 - A stream is a header word (a `u16` count of index words, a `u24` count of data words, then 11 bytes that are not interpreted), an index table and data, each padded to a word. The index has one entry per transform unit (TU): `A(16) X B(16) X`, where A and B are the byte lengths of the TU's two halves and X are quantiser-index (QI) nibbles (one for g0 and g4, three for g1-g3: HL, LH, HH). Halves per tile = ceil((VS - 2 - s) / 8) + 1 with s the first-half shift (below), TUs = ceil(halves / 2): 105 and 137 for the sample sizes. Halves are numbered n = 2 * TU + h; each is byte-aligned and consecutive halves are contiguous.
 - QI values seen, identical in Compressed and HQ files: level 1 component 0 (1,1,2), components 1-2 (2,2,3); level 2 (0,0,1); level 3 (0,0,0); LL3 0; residual 2. Compressed and HQ differ only in the coefficient data.