From ff8d0277665304957bc2ef5d44fb07c60232bfd5 Mon Sep 17 00:00:00 2001 From: radialmonster Date: Thu, 8 Oct 2026 14:02:01 -0400 Subject: [PATCH] LR-IMP-FORMATS: NEF lossy type 2 curve of version 0x44 0x40 spans a quarter of the sample range Maker note 0x0096 of the D780, D6, Z 5, Z 6, Z 7 and Z 7_2 starts 0x44 0x40 and holds 257 points, but its first interval rises by 2^bits / 1024 where the 0x44 0x20 tables rise by 2^bits / 256: the curve is the identity in the dark part and covers 2^bits / 4 codes, not 2^bits. The decoder spaced the points 2^bits / 256 codes apart for both, so these files decoded to a fraction of their values, far below the black level. parse_table now takes the span from the version (0x20: all codes, 0x40: a quarter). Other versions decode to identical samples. A known-answer test built from hand-coded table bytes and bit strings fails without the change. Bumps RENDER_CACHE_VERSION to 12. --- crates/engine/src/media.rs | 2 +- crates/raw/src/vendor/nefc.rs | 68 +++++++++++++++++++++++++++++++++-- docs/parity.md | 2 +- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 76a9bf1a..7aafbf34 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 11; +pub const RENDER_CACHE_VERSION: u64 = 12; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/raw/src/vendor/nefc.rs b/crates/raw/src/vendor/nefc.rs index c0260d95..8ac9e73c 100644 --- a/crates/raw/src/vendor/nefc.rs +++ b/crates/raw/src/vendor/nefc.rs @@ -30,7 +30,13 @@ //! first two pixels of the previous row of the same parity, starting at the seeds of maker note `0x0096`. //! - Lossless (`0x0096` version `0x46`): the decoded values are the samples. Lossy (version `0x44`): they index a //! curve. Type 1 (`0x44 0x10`) stores the full curve; type 2 (`0x44 0x20`, `0x44 0x40`) stores 257 points, -//! one every `2^bits / 256` codes, linearly interpolated. +//! one every `2^bits / 256` codes (`0x20`) or `2^bits / 1024` codes (`0x40`), linearly interpolated. The span +//! was established from the files: in every `0x20` file the first interval of the 257 points rises by +//! `2^bits / 256` (16 at 12 bit, 64 at 14 bit) and in every `0x40` file by `2^bits / 1024` (4 and 16), i.e. +//! the curve is the identity in the dark part, one code per sample. Read with a point every `2^bits / 256` +//! codes instead, the 12 `0x40` files that are not split decoded to values that never exceeded 255 (12 bit) or +//! 1023 (14 bit) and whose darkest 0.01 % sat at a quarter of the black level of maker note `0x003d` (63 against +//! 252 in 12-bit units); with the first-interval step they sit at that black level. //! - Values in the maker note are in the maker note's byte order (newer bodies write little-endian notes). //! //! Not supported (returned as [`RawError::Unsupported`], the embedded preview is used instead): @@ -127,8 +133,10 @@ pub(crate) fn parse_table(t: &[u8], order: ByteOrder, bits: u32) -> Result range || !range.is_multiple_of(n - 1) { return Err(RawError::Unsupported(format!("NEF: {n}-point lossy curve for {bits}-bit data"))); } @@ -355,6 +363,60 @@ mod tests { } } + /// Maker note `0x0096` as bytes, written out by hand: version `0x44 `, four seeds, 257 points (little-endian + /// as newer bodies write it) of a curve whose first 256 points rise by `step` and whose last is the white + /// level, and a zero split value at offset 562. + fn hand_table(v1: u8, seed: u16, step: u16, white: u16) -> Vec { + let mut t = vec![0x44, v1]; + for _ in 0..4 { + t.extend_from_slice(&seed.to_le_bytes()); + } + t.extend_from_slice(&257u16.to_le_bytes()); + for k in 0..256u16 { + t.extend_from_slice(&(k * step).to_le_bytes()); + } + t.extend_from_slice(&white.to_le_bytes()); + t.resize(624, 0); + t + } + + /// Known answers for the `0x44 0x40` curve, from bytes and bit strings written by hand (nothing here uses the + /// decoder's span rule or the test encoder): the first interval of the points rises by `2^bits / 1024`, so + /// the curve is the identity for one code per sample and spans `2^bits / 4` codes. The `0x20` curve of the + /// same sizes keeps its span of all `2^bits` codes. + #[test] + fn lossy_type2_version_0x40_spans_a_quarter_of_the_range() { + // 12 bit: points 0, 4, 8 … 1020, 4095 + let t = parse_table(&hand_table(0x40, 252, 4, 4095), ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!(curve.len(), 1025); + assert_eq!(curve[0..6], [0, 1, 2, 3, 4, 5]); + assert_eq!((curve[252], curve[255], curve[1020]), (252, 255, 1020)); + assert_eq!((curve[1021], curve[1022], curve[1023], curve[1024]), (1788, 2557, 3326, 4095)); + // the pixels p0 = 252 + 4, p1 = 252, p2 = p0 - 1, p3 = p1, coded as 011 100 | 11110 | 1110 0 | 11110 + // (LOSSY_12: category 3 = 011, 0 = 11110, 1 = 1110; extra bits 100 = +4, 0 = -1) + let out = decode(&[0x73, 0xdc, 0xf0], 4, 1, 12, &t).unwrap(); + assert_eq!(out, [256, 252, 255, 252]); + // 14 bit: points 0, 16, 32 … 4080, 16383 + let t = parse_table(&hand_table(0x40, 1008, 16, 16383), ByteOrder::Little, 14).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!(curve.len(), 4097); + assert_eq!(curve[0..6], [0, 1, 2, 3, 4, 5]); + assert_eq!((curve[1008], curve[4080], curve[4096]), (1008, 4080, 16383)); + // the same pixels coded as 1100 100 | 111110 | 11110 0 | 111110 + // (LOSSY_14: category 3 = 1100, 0 = 111110, 1 = 11110) + let out = decode(&[0xc9, 0xf7, 0x9f, 0x00], 4, 1, 14, &t).unwrap(); + assert_eq!(out, [1012, 1008, 1011, 1008]); + // version 0x20 with the same points is a curve over all 2^bits codes: the points are 16 codes apart + let t = parse_table(&hand_table(0x20, 252, 16, 4095), ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!((curve.len(), curve[16], curve[17], curve[4095]), (4097, 16, 17, 4094)); + // 200 points can't be spaced evenly over the 1024 codes of a 12-bit 0x40 table + let mut odd = hand_table(0x40, 252, 4, 4095); + odd[10..12].copy_from_slice(&200u16.to_le_bytes()); + assert!(parse_table(&odd, ByteOrder::Little, 12).is_err()); + } + #[test] fn lossy_type1_full_curve() { let mut t = vec![0x44, 0x10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 5]; diff --git a/docs/parity.md b/docs/parity.md index d5a31ea9..71a2ad8f 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -124,7 +124,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported; profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported | | LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) | | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars | -| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF, WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF, RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF. 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). CR3 opens from its full-size embedded JPEG with full metadata (Exif, GPS, XMP from the `CMT1`/`CMT2`/`CMT4` boxes, `crates/meta/src/cr3.rs`) but its raw data is not decoded yet. Missing: CR3 raw decoding (CRX), compressed RAF/ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode | +| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF, WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF, RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF. 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 opens from its full-size embedded JPEG with full metadata (Exif, GPS, XMP from the `CMT1`/`CMT2`/`CMT4` boxes, `crates/meta/src/cr3.rs`) but its raw data is not decoded yet. Missing: CR3 raw decoding (CRX), compressed RAF/ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode | | LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies). Grow the CC0 corpus per model and test each sample decodes with plausible colour | | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic | | LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving |