Merge pull request #158 from storytold/fix/issue-148-rx100m3-green
Issue #148: Raw: read white balance, black level and crop of pre-2017 Sony ARWs
This commit is contained in:
Brandon Thomas
GitHub
commit
5cee0c08d0
|
|
@ -161,7 +161,8 @@ The milestone estimates in the table above were made before work started and are
|
|||
## Raw format coverage and known gaps
|
||||
|
||||
Decoded (CC0 corpus from raw.pixls.us, `cargo xtask corpus --download`, `crates/raw/tests/corpus.rs`): DNG (uncompressed,
|
||||
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear), CR2, ARW (uncompressed, ARW2, LJ92), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed (Bayer and
|
||||
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear), CR2, ARW (uncompressed, ARW2, LJ92; as-shot white balance and black level of pre-2017 bodies from the enciphered
|
||||
maker-note `Tag2010` and the encrypted `SR2SubIFD`, both recovered by black-box analysis, `crates/raw/src/vendor/arw.rs`), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed (Bayer and
|
||||
X-Trans), RW2 packed 12/14-bit, PEF (uncompressed and Huffman), ORF uncompressed (16-bit and 12-bit packed). Every
|
||||
supported container also yields its embedded JPEG preview (CR3 too), and the engine shows that preview for raw variants
|
||||
it can't decode yet.
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ use lightcraft_raster::{Histogram, Rgb32f, Rgba8};
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
|
||||
pub const RENDER_CACHE_VERSION: u64 = 7;
|
||||
pub const RENDER_CACHE_VERSION: u64 = 8;
|
||||
|
||||
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
|
||||
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
|
||||
|
|
|
|||
|
|
@ -14,6 +14,12 @@
|
|||
//! a 128-bit little-endian block: 11-bit max, 11-bit min, 4-bit index of the max, 4-bit index of the min and 14
|
||||
//! seven-bit deltas above the min, scaled by the smallest shift that fits `max − min` into 7 bits.
|
||||
//!
|
||||
//! White balance: the raw IFD's `0x7313` levels when present (bodies from about 2017 on); otherwise the
|
||||
//! `WB_RGBLevels` of the maker note's enciphered `Tag2010` block (see [`DECIPHER`] and [`TAG2010_WB`]). Without
|
||||
//! either, an ARW opened with unit multipliers, i.e. a strong green cast (issue #148). Black level: the raw IFD's
|
||||
//! `0x7310`, else the level stored in the encrypted `SR2SubIFD` (see [`SR2_BLACK_AT`]; 800 rather than the
|
||||
//! default 512 on 1″-sensor bodies such as the RX100 series), else 512 (14-bit) / 128 (12-bit).
|
||||
//!
|
||||
//! Also: uncompressed 16-bit ARW, and lossless-compressed ARW (Compression 7, ILCE-7M4 and later): LJ92 tiles whose
|
||||
//! frames hold one 2×2 CFA cell per four-component sample ([`read_quad_tiles`]). Other lossless-JPEG layouts go
|
||||
//! through the generic TIFF path.
|
||||
|
|
@ -23,7 +29,7 @@ use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, Ra
|
|||
use lightcraft_geom::Orientation;
|
||||
use lightcraft_tiff::image::{Chunk, ImageInfo, Layout, chunk_bytes};
|
||||
use lightcraft_tiff::tags::{self as t, photometric};
|
||||
use lightcraft_tiff::{Ifd, Tiff};
|
||||
use lightcraft_tiff::{Ifd, Tiff, makernote};
|
||||
use rayon::prelude::*;
|
||||
|
||||
const TONE_CURVE: u16 = 0x7010;
|
||||
|
|
@ -31,6 +37,137 @@ const BLACK_LEVEL: u16 = 0x7310;
|
|||
const WB_RGGB: u16 = 0x7313;
|
||||
const CROP_TOP_LEFT: u16 = 0x74c7;
|
||||
const CROP_SIZE: u16 = 0x74c8;
|
||||
/// Maker-note tags (ExifTool Sony tag names): the enciphered `Tag2010` block and `FullImageSize` (height, width).
|
||||
const MN_TAG2010: u16 = 0x2010;
|
||||
const MN_FULL_IMAGE_SIZE: u16 = 0xb02b;
|
||||
|
||||
/// Inverse of Sony's maker-note byte substitution. ExifTool's Sony tag documentation states that the data of
|
||||
/// tags `0x2010`, `0x9050` and `0x94xx` "is encrypted by a simple substitution cipher" (no decoder source was
|
||||
/// consulted). The substitution, checked black-box on CC0 raw.pixls.us samples: every byte `p < 249` is stored as
|
||||
/// `p³ mod 249` (a bijection on 0..249, as 3 is coprime to φ(249) = 164) and bytes 249–255 are stored unchanged.
|
||||
/// Verified on 15 files from 13 bodies covering seven `Tag2010` layouts: the deciphered `SonyISO` field encodes
|
||||
/// the Exif ISO (`100 · 2^(16 − v/256)`), the green level is 255–256 throughout, and on the ILCE-7M3 the deciphered
|
||||
/// `WB_RGBLevels` equal the plain raw-IFD `0x7313` levels.
|
||||
const DECIPHER: [u8; 256] = {
|
||||
let mut inv = [0u8; 256];
|
||||
let mut p = 0usize;
|
||||
while p < 256 {
|
||||
let c = if p < 249 { p * p % 249 * p % 249 } else { p };
|
||||
inv[c] = p as u8;
|
||||
p += 1;
|
||||
}
|
||||
inv
|
||||
};
|
||||
|
||||
/// `Tag2010` layouts (ExifTool "Sony Tag2010a" … "Tag2010i" tables): the models each is documented for and the
|
||||
/// byte offset of `WB_RGBLevels` (three `u16`, the as-shot white balance as R, G, B gains; Tag2010g and h share
|
||||
/// the offset). Newer bodies (Tag2010i) also write the levels in plain form in the raw IFD (`0x7313`), which takes
|
||||
/// precedence.
|
||||
const TAG2010_WB: &[(&[&str], usize)] = &[
|
||||
(&["NEX-5N"], 4476),
|
||||
(&["SLT-A65", "SLT-A77", "NEX-7", "NEX-VG20E"], 4480),
|
||||
(&["SLT-A37", "SLT-A57", "NEX-F3"], 4444),
|
||||
(&["DSC-HX10V", "DSC-HX20V", "DSC-HX200V", "DSC-TX66", "DSC-TX200V", "DSC-TX300V", "DSC-WX50", "DSC-WX100", "DSC-WX150"], 4568),
|
||||
(
|
||||
&[
|
||||
"SLT-A58",
|
||||
"SLT-A99",
|
||||
"ILCE-3000",
|
||||
"ILCE-3500",
|
||||
"NEX-3N",
|
||||
"NEX-5R",
|
||||
"NEX-5T",
|
||||
"NEX-6",
|
||||
"NEX-VG30E",
|
||||
"NEX-VG900",
|
||||
"DSC-RX100",
|
||||
"DSC-RX1",
|
||||
"DSC-RX1R",
|
||||
"DSC-HX300",
|
||||
"DSC-HX50V",
|
||||
"DSC-TX30",
|
||||
"DSC-WX60",
|
||||
"DSC-WX200",
|
||||
"DSC-WX300",
|
||||
],
|
||||
4532,
|
||||
),
|
||||
(&["DSC-RX100M2", "DSC-QX10", "DSC-QX100"], 4204),
|
||||
(
|
||||
&[
|
||||
"DSC-HX60V",
|
||||
"DSC-HX350",
|
||||
"DSC-HX400V",
|
||||
"DSC-QX30",
|
||||
"DSC-RX10",
|
||||
"DSC-RX100M3",
|
||||
"DSC-WX220",
|
||||
"DSC-WX350",
|
||||
"ILCE-7",
|
||||
"ILCE-7R",
|
||||
"ILCE-7S",
|
||||
"ILCE-7M2",
|
||||
"ILCE-5000",
|
||||
"ILCE-5100",
|
||||
"ILCE-6000",
|
||||
"ILCE-QX1",
|
||||
"ILCA-68",
|
||||
"ILCA-77M2",
|
||||
"DSC-HX80",
|
||||
"DSC-HX90V",
|
||||
"DSC-RX0",
|
||||
"DSC-RX1RM2",
|
||||
"DSC-RX10M2",
|
||||
"DSC-RX10M3",
|
||||
"DSC-RX100M4",
|
||||
"DSC-RX100M5",
|
||||
"DSC-WX500",
|
||||
"ILCE-6300",
|
||||
"ILCE-6500",
|
||||
"ILCE-7RM2",
|
||||
"ILCE-7SM2",
|
||||
"ILCA-99M2",
|
||||
],
|
||||
612,
|
||||
),
|
||||
(
|
||||
&[
|
||||
"ILCE-6100",
|
||||
"ILCE-6400",
|
||||
"ILCE-6600",
|
||||
"ILCE-7C",
|
||||
"ILCE-7M3",
|
||||
"ILCE-7RM3",
|
||||
"ILCE-7RM4",
|
||||
"ILCE-9",
|
||||
"ILCE-9M2",
|
||||
"DSC-RX0M2",
|
||||
"DSC-RX10M4",
|
||||
"DSC-RX100M6",
|
||||
"DSC-RX100M5A",
|
||||
"DSC-RX100M7",
|
||||
"DSC-HX99",
|
||||
],
|
||||
594,
|
||||
),
|
||||
];
|
||||
|
||||
/// As-shot white balance from the maker note's enciphered `Tag2010` block (bodies that do not write `0x7313` in
|
||||
/// the raw IFD: most ARWs before 2017). `model` is the Exif model; Sony appends a regional "V" to some names
|
||||
/// (SLT-A77V), which the documented lists omit.
|
||||
fn tag2010_wb(model: &str, block: &[u8], order: lightcraft_tiff::ByteOrder) -> Option<[f32; 3]> {
|
||||
let model = model.trim();
|
||||
let &(_, offset) = TAG2010_WB.iter().find(|(models, _)| models.iter().any(|m| model == *m || model.strip_suffix('V') == Some(*m)))?;
|
||||
let bytes: Vec<u8> = block.get(offset..offset + 6)?.iter().map(|&b| DECIPHER[b as usize]).collect();
|
||||
let level = |i: usize| order.read_u16(&bytes, 2 * i).map(f32::from);
|
||||
let (r, g, b) = (level(0)?, level(1)?, level(2)?);
|
||||
// levels are fixed-point gains (green 255–256 on every sample seen); reject implausible values from an unknown layout
|
||||
if !(16.0..=16384.0).contains(&g) {
|
||||
return None;
|
||||
}
|
||||
let (r, b) = (r / g, b / g);
|
||||
((0.2..=8.0).contains(&r) && (0.2..=8.0).contains(&b)).then_some([r, 1.0, b])
|
||||
}
|
||||
|
||||
/// The inverse tone curve: 11-bit code → 14-bit sensor value.
|
||||
pub(crate) fn code_curve(thresholds: &[u64]) -> Vec<u16> {
|
||||
|
|
@ -163,6 +300,73 @@ fn read_quad_tiles(bytes: &[u8], info: &ImageInfo) -> Result<Vec<u16>> {
|
|||
Ok(out)
|
||||
}
|
||||
|
||||
/// `SR2Private` tags (ExifTool "Sony SR2Private" table; the IFD is referenced by IFD0's `DNGPrivateData`).
|
||||
const SR2_SUBIFD_OFFSET: u16 = 0x7200;
|
||||
const SR2_SUBIFD_LENGTH: u16 = 0x7201;
|
||||
const SR2_SUBIFD_KEY: u16 = 0x7221;
|
||||
/// Black level inside the encrypted `SR2SubIFD` (bodies that do not write `0x7310` in the raw IFD), recovered by
|
||||
/// black-box known-plaintext analysis of CC0 raw.pixls.us samples; no description of Sony's cipher was used.
|
||||
///
|
||||
/// Every Sony ARW we examined (16 files, 15 bodies, 2012–2021) stores the same key (`11 22 33 44`), so its
|
||||
/// keystream is the same in every file. The DSC-RX100M6 and ILCE-7M3 also write their black levels in plain form
|
||||
/// (`0x7310`: 800 and 512); XOR-ing their `SR2SubIFD`s against each other matched `pack(512×4) ^ pack(800×4)` at
|
||||
/// exactly one position, byte 2510, which gave the eight keystream bytes there. Applied to the other files at that
|
||||
/// position they decode to four equal levels that agree with each sensor's dark-pixel floor: 800 on every 1″-sensor
|
||||
/// body (RX10, RX100 … RX100M5), 512 on the APS-C/full-frame ones (NEX-5N/6, SLT-A77, ILCE-6000/7/7M2/7RM2), in
|
||||
/// both `SR2SubIFD` sizes seen (29252 and 56958 bytes). Newer layouts (ILCE-7M4, 35398 bytes) put other data there
|
||||
/// and are rejected by the plausibility checks; those bodies write `0x7310` anyway.
|
||||
const SR2_BLACK_AT: usize = 2510;
|
||||
const SR2_BLACK_KEYSTREAM: [u8; 8] = [0x43, 0xcb, 0x86, 0xb6, 0x11, 0xd2, 0x1a, 0x73];
|
||||
const SR2_KEY: [u8; 4] = [0x11, 0x22, 0x33, 0x44];
|
||||
|
||||
/// The per-channel black level from the encrypted `SR2SubIFD` (see [`SR2_BLACK_AT`]), in 14-bit units; `None`
|
||||
/// unless the file uses the known key and the four decoded levels are equal and plausible.
|
||||
fn sr2_black(bytes: &[u8], ifd0: &Ifd, order: lightcraft_tiff::ByteOrder) -> Option<f32> {
|
||||
let private = ifd0.bytes(t::DNG_PRIVATE_DATA)?;
|
||||
let at = order.read_u32(private, 0)? as u64;
|
||||
let opts = lightcraft_tiff::ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() };
|
||||
let (sr2, _) = lightcraft_tiff::parse_ifd_at(bytes, at, order, 0, false, &opts).ok()?;
|
||||
if sr2.bytes(SR2_SUBIFD_KEY)? != SR2_KEY {
|
||||
return None;
|
||||
}
|
||||
let (start, len) = (usize::try_from(sr2.u64(SR2_SUBIFD_OFFSET)?).ok()?, usize::try_from(sr2.u64(SR2_SUBIFD_LENGTH)?).ok()?);
|
||||
if len < SR2_BLACK_AT + 8 {
|
||||
return None;
|
||||
}
|
||||
sr2_black_levels(bytes.get(start.checked_add(SR2_BLACK_AT)?..start.checked_add(SR2_BLACK_AT + 8)?)?, order)
|
||||
}
|
||||
|
||||
/// Decipher the eight `SR2SubIFD` bytes at [`SR2_BLACK_AT`] into one black level (the mean of four near-equal,
|
||||
/// plausible per-channel levels).
|
||||
fn sr2_black_levels(cipher: &[u8], order: lightcraft_tiff::ByteOrder) -> Option<f32> {
|
||||
let plain: Vec<u8> = cipher.iter().zip(SR2_BLACK_KEYSTREAM).map(|(c, k)| c ^ k).collect();
|
||||
let levels: Vec<u16> = (0..4).filter_map(|i| order.read_u16(&plain, 2 * i)).collect();
|
||||
let (&lo, &hi) = (levels.iter().min()?, levels.iter().max()?);
|
||||
(levels.len() == 4 && (64..=4096).contains(&lo) && hi - lo <= 64).then(|| levels.iter().map(|&v| f32::from(v)).sum::<f32>() / 4.0)
|
||||
}
|
||||
|
||||
/// The image area for files without Sony's crop tags (`0x74c7/0x74c8`, written since about 2017): the DNG-style
|
||||
/// default crop when the raw IFD has one, else the maker note's `FullImageSize` (the camera JPEG's size) anchored
|
||||
/// at the top-left. Older bodies store a few columns of padding at the right edge of the raw frame (constant
|
||||
/// values, 8–32 columns on the samples we checked) inside a frame 16–48 pixels wider than `FullImageSize`, so
|
||||
/// the anchored crop removes them while keeping the CFA phase.
|
||||
fn default_crop(raw: &Ifd, mn: Option<&makernote::MakerNote>, w: usize, h: usize) -> Rect {
|
||||
let full = Rect::new(0, 0, w, h);
|
||||
if let (Some([x, y]), Some([cw, ch])) = (raw.u64s(t::DEFAULT_CROP_ORIGIN).as_deref(), raw.u64s(t::DEFAULT_CROP_SIZE).as_deref())
|
||||
&& *cw > 0
|
||||
&& *ch > 0
|
||||
{
|
||||
return Rect::new(*x as usize, *y as usize, *cw as usize, *ch as usize).clipped(w, h);
|
||||
}
|
||||
match mn.and_then(|m| m.ifd.u64s(MN_FULL_IMAGE_SIZE)).as_deref() {
|
||||
// only a plausible trim: never more than 64 pixels per side, never an enlargement
|
||||
Some([fh, fw]) if *fw as usize <= w && *fh as usize <= h && *fw as usize + 64 >= w && *fh as usize + 64 >= h => {
|
||||
Rect::new(0, 0, *fw as usize, *fh as usize)
|
||||
}
|
||||
_ => full,
|
||||
}
|
||||
}
|
||||
|
||||
fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
|
||||
tiff.all_ifds()
|
||||
.into_iter()
|
||||
|
|
@ -215,26 +419,43 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
|||
_ => (read_image_in(mode, bytes, &info, tiff.order, Packing::Msb)?, bits),
|
||||
};
|
||||
let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float ARW".into())) };
|
||||
// "12-bit uncompressed" files (e.g. ILCE-7RM2) say BitsPerSample 12 but store 16-bit words on the 14-bit scale
|
||||
// of the other modes (black 512, peaks near 16383): black and white follow the data, `bits` keeps the tag
|
||||
let scale_bits = if info.compression == 1 && out_bits < 14 && strip_len >= (w * h * 2) as u64 && samples.iter().any(|&v| v >> (out_bits + 1) != 0)
|
||||
{
|
||||
14
|
||||
} else {
|
||||
out_bits
|
||||
};
|
||||
|
||||
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
|
||||
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() },
|
||||
_ => Cfa::bayer_static("RGGB"),
|
||||
};
|
||||
let default_black = if out_bits >= 14 { 512.0 } else { 128.0 };
|
||||
let default_black = if scale_bits >= 14 { 512.0 } else { 128.0 };
|
||||
let black = match raw.f64s(BLACK_LEVEL).as_deref() {
|
||||
Some([a, b, c, d]) => {
|
||||
BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![*a as f32, *b as f32, *c as f32, *d as f32], ..Default::default() }
|
||||
}
|
||||
_ => BlackLevel::uniform(default_black),
|
||||
_ => BlackLevel::uniform(sr2_black(bytes, ifd0, tiff.order).filter(|_| scale_bits >= 14).unwrap_or(default_black)),
|
||||
};
|
||||
let white = raw.f64(t::WHITE_LEVEL).map(|v| v as f32).filter(|v| *v > 0.0).unwrap_or_else(|| super::white_from_data(samples, out_bits));
|
||||
let wb = raw.f64s(WB_RGGB).filter(|v| v.len() == 4 && v[1] > 0.0 && v[0] > 0.0 && v[3] > 0.0).map(|v| {
|
||||
let g = (v[1] + v[2]) / 2.0;
|
||||
[(v[0] / g) as f32, 1.0, (v[3] / g) as f32]
|
||||
});
|
||||
let white = raw.f64(t::WHITE_LEVEL).map(|v| v as f32).filter(|v| *v > 0.0).unwrap_or_else(|| super::white_from_data(samples, scale_bits));
|
||||
let model = ifd0.string(t::MODEL).unwrap_or_default();
|
||||
let mn = tiff
|
||||
.exif()
|
||||
.and_then(|e| e.get(t::MAKER_NOTE))
|
||||
.and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &ifd0.string(t::MAKE).unwrap_or_default()));
|
||||
let wb = raw
|
||||
.f64s(WB_RGGB)
|
||||
.filter(|v| v.len() == 4 && v[1] > 0.0 && v[0] > 0.0 && v[3] > 0.0)
|
||||
.map(|v| {
|
||||
let g = (v[1] + v[2]) / 2.0;
|
||||
[(v[0] / g) as f32, 1.0, (v[3] / g) as f32]
|
||||
})
|
||||
.or_else(|| mn.as_ref().and_then(|m| tag2010_wb(&model, m.ifd.bytes(MN_TAG2010)?, m.order)));
|
||||
let crop = match (raw.u64s(CROP_TOP_LEFT).as_deref(), raw.u64s(CROP_SIZE).as_deref()) {
|
||||
(Some([x, y]), Some([cw, ch])) if *cw > 0 && *ch > 0 => Rect::new(*x as usize, *y as usize, *cw as usize, *ch as usize).clipped(w, h),
|
||||
_ => Rect::new(0, 0, w, h),
|
||||
_ => default_crop(raw, mn.as_ref(), w, h),
|
||||
};
|
||||
let mut metadata = lightcraft_meta::from_tiff(&tiff);
|
||||
metadata.width = Some(crop.width as u32);
|
||||
|
|
@ -329,6 +550,58 @@ mod tests {
|
|||
let _ = code_curve(&[60000, 1, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decipher_inverts_the_cube_substitution() {
|
||||
let mut seen = [false; 256];
|
||||
for p in 0..=255u8 {
|
||||
let c = if p < 249 { (p as u32).pow(3) % 249 } else { p as u32 } as u8;
|
||||
assert_eq!(DECIPHER[c as usize], p);
|
||||
seen[DECIPHER[p as usize] as usize] = true;
|
||||
}
|
||||
assert!(seen.iter().all(|&s| s), "not a bijection");
|
||||
assert_eq!((DECIPHER[8], DECIPHER[27], DECIPHER[94]), (2, 3, 7)); // 7³ = 343 ≡ 94
|
||||
}
|
||||
|
||||
fn encipher(plain: &[u8]) -> Vec<u8> {
|
||||
plain.iter().map(|&p| if p < 249 { ((p as u32).pow(3) % 249) as u8 } else { p }).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tag2010_white_balance_by_model() {
|
||||
use lightcraft_tiff::ByteOrder::Little;
|
||||
let mut plain = vec![0u8; 700];
|
||||
for (i, v) in [669u16, 256, 441].iter().enumerate() {
|
||||
plain[612 + 2 * i..614 + 2 * i].copy_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
let block = encipher(&plain);
|
||||
let wb = tag2010_wb("DSC-RX100M3", &block, Little).unwrap();
|
||||
assert!((wb[0] - 669.0 / 256.0).abs() < 1e-6 && wb[1] == 1.0 && (wb[2] - 441.0 / 256.0).abs() < 1e-6, "{wb:?}");
|
||||
// same layout (Tag2010h), regional "V" suffix, other layouts, unknown models, truncated or implausible data
|
||||
assert!(tag2010_wb("ILCE-7RM2", &block, Little).is_some());
|
||||
assert!(tag2010_wb("DSC-RX100M3V", &block, Little).is_some());
|
||||
assert!(tag2010_wb("SLT-A77V", &block, Little).is_none()); // Tag2010b: offset 4480 is past the block
|
||||
assert!(tag2010_wb("DSC-RX100M3X", &block, Little).is_none());
|
||||
assert!(tag2010_wb("ILCE-1", &block, Little).is_none());
|
||||
assert!(tag2010_wb("DSC-RX100M3", &block[..615], Little).is_none());
|
||||
assert!(tag2010_wb("DSC-RX100M3", &encipher(&[0u8; 700]), Little).is_none());
|
||||
let mut odd = plain.clone();
|
||||
odd[612..614].copy_from_slice(&9000u16.to_le_bytes()); // R/G ≈ 35
|
||||
assert!(tag2010_wb("DSC-RX100M3", &encipher(&odd), Little).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sr2_black_level_decodes_and_rejects_garbage() {
|
||||
use lightcraft_tiff::ByteOrder::Little;
|
||||
let cipher =
|
||||
|levels: [u16; 4]| -> Vec<u8> { levels.iter().flat_map(|v| v.to_le_bytes()).zip(SR2_BLACK_KEYSTREAM).map(|(p, k)| p ^ k).collect() };
|
||||
assert_eq!(sr2_black_levels(&cipher([800; 4]), Little), Some(800.0));
|
||||
assert_eq!(sr2_black_levels(&cipher([512, 512, 514, 512]), Little), Some(512.5));
|
||||
assert_eq!(sr2_black_levels(&cipher([65320, 64928, 65260, 488]), Little), None); // ILCE-7M4's layout
|
||||
assert_eq!(sr2_black_levels(&cipher([0; 4]), Little), None);
|
||||
assert_eq!(sr2_black_levels(&cipher([800; 4])[..5], Little), None);
|
||||
assert_eq!(sr2_black_levels(&[], Little), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quad_tiles_place_cfa_cells() {
|
||||
// 14×12 mosaic in 8×8 tiles (the right and bottom tiles overhang), each tile a 4×4 frame of 2×2 cells,
|
||||
|
|
|
|||
|
|
@ -230,3 +230,33 @@ fn corpus_adobe_dngs_carry_profile_looks() {
|
|||
}
|
||||
eprintln!("{seen} Adobe-converted DNGs checked");
|
||||
}
|
||||
|
||||
/// Issue #148: Sony ARWs from before ~2017 carry no plain white-balance, black-level or crop tags in the raw IFD.
|
||||
/// White balance comes from the maker note's enciphered `Tag2010`, the black level from the encrypted `SR2SubIFD`
|
||||
/// and the crop from `FullImageSize`; without them the RX100 III opened bright green. Expected values: the black
|
||||
/// levels agree with each sensor's dark-pixel floor, the gains with the neutral sky of the camera JPEG.
|
||||
#[test]
|
||||
fn corpus_sony_pre2017_colour_metadata() {
|
||||
let dir = corpus_root().join("raw");
|
||||
// (file, black, approximate R and B gains, crop width × height)
|
||||
let cases = [
|
||||
("arw-sony-rx100m3.arw", 800.0, [2.61, 1.72], (5472, 3648)),
|
||||
("arw-sony-rx100.arw", 800.0, [2.23, 2.00], (5472, 3648)),
|
||||
("arw-sony-a7rm2-12bit-uncompressed.arw", 512.0, [2.58, 1.46], (7952, 5304)),
|
||||
];
|
||||
let mut seen = 0;
|
||||
for (name, black, [r, b], (cw, ch)) in cases {
|
||||
let Ok(bytes) = std::fs::read(dir.join(name)) else {
|
||||
eprintln!("skip: {name} absent");
|
||||
continue;
|
||||
};
|
||||
let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}"));
|
||||
assert_eq!(img.black.mean(), black, "{name}: black level");
|
||||
let wb = img.wb_multipliers.unwrap_or_else(|| panic!("{name}: no as-shot white balance"));
|
||||
assert!((wb[0] - r).abs() < 0.01 && wb[1] == 1.0 && (wb[2] - b).abs() < 0.01, "{name}: white balance {wb:?}");
|
||||
assert_eq!((img.crop.width, img.crop.height), (cw, ch), "{name}: crop");
|
||||
assert!(img.white_at(0) > 16000.0, "{name}: white {} (14-bit scale)", img.white_at(0));
|
||||
seen += 1;
|
||||
}
|
||||
eprintln!("pre-2017 Sony ARW colour metadata checked on {seen} files");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Synthetic tests cover colour with a nonlinear camera tone, preserved scene-linea
|
|||
|
||||
Five local Sony ILCE-6700 ARWs were accepted. Held-out per-channel linear-display RMS values were 0.04244, 0.03349, 0.04552, 0.02871 and 0.03100. All five unedited supplied JPEG exports stayed pixel-identical to the installed pre-change renderer at equal size/quality. Four supplied JPEGs use Display P3 and two have a narrower crop than the RAW, so they are visual references rather than aligned pixel ground truth.
|
||||
|
||||
Public CC0 samples (raw.pixls.us; 10 files, ILCE-6000, -6400 ×2, -6700, -7M3, -7M4 ×2, -7RM4, -7SM3, DSC-RX100M3), rendered at 1200 px and compared with each file's embedded JPEG (mean CIE76 ΔE at 200 px): 7 fits accepted, ΔE 17–26 → 3–10 with matching mean lightness. Rejected and unchanged: the RX100M3 (held-out RMS 0.103; its fallback render already has a strong green cast), an ILCE-7RM4 dusk scene whose fit was worse than the fallback (ΔE 7.6), and a reduced-size lossless ARW (no fit attempted). Saturated hues can still differ (an orange-red hull renders yellow).
|
||||
Public CC0 samples (raw.pixls.us; 10 files, ILCE-6000, -6400 ×2, -6700, -7M3, -7M4 ×2, -7RM4, -7SM3, DSC-RX100M3), rendered at 1200 px and compared with each file's embedded JPEG (mean CIE76 ΔE at 200 px): 7 fits accepted, ΔE 17–26 → 3–10 with matching mean lightness. (The RX100M3 was rejected at the time because the decoder read neither its white balance nor its black level and its render was green; since issue #148 its fit is accepted, held-out RMS 0.153 → 0.087.) Rejected and unchanged: an ILCE-7RM4 dusk scene whose fit was worse than the fallback (ΔE 7.6), and a reduced-size lossless ARW (no fit attempted). Saturated hues can still differ (an orange-red hull renders yellow).
|
||||
|
||||
On one sample, Exposure +1 EV, Saturation +100, Contrast +50, warm/cool WB, tint, highlights and shadows all changed the output. The same matrix and curve were selected for 400-pixel and full-resolution renders. Headless UI checks wait until the loupe reports `source: render` and no jobs remain; the camera JPEG stand-in does not count as verification.
|
||||
|
||||
|
|
|
|||
|
|
@ -122,8 +122,8 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
|
|||
| LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported; profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported |
|
||||
| LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) |
|
||||
| LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars |
|
||||
| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF, WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW, NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF, RW2, PEF, ORF. Missing: CR3, compressed RAF/ORF, RW2 v4, NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode |
|
||||
| LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~40 CC0 corpus files (plus ~60 NEFs checked by hand) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies). Grow the CC0 corpus per model and test each sample decodes with plausible colour |
|
||||
| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF, WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF, RW2, PEF, ORF. Missing: CR3, compressed RAF/ORF, RW2 v4, NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode |
|
||||
| LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~40 CC0 corpus files (plus ~60 NEFs checked by hand) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies). Grow the CC0 corpus per model and test each sample decodes with plausible colour |
|
||||
| LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic |
|
||||
| LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving |
|
||||
| LR-IMP-DNG-CONVERT | Convert to DNG on import [Classic] | P2 | ✅ | `cmd:library.import` (`dng`), `crates/ui-egui/src/import.rs` | copy imports: Raw files ▸ Copy as DNG (lossless; the card is untouched); the raw copy is removed only once its DNG is verified and on disk (issue #106) |
|
||||
|
|
|
|||
|
|
@ -133,3 +133,8 @@
|
|||
now develop from the raw data instead of the camera's embedded JPEG, so a B&W or other picture style set in the
|
||||
camera no longer gets baked in. Photos already imported as "preview only" switch over on Reload. (Files that
|
||||
Nikon splits into two differently compressed halves still use the preview for now.)
|
||||
- Sony ARWs from before about 2017 (RX100, RX100 II–V, RX10, NEX, SLT, ILCE-6000, A7 / A7 II / A7R II and their
|
||||
siblings) no longer open bright green (issue #148): their as-shot white balance and black level are read from the
|
||||
file (Sony stores them only in scrambled maker-note data on these bodies), the few columns of padding at the right
|
||||
edge are cropped away, and "12-bit uncompressed" files are no longer clipped. The RX100 series renders much closer
|
||||
to the camera's own JPEG.
|
||||
|
|
|
|||
|
|
@ -234,6 +234,10 @@ const RAW_SAMPLES: &[(&str, &str)] = &[
|
|||
("arw-sony-a7m4-lossless-l.arw", "https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06674_FullFrame-LossLess-Compressed-Large.ARW"),
|
||||
("arw-sony-a7m4-lossless-m.arw", "https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06675_FullFrame-LossLess-Compressed-Medium.ARW"),
|
||||
("arw-sony-a7m4-lossless-s.arw", "https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06676_FullFrame-LossLess-Compressed-Small.ARW"),
|
||||
// pre-2017 bodies: white balance only in the enciphered maker note, black level only in the SR2SubIFD (#148)
|
||||
("arw-sony-rx100m3.arw", "https://raw.pixls.us/data/Sony/DSC-RX100M3/DSC00734.ARW"),
|
||||
("arw-sony-rx100.arw", "https://raw.pixls.us/data/Sony/DSC-RX100/DSC00838.ARW"),
|
||||
("arw-sony-a7rm2-12bit-uncompressed.arw", "https://raw.pixls.us/data/Sony/ILCE-7RM2/12-bit-uncompressed.ARW"),
|
||||
// CR2 colour-filter layouts differ by model (issue #85): CR2CFAPattern 3 (GBRG) and 1 (RGGB) samples
|
||||
("cr2-canon-40d.cr2", "https://raw.pixls.us/data/Canon/EOS%2040D/_MG_0153.CR2"),
|
||||
("cr2-canon-550d.cr2", "https://raw.pixls.us/data/Canon/EOS%20550D/IMG_4047.CR2"),
|
||||
|
|
|
|||
Loading…
Reference in New Issue