ORF: black level from the main maker note for bodies without ImageProcessing; gate and bound the 0x1017/0x1018 gains (part of #1273)
This commit is contained in:
Marco Arnulfo
parent
73a10f5b6d
commit
5eee65edac
|
|
@ -19,8 +19,7 @@
|
|||
//! six CC0 files (C5060WZ, C7070WZ, SP-510UZ, SP-550UZ, SP-565UZ, SP-570UZ). Their Exif `CFAPattern` states the
|
||||
//! layout (it matches the embedded thumbnails), the maker note's entries `0x1017`/`0x1018` hold the red and
|
||||
//! blue gain (first value, 256 = 1.0; checked against neutral areas of the thumbnail), and nothing states a
|
||||
//! black level or an active area. The E-10 (16-bit words) has no ImageProcessing sub-directory either and keeps
|
||||
//! its gains in the same two entries.
|
||||
//! black level or an active area.
|
||||
//! - 12-bit blocks (E-300, E-500, E-330; the E-M5 Mark II and PEN-F high-resolution files): one strip of 12.8 bits
|
||||
//! per pixel, `width * 8 / 5` bytes per row, made of 16-byte blocks. A block is 15 sample bytes followed by one
|
||||
//! pad byte that is always zero; the 15 bytes are a little-endian 120-bit string holding ten 12-bit samples, sample
|
||||
|
|
@ -55,6 +54,8 @@ const WB_RB: u16 = 0x0100;
|
|||
const BLACK: u16 = 0x0600;
|
||||
const MN_WB_RED: u16 = 0x1017;
|
||||
const MN_WB_BLUE: u16 = 0x1018;
|
||||
const MN_BLACK: u16 = 0x1012;
|
||||
const MN_VALID_BITS: u16 = 0x102c;
|
||||
const CROP: [u16; 4] = [0x0612, 0x0613, 0x0614, 0x0615];
|
||||
|
||||
/// The Olympus maker note.
|
||||
|
|
@ -76,6 +77,17 @@ pub(crate) fn sub_ifd(bytes: &[u8], mn: &MakerNote, tag: u16) -> Option<Ifd> {
|
|||
lightcraft_tiff::parse_ifd_at(bytes, at, mn.order, mn.base, false, &opts).ok().map(|(i, _)| i)
|
||||
}
|
||||
|
||||
/// Black level of a body without ImageProcessing: the main note's `0x1012` BlackLevel (the mean of its four values)
|
||||
/// in `0x102c` ValidBits units, scaled to the decoded `bits`. The E-10 states 32 at 10 bits, and its 12-bit samples
|
||||
/// bottom out at 124 to 132; the two-field compacts state 0 at 12 bits.
|
||||
fn maker_note_black(m: &MakerNote, bits: u32) -> Option<f32> {
|
||||
let bits = bits.min(16);
|
||||
let v = m.ifd.f64s(MN_BLACK).filter(|v| v.len() == 4)?;
|
||||
let valid = m.ifd.u64(MN_VALID_BITS).and_then(|b| u32::try_from(b).ok()).filter(|b| (8..=bits).contains(b))?;
|
||||
let level = v.iter().sum::<f64>() / 4.0 * f64::from(1u32 << (bits - valid));
|
||||
(level >= 0.0 && level < f64::from(1u32 << bits) / 4.0).then_some(level as f32)
|
||||
}
|
||||
|
||||
/// Unpack one row of 12-bit samples stored as little-endian 32-bit words read MSB-first.
|
||||
pub(crate) fn unpack_row_le32_msb(src: &[u8], bits: u32, out: &mut [u16]) {
|
||||
let swapped: Vec<u8> = src
|
||||
|
|
@ -196,6 +208,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
|||
let chunks = info.chunks(bytes.len() as u64);
|
||||
let total: u64 = chunks.iter().map(|c| c.len).sum();
|
||||
let stated = cfa_from_exif(&tiff);
|
||||
let mut two_field = false;
|
||||
let (mut data, bits) = if info.compression != 1 {
|
||||
return Err(RawError::Unsupported(format!("ORF compression {}", info.compression)));
|
||||
} else if total >= (n as u64) * 2 {
|
||||
|
|
@ -232,6 +245,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
|||
.then(|| field_rows(&info.offsets.iter().copied().zip(info.byte_counts.iter().copied()).collect::<Vec<_>>(), w, h, bytes.len()))
|
||||
.flatten()
|
||||
{
|
||||
two_field = true;
|
||||
let d = if mode == Mode::Full || stated.is_none() {
|
||||
let (rb, h0) = (w * 3 / 2, h.div_ceil(2));
|
||||
let mut d = vec![0u16; n];
|
||||
|
|
@ -281,7 +295,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
|||
let values = a.pattern.iter().map(|&c| [v[0], (v[1] + v[2]) / 2.0, v[3]][c as usize] as f32).collect();
|
||||
BlackLevel { repeat_rows: 2, repeat_cols: 2, values, ..Default::default() }
|
||||
}
|
||||
_ => BlackLevel::uniform(0.0),
|
||||
_ => BlackLevel::uniform(mn.as_ref().filter(|_| ip.is_none()).and_then(|m| maker_note_black(m, bits)).unwrap_or(0.0)),
|
||||
};
|
||||
let wb = ip
|
||||
.as_ref()
|
||||
|
|
@ -289,9 +303,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
|||
.filter(|v| v.len() >= 2 && v[0] > 0.0 && v[1] > 0.0)
|
||||
.map(|v| [(v[0] / 256.0) as f32, 1.0, (v[1] / 256.0) as f32])
|
||||
.or_else(|| {
|
||||
// two-field bodies and the E-10: the red and blue gains lead the entries of the main maker note
|
||||
let m = mn.as_ref()?;
|
||||
let g = |tag| m.ifd.f64s(tag).and_then(|v| v.first().copied()).filter(|g| *g > 0.0);
|
||||
// the red and blue gains lead the main note's entries 0x1017/0x1018 on the two-field compacts and on
|
||||
// the E-10 (16-bit words, no ImageProcessing; 384 and 406 on its CC0 file 5424)
|
||||
let m = mn.as_ref().filter(|_| two_field || ip.is_none())?;
|
||||
let g = |tag| m.ifd.f64s(tag).and_then(|v| v.first().copied()).filter(|g| *g > 25.6 && *g < 2560.0);
|
||||
Some([(g(MN_WB_RED)? / 256.0) as f32, 1.0, (g(MN_WB_BLUE)? / 256.0) as f32])
|
||||
});
|
||||
let white = white_from_data(samples, bits);
|
||||
|
|
@ -372,36 +387,102 @@ mod tests {
|
|||
b
|
||||
}
|
||||
|
||||
/// An old-style little-endian Olympus maker note (`OLYMP\0`, version 1, then the IFD) holding two-short
|
||||
/// entries, whose values fit inline.
|
||||
fn olymp_note(entries: &[(u16, [u16; 2])]) -> Vec<u8> {
|
||||
let mut b = b"OLYMP\0\x01\0".to_vec();
|
||||
/// An old-style little-endian Olympus maker note (`OLYMP\0`, version 1, then the IFD) of SHORT (type 3) or
|
||||
/// UNDEFINED (type 7) entries. Values longer than four bytes follow the IFD; their offsets count from the TIFF
|
||||
/// header, so `at` is where the note sits in the file.
|
||||
fn olymp_note(at: usize, entries: &[(u16, u16, Vec<u8>)]) -> Vec<u8> {
|
||||
let ifd_end = 8 + 2 + 12 * entries.len() + 4;
|
||||
let (mut b, mut extra) = (b"OLYMP\0\x01\0".to_vec(), Vec::new());
|
||||
b.extend_from_slice(&(entries.len() as u16).to_le_bytes());
|
||||
for (tag, [v0, v1]) in entries {
|
||||
for (tag, typ, value) in entries {
|
||||
b.extend_from_slice(&tag.to_le_bytes());
|
||||
b.extend_from_slice(&3u16.to_le_bytes());
|
||||
b.extend_from_slice(&2u32.to_le_bytes());
|
||||
b.extend_from_slice(&v0.to_le_bytes());
|
||||
b.extend_from_slice(&v1.to_le_bytes());
|
||||
b.extend_from_slice(&typ.to_le_bytes());
|
||||
b.extend_from_slice(&(if *typ == 3 { value.len() / 2 } else { value.len() } as u32).to_le_bytes());
|
||||
if value.len() <= 4 {
|
||||
let mut v = value.clone();
|
||||
v.resize(4, 0);
|
||||
b.extend_from_slice(&v);
|
||||
} else {
|
||||
b.extend_from_slice(&((at + ifd_end + extra.len()) as u32).to_le_bytes());
|
||||
extra.extend_from_slice(value);
|
||||
}
|
||||
}
|
||||
b.extend_from_slice(&0u32.to_le_bytes());
|
||||
b.extend_from_slice(&extra);
|
||||
b
|
||||
}
|
||||
|
||||
/// SHORT values, little-endian.
|
||||
fn shorts(v: &[u16]) -> Vec<u8> {
|
||||
v.iter().flat_map(|x| x.to_le_bytes()).collect()
|
||||
}
|
||||
|
||||
/// A 16 × 4 ORF of 12-bit samples in 16-bit words with an [`olymp_note`] of `entries`, decoded.
|
||||
fn decode_noted(entries: &[(u16, u16, Vec<u8>)]) -> RawImage {
|
||||
let (w, h) = (16u32, 4u32);
|
||||
let words: Vec<u8> = (0..w * h).flat_map(|i| (((200 + i * 37) % 4096) as u16).to_le_bytes()).collect();
|
||||
let file = |at| orf_with_note(w, h, 16, words.clone(), Some(RGGB), Some(olymp_note(at, entries)));
|
||||
let at = file(0).windows(6).position(|b| b == b"OLYMP\0").unwrap();
|
||||
let bytes = file(at);
|
||||
assert_eq!(bytes.windows(6).position(|b| b == b"OLYMP\0"), Some(at));
|
||||
crate::decode(&bytes).unwrap()
|
||||
}
|
||||
|
||||
/// An ImageProcessing sub-directory stored inline (old-style note) with one unrelated entry and no levels.
|
||||
fn bare_image_processing() -> (u16, u16, Vec<u8>) {
|
||||
let mut ifd = 1u16.to_le_bytes().to_vec();
|
||||
ifd.extend_from_slice(&[0, 0, 3, 0, 1, 0, 0, 0, 1, 0, 0, 0]);
|
||||
ifd.extend_from_slice(&0u32.to_le_bytes());
|
||||
(IMAGE_PROCESSING, 7, ifd)
|
||||
}
|
||||
|
||||
/// Issue #1273: the E-10 (16-bit words, no ImageProcessing sub-directory) keeps its red and blue gains in
|
||||
/// maker-note entries `0x1017`/`0x1018`, as the two-field compacts do.
|
||||
#[test]
|
||||
fn as_shot_white_balance_from_the_main_maker_note() {
|
||||
let (w, h) = (16usize, 4usize);
|
||||
let words: Vec<u8> = (0..w * h).flat_map(|i| (((100 + i * 37) % 4096) as u16).to_le_bytes()).collect();
|
||||
let wb = |entries: &[(u16, [u16; 2])]| {
|
||||
crate::decode(&orf_with_note(w as u32, h as u32, 16, words.clone(), Some(RGGB), Some(olymp_note(entries)))).unwrap().wb_multipliers
|
||||
let wb = |red: u16, blue: Option<u16>| {
|
||||
let mut e = vec![(MN_WB_RED, 3, shorts(&[red, 64]))];
|
||||
e.extend(blue.map(|b| (MN_WB_BLUE, 3, shorts(&[b, 64]))));
|
||||
decode_noted(&e).wb_multipliers
|
||||
};
|
||||
assert_eq!(wb(&[(MN_WB_RED, [384, 64]), (MN_WB_BLUE, [406, 64])]), Some([1.5, 1.0, 406.0 / 256.0]));
|
||||
// both gains are needed, and a zero gain is none
|
||||
assert_eq!(wb(&[(MN_WB_RED, [384, 64])]), None);
|
||||
assert_eq!(wb(&[(MN_WB_RED, [0, 64]), (MN_WB_BLUE, [406, 64])]), None);
|
||||
assert_eq!(wb(&[]), None);
|
||||
assert_eq!(wb(384, Some(406)), Some([1.5, 1.0, 406.0 / 256.0]));
|
||||
// both gains are needed, within 0.1 to 10 (25.6 to 2560 in 256 units)
|
||||
assert_eq!(wb(384, None), None);
|
||||
assert_eq!(wb(0, Some(406)), None);
|
||||
assert_eq!(wb(25, Some(406)), None);
|
||||
assert_eq!(wb(384, Some(2560)), None);
|
||||
assert_eq!(wb(27, Some(2559)), Some([27.0 / 256.0, 1.0, 2559.0 / 256.0]));
|
||||
assert_eq!(decode_noted(&[]).wb_multipliers, None);
|
||||
// a body with ImageProcessing but without its levels doesn't fall back to these entries
|
||||
let e = [bare_image_processing(), (MN_WB_RED, 3, shorts(&[384, 64])), (MN_WB_BLUE, 3, shorts(&[406, 64]))];
|
||||
assert_eq!(decode_noted(&e).wb_multipliers, None);
|
||||
}
|
||||
|
||||
/// Issue #1273: a body without ImageProcessing states its black level in the main note, `0x1012` (four
|
||||
/// values) in `0x102c` ValidBits units: the E-10's 32 at 10 bits is 128 on its 12-bit samples. The two-field
|
||||
/// compacts state 0 at 12 bits.
|
||||
#[test]
|
||||
fn black_level_from_the_main_maker_note() {
|
||||
let black = |level: u16, valid: Option<u16>| {
|
||||
let mut e = vec![(MN_BLACK, 3, shorts(&[level; 4]))];
|
||||
e.extend(valid.map(|v| (MN_VALID_BITS, 3, shorts(&[v, 0]))));
|
||||
let r = decode_noted(&e);
|
||||
assert_eq!(r.bits, 12);
|
||||
r.black.values
|
||||
};
|
||||
assert_eq!(black(32, Some(10)), vec![128.0]);
|
||||
assert_eq!(black(0, Some(12)), vec![0.0]);
|
||||
assert_eq!(black(32, Some(12)), vec![32.0]);
|
||||
// the four values are averaged
|
||||
let e = [(MN_BLACK, 3, shorts(&[30, 32, 32, 34])), (MN_VALID_BITS, 3, shorts(&[10, 0]))];
|
||||
assert_eq!(decode_noted(&e).black.values, vec![128.0]);
|
||||
// no ValidBits, ValidBits above the samples' depth, a level of a quarter of the range or more: no black
|
||||
assert_eq!(black(32, None), vec![0.0]);
|
||||
assert_eq!(black(32, Some(14)), vec![0.0]);
|
||||
assert_eq!(black(256, Some(10)), vec![0.0]);
|
||||
// ImageProcessing present (without BlackLevel2): the main note isn't read
|
||||
let e = [bare_image_processing(), (MN_BLACK, 3, shorts(&[32; 4])), (MN_VALID_BITS, 3, shorts(&[10, 0]))];
|
||||
assert_eq!(decode_noted(&e).black.values, vec![0.0]);
|
||||
}
|
||||
|
||||
/// A 12-bit mosaic stored as 16-bit words: `site(cx, cy)` gives the four values of the 2×2 cell at (cx, cy).
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -266,6 +266,7 @@ Beyond decoding:
|
|||
|
||||
| Date | Change | Summary |
|
||||
|---|---|---|
|
||||
| 2026-10-12 | minor | Olympus E-10: as-shot white balance (`0x1017`/`0x1018`) and black level (`0x1012` in `0x102c` ValidBits units) from the main maker note, issue #1273; ΔE76 against its thumbnail 18.5 → 11.5, camera-look fit now accepted; counts unchanged |
|
||||
| 2026-10-10 | minor | Compressed ORF: pointer to the two Olympus patents that describe the method, as a clean-room starting point |
|
||||
| 2026-10-11 | minor | EOS R5 Mark II verified (CRX QP-map first literal and limited-length escape, odd-height Bayer planes; 4 CC0 files pinned, QP maps of 33,665 private files): 235 verified, 942 unverified; use-weighted 85% (42% on verified bodies); decoding 77%, RAW readiness unchanged at ~56% |
|
||||
| 2026-10-10 (later) | major | Model-by-model measurement against Adobe's published list (1,446 models: 234 verified, 943 unverified, 233 preview, 36 unsupported); `docs/raw/` data and re-measure script; use-weighted share 84% (40% on verified bodies); variant matrix; colour and lens sections with scores; ranked next cameras; RAW readiness ~56% from written parts |
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
# What's new in LightCraft
|
||||
|
||||
## 2026-10-12
|
||||
|
||||
### Older raw cameras
|
||||
- Olympus E-10 raws open with the camera's white balance (White Balance ▸ As Shot) and without the grey haze in
|
||||
the shadows: the black level recorded in the file is now subtracted.
|
||||
|
||||
## 2026-10-11
|
||||
|
||||
### JPEG XL export
|
||||
|
|
|
|||
Loading…
Reference in New Issue