From d6a2842ada0b62559a2257f17811d1529e87d73d Mon Sep 17 00:00:00 2001 From: ethanrous Date: Sat, 6 Jun 2026 23:07:21 -0400 Subject: [PATCH 1/5] feat: add Sony ARW2 tone-curve builder (SonyToneCurve 0x7010) Port of dcraw/LibRaw's sony_curve construction: reduces four u16 control points with `>> 2 & 0xfff`, brackets with 0 and 4095, and fills each segment incrementing by `1 << i`. Returns a 0x4000-entry Vec for use as `curve[pixel << 1]` during ARW2 block decoding. Includes unit test validating the A7 IV control points against expected dcraw output. --- agno/src/sony_decoder.rs | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/agno/src/sony_decoder.rs b/agno/src/sony_decoder.rs index caa9ce9..157a7be 100644 --- a/agno/src/sony_decoder.rs +++ b/agno/src/sony_decoder.rs @@ -436,6 +436,39 @@ pub fn decrypt_sr2_data(decryptor: &mut SonyDecryptor, data: &mut [u8], key: u32 // starting at bit offset 30, each 7-bit code -> value = (code << sh) + min // positions imax/imin are set to max/min respectively. // We decode blocks until we fill active_width pixels. Any trailing row bytes are ignored. +/// Build the Sony ARW2 linearization (tone) curve from the `SonyToneCurve` tag (0x7010). +/// +/// Port of dcraw/LibRaw: the four control points are reduced with `>> 2 & 0xfff` and +/// bracketed by 0 and 4095; within segment `i` the curve increments by `1 << i`. During +/// ARW2 decoding the curve is indexed by `pixel << 1`, expanding the 11-bit codes to the +/// ~14-bit linear domain (max ~`0x3ff0`). Returns a 0x4000-entry table; only indices +/// 0..=4094 are ever read (pixel codes are clamped to 0x7ff). +pub fn build_sony_tone_curve(points: [u16; 4]) -> Vec { + let mut curve = vec![0u16; 0x4000]; + + // Missing/zero tag: identity passthrough so ARW2 still decodes (no expansion). + if points == [0, 0, 0, 0] { + for (i, c) in curve.iter_mut().enumerate() { + *c = i.min(0x3fff) as u16; + } + return curve; + } + + let mut sc = [0usize; 6]; + sc[5] = 4095; + for i in 0..4 { + sc[i + 1] = ((points[i] >> 2) & 0xfff) as usize; + } + + for i in 0..5 { + // Segments may be empty if the control points are non-monotonic; that's fine. + for j in (sc[i] + 1)..=sc[i + 1] { + curve[j] = curve[j - 1].saturating_add(1u16 << i); + } + } + curve +} + #[allow(clippy::needless_range_loop)] pub fn sony_arw2_load_raw( reader: &mut R, @@ -608,3 +641,23 @@ pub fn read_concatenated_strips( } Ok(buf) } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn build_sony_tone_curve_matches_dcraw() { + // SonyToneCurve from an A7 IV ARW2: [8000, 10400, 12900, 14100]. + // dcraw/LibRaw: sony_curve[i+1] = (point >> 2) & 0xfff, bracketed by 0 and 4095, + // giving {0, 2000, 2600, 3225, 3525, 4095}; segment i increments by (1 << i). + let curve = build_sony_tone_curve([8000, 10400, 12900, 14100]); + assert_eq!(curve[0], 0); + assert_eq!(curve[1], 1); // segment 0, step 1 + assert_eq!(curve[2000], 2000); // end of segment 0 + assert_eq!(curve[2001], 2002); // segment 1, step 2 + assert_eq!(curve[2600], 3200); // end of segment 1 + assert_eq!(curve[4094], 17204); // max real index: pix 0x7ff -> (0x7ff << 1) = 0xffe + } +} From 60ed9eecd9778542418c19f47ed153647e2321b2 Mon Sep 17 00:00:00 2001 From: ethanrous Date: Sat, 6 Jun 2026 23:14:42 -0400 Subject: [PATCH 2/5] test: cover ARW2 tone-curve identity/non-monotonic; keep arw2 comment attached --- agno/src/sony_decoder.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/agno/src/sony_decoder.rs b/agno/src/sony_decoder.rs index 157a7be..cdfcc86 100644 --- a/agno/src/sony_decoder.rs +++ b/agno/src/sony_decoder.rs @@ -429,13 +429,6 @@ pub fn decrypt_sr2_data(decryptor: &mut SonyDecryptor, data: &mut [u8], key: u32 // }) // } -// Port of LibRaw::sony_arw2_load_raw (block-based: 16 bytes -> 16 pixels) -// For each row, the stream contains 16-byte blocks: -// - First 4 bytes (LE) carry fields: max(11b), min(11b), imax(4b), imin(4b) -// - Remaining 12 bytes carry 14 packed 7-bit codes, MSB-first within the 16-byte span: -// starting at bit offset 30, each 7-bit code -> value = (code << sh) + min -// positions imax/imin are set to max/min respectively. -// We decode blocks until we fill active_width pixels. Any trailing row bytes are ignored. /// Build the Sony ARW2 linearization (tone) curve from the `SonyToneCurve` tag (0x7010). /// /// Port of dcraw/LibRaw: the four control points are reduced with `>> 2 & 0xfff` and @@ -455,6 +448,7 @@ pub fn build_sony_tone_curve(points: [u16; 4]) -> Vec { } let mut sc = [0usize; 6]; + sc[0] = 0; sc[5] = 4095; for i in 0..4 { sc[i + 1] = ((points[i] >> 2) & 0xfff) as usize; @@ -469,6 +463,13 @@ pub fn build_sony_tone_curve(points: [u16; 4]) -> Vec { curve } +// Port of LibRaw::sony_arw2_load_raw (block-based: 16 bytes -> 16 pixels) +// For each row, the stream contains 16-byte blocks: +// - First 4 bytes (LE) carry fields: max(11b), min(11b), imax(4b), imin(4b) +// - Remaining 12 bytes carry 14 packed 7-bit codes, MSB-first within the 16-byte span: +// starting at bit offset 30, each 7-bit code -> value = (code << sh) + min +// positions imax/imin are set to max/min respectively. +// We decode blocks until we fill active_width pixels. Any trailing row bytes are ignored. #[allow(clippy::needless_range_loop)] pub fn sony_arw2_load_raw( reader: &mut R, @@ -660,4 +661,16 @@ mod tests { assert_eq!(curve[2600], 3200); // end of segment 1 assert_eq!(curve[4094], 17204); // max real index: pix 0x7ff -> (0x7ff << 1) = 0xffe } + + #[test] + fn build_sony_tone_curve_identity_and_nonmonotonic() { + // Missing tag (all zero) -> identity passthrough. + let flat = build_sony_tone_curve([0, 0, 0, 0]); + assert_eq!(flat[0], 0); + assert_eq!(flat[1000], 1000); + assert_eq!(flat[4094], 4094); + // Non-monotonic control points must not panic; unset entries stay 0. + let curve = build_sony_tone_curve([10400, 8000, 12900, 14100]); + assert_eq!(curve[0], 0); + } } From 7c4f7cf27f6078df0b5837273b1ffe7938c85781 Mon Sep 17 00:00:00 2001 From: ethanrous Date: Sat, 6 Jun 2026 23:17:43 -0400 Subject: [PATCH 3/5] fix: correct Sony ARW2 column de-interleave and tone-curve expansion ARW2 (Sony cRAW, e.g. A7 IV/ILCE-7M4) decoded with a vertical comb artifact and ~8x too dark because each 16-pixel block was written to consecutive columns and the 11-bit codes were stored without the Sony tone curve. Port LibRaw's sony_arw2_load_raw: write every other column with alternating even/odd phase and expand through curve[pix << 1] built from the SonyToneCurve tag; set white_level to 0x3ff0. --- agno/src/agno_image/load/sony.rs | 13 ++-- agno/src/sony_decoder.rs | 105 +++++++++++++++++++++++-------- 2 files changed, 87 insertions(+), 31 deletions(-) diff --git a/agno/src/agno_image/load/sony.rs b/agno/src/agno_image/load/sony.rs index 19d9a24..21edfc1 100644 --- a/agno/src/agno_image/load/sony.rs +++ b/agno/src/agno_image/load/sony.rs @@ -12,8 +12,8 @@ use crate::{ exif::{ ExifContext, ExifValue, spec::{ - BLACK_LEVEL, COLOR_MATRIX1, COLOR_MATRIX2, COLOR_MATRIX3, SR2_COLOR_MATRIX, - WB_RGGBLEVELS, + BLACK_LEVEL, COLOR_MATRIX1, COLOR_MATRIX2, COLOR_MATRIX3, SONY_TONE_CURVE, + SR2_COLOR_MATRIX, WB_RGGBLEVELS, }, }, sony_decoder::{self, DecodeError, Dimensions}, @@ -53,8 +53,13 @@ pub fn load_sony_raw( // Auto-select decoder based on detection let decoded = match variant { SonyVariant::Arw2Compressed => { - // ARW2: compressed row length equals pixel width; decoder expects row_len == active_width - match sony_decoder::sony_arw2_load_raw(&mut cursor, dims) { + // ARW2 codes are tone-curve compressed; expand via the Sony tone curve (tag 0x7010). + let tone_points: [u16; 4] = match ctx.get_tag_value(SONY_TONE_CURVE) { + Some(ExifValue::Short(v)) if v.len() >= 4 => [v[0], v[1], v[2], v[3]], + _ => [0u16; 4], + }; + let tone_curve = sony_decoder::build_sony_tone_curve(tone_points); + match sony_decoder::sony_arw2_load_raw(&mut cursor, dims, &tone_curve) { Ok(result) => result, Err(e) => return Err(Box::new(e)), } diff --git a/agno/src/sony_decoder.rs b/agno/src/sony_decoder.rs index cdfcc86..180a533 100644 --- a/agno/src/sony_decoder.rs +++ b/agno/src/sony_decoder.rs @@ -463,31 +463,36 @@ pub fn build_sony_tone_curve(points: [u16; 4]) -> Vec { curve } -// Port of LibRaw::sony_arw2_load_raw (block-based: 16 bytes -> 16 pixels) -// For each row, the stream contains 16-byte blocks: -// - First 4 bytes (LE) carry fields: max(11b), min(11b), imax(4b), imin(4b) -// - Remaining 12 bytes carry 14 packed 7-bit codes, MSB-first within the 16-byte span: -// starting at bit offset 30, each 7-bit code -> value = (code << sh) + min -// positions imax/imin are set to max/min respectively. -// We decode blocks until we fill active_width pixels. Any trailing row bytes are ignored. +// Port of LibRaw::sony_arw2_load_raw (block-based: 16 bytes -> 16 pixels). +// Each 16-byte block decodes 16 pixels that are written to every OTHER column (stride 2); +// consecutive blocks alternate between the even and odd column phase of a 32-column span. +// The 11-bit codes are expanded through the Sony tone curve (`curve[pix << 1]`) into the +// ~14-bit linear domain. Skipping the de-interleave produces a vertical comb artifact; +// skipping the curve makes the image ~8x too dark. #[allow(clippy::needless_range_loop)] pub fn sony_arw2_load_raw( reader: &mut R, dims: Dimensions, + tone_curve: &[u16], ) -> Result { - let row_len = dims.output_width; // bytes per compressed row in ARW2 equal to pixel width + let raw_width = dims.raw_width; let mut pixels = vec![0u16; dims.raw_width * dims.raw_height]; - let mut row_buf = vec![0u8; row_len + 1]; + // dcraw allocates raw_width + 1 so the 16-bit reads inside a block can over-read by one byte. + let mut row_buf = vec![0u8; raw_width + 1]; for row in 0..dims.output_height { - // Read one row of compressed bytes - reader.read_exact(&mut row_buf[..row_len])?; + reader.read_exact(&mut row_buf[..raw_width])?; + row_buf[raw_width] = 0; - let mut out_col = 0usize; let mut dp = 0usize; + let mut col: usize = 0; + + while col < raw_width.saturating_sub(30) { + if dp + 16 > raw_width { + break; + } - while out_col < dims.output_width && dp + 16 <= row_len { let header = u32::from_le_bytes([ row_buf[dp], row_buf[dp + 1], @@ -506,41 +511,42 @@ pub fn sony_arw2_load_raw( } let mut pix16 = [0u16; 16]; - let mut bit = 30; - + let mut bit = 30usize; for i in 0..16usize { if i == imax { pix16[i] = max_v as u16; } else if i == imin { pix16[i] = min_v as u16; } else { - let byte_index = dp + ((bit >> 3) as usize); - if byte_index + 1 >= row_buf.len() { - return Err(DecodeError::CorruptData("Sony ARW2: row buffer overread")); - } + let byte_index = dp + (bit >> 3); let two = u16::from_le_bytes([row_buf[byte_index], row_buf[byte_index + 1]]) as i32; let code7 = (two >> (bit & 7)) & 0x7f; - let value = ((code7 << sh) + min_v) as i32; + let mut value = (code7 << sh) + min_v; + if value > 0x7ff { + value = 0x7ff; + } pix16[i] = value as u16; bit += 7; } } - let run = std::cmp::min(16, dims.output_width - out_col); - for i in 0..run { - let dst = row * dims.raw_width + (out_col + i); - pixels[dst] = pix16[i]; + // De-interleaved write with tone-curve expansion (curve indexed by pix << 1). + let mut c = col; + for i in 0..16usize { + if c < dims.output_width { + pixels[row * dims.raw_width + c] = tone_curve[(pix16[i] as usize) << 1]; + } + c += 2; } - - out_col += 16; + col = c - if c & 1 == 1 { 1 } else { 31 }; dp += 16; } } Ok(SonyLoadResult { pixels, - white_level: 0x3fff, + white_level: 0x3ff0, }) } @@ -673,4 +679,49 @@ mod tests { let curve = build_sony_tone_curve([10400, 8000, 12900, 14100]); assert_eq!(curve[0], 0); } + + // Build one 16-byte ARW2 block whose 16 decoded pixels are all `value`. + // header: max=min=value, imax=0, imin=1 -> pix[0]=max, pix[1]=min, and every other + // pixel decodes code=0 (the 12 payload bytes are zero) -> (0 << sh) + min = value. + fn make_uniform_block(value: u16) -> [u8; 16] { + let v = (value & 0x7ff) as u32; + let header = v | (v << 11) | (0u32 << 22) | (1u32 << 26); + let mut block = [0u8; 16]; + block[0..4].copy_from_slice(&header.to_le_bytes()); + block + } + + #[test] + fn arw2_deinterleaves_columns_and_applies_tone_curve() { + // One 32x1 row = block0 (value A) + block1 (value B). Correct ARW2 decoding writes + // block0 to even columns and block1 to odd columns, after expanding through the curve + // (indexed by pix << 1). Use an identity curve so curve[p << 1] == p << 1, which makes + // both the interleave AND the `<< 1` indexing observable in the assertions. + let dims = Dimensions { + raw_width: 32, + raw_height: 1, + output_width: 32, + output_height: 1, + }; + let identity: Vec = (0..0x4000u32).map(|i| i as u16).collect(); + + let a: u16 = 100; + let b: u16 = 50; + let mut row = Vec::with_capacity(32); + row.extend_from_slice(&make_uniform_block(a)); + row.extend_from_slice(&make_uniform_block(b)); + + let mut cur = Cursor::new(row); + let res = sony_arw2_load_raw(&mut cur, dims, &identity).unwrap(); + + for col in 0..32usize { + let expected = if col % 2 == 0 { + (a as u16) << 1 // even columns come from block0 + } else { + (b as u16) << 1 // odd columns come from block1 + }; + assert_eq!(res.pixels[col], expected, "column {col}"); + } + assert_eq!(res.white_level, 0x3ff0); + } } From 4454e8e189f7bc23852eb3ca10734983171f6224 Mon Sep 17 00:00:00 2001 From: ethanrous Date: Sat, 6 Jun 2026 23:26:05 -0400 Subject: [PATCH 4/5] fix: restore Sony ARW2 row-buffer overread guard for malformed blocks Malformed blocks where imax == imin cause 15 ordinary pixels to be decoded (instead of 14), pushing `byte_index + 1` to `dp + 17 = raw_width + 1` on the last block of a row. The previous version of `sony_arw2_load_raw` rejected that with a clean error return; the rewrite dropped the check. Restored it. Also appended a comment documenting the tone_curve length invariant (`>= 0x1000` entries; largest index is `0x7ff << 1` = 0xffe). --- agno/src/sony_decoder.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/agno/src/sony_decoder.rs b/agno/src/sony_decoder.rs index 180a533..19d354b 100644 --- a/agno/src/sony_decoder.rs +++ b/agno/src/sony_decoder.rs @@ -469,6 +469,8 @@ pub fn build_sony_tone_curve(points: [u16; 4]) -> Vec { // The 11-bit codes are expanded through the Sony tone curve (`curve[pix << 1]`) into the // ~14-bit linear domain. Skipping the de-interleave produces a vertical comb artifact; // skipping the curve makes the image ~8x too dark. +// `tone_curve` must have at least 0x1000 entries (as built by `build_sony_tone_curve`); +// pixel codes are clamped to 0x7ff, so the largest index read is `0x7ff << 1` = 0xffe. #[allow(clippy::needless_range_loop)] pub fn sony_arw2_load_raw( reader: &mut R, @@ -519,6 +521,9 @@ pub fn sony_arw2_load_raw( pix16[i] = min_v as u16; } else { let byte_index = dp + (bit >> 3); + if byte_index + 1 >= row_buf.len() { + return Err(DecodeError::CorruptData("Sony ARW2: row buffer overread")); + } let two = u16::from_le_bytes([row_buf[byte_index], row_buf[byte_index + 1]]) as i32; let code7 = (two >> (bit & 7)) & 0x7f; From dabc07a59e1753560508172eb256faa138a02b24 Mon Sep 17 00:00:00 2001 From: ethanrous Date: Sun, 7 Jun 2026 12:56:09 -0400 Subject: [PATCH 5/5] docs: clarify ARW2 identity tone-curve fallback is a linear <<1 expansion --- agno/src/sony_decoder.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agno/src/sony_decoder.rs b/agno/src/sony_decoder.rs index 19d354b..c449cb5 100644 --- a/agno/src/sony_decoder.rs +++ b/agno/src/sony_decoder.rs @@ -439,7 +439,9 @@ pub fn decrypt_sr2_data(decryptor: &mut SonyDecryptor, data: &mut [u8], key: u32 pub fn build_sony_tone_curve(points: [u16; 4]) -> Vec { let mut curve = vec![0u16; 0x4000]; - // Missing/zero tag: identity passthrough so ARW2 still decodes (no expansion). + // Missing/zero tag: identity curve (curve[i] = i). The decoder indexes curve[pix << 1], + // so this still applies a linear << 1 expansion of the 11-bit codes (no Sony tone shaping), + // letting ARW2 decode without the camera's highlight curve. if points == [0, 0, 0, 0] { for (i, c) in curve.iter_mut().enumerate() { *c = i.min(0x3fff) as u16;