docs+feat: decode LC Raw Data/Chromatogram Ch6, characterize Ch5's untestable grammar (#21)

CHANGELOG.md

@@ -17,6 +17,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   the `\x05SummaryInformation` stream (which `.lcd`/`.qgd` don't carry -
   see `docs/format/06-known-limitations.md` #9). Resolves
   Sigilweaver/OpenSZRaw#9.
+- `LC Raw Data/Chromatogram Ch6` (a conventional UV/RID-style LC detector
+  channel, distinct from the still-undecoded `PDA 3D Raw Data`) is now
+  decoded and exposed via `SpectrumSource::iter_chromatograms`, tagged
+  with PSI-MS `MS:1000811` "electromagnetic radiation chromatogram" - see
+  `docs/format/04-lcd-chromatogram-pda.md` and
+  `docs/format/06-known-limitations.md` #11. `Ch5` shares the same
+  verified page/tokenization framing but is deliberately left undecoded:
+  every sample in every locally available file is the same single
+  repeated value, which makes its numeric grammar (delta vs. absolute
+  convention) untestable from this corpus rather than resolved. Partially
+  resolves Sigilweaver/OpenSZRaw#21.
+
+  Contributed by @Nabejo.
 
 ### Documentation
 

crates/openszraw/src/lib.rs

@@ -18,6 +18,12 @@
 //! top-level CFBF storage is present, never by filename. `PDA 3D Raw Data`
 //! (secondary UV detector) is out of scope for this crate, see
 //! `docs/format/04-lcd-chromatogram-pda.md`.
+//!
+//! A separate, unrelated stream - `LC Raw Data/Chromatogram ChN`
+//! (conventional UV/RID-style LC detector channels) - *is* decoded and
+//! exposed via [`openmassspec_core::SpectrumSource::iter_chromatograms`],
+//! see `raw::lc_chrom` and `docs/format/04-lcd-chromatogram-pda.md`'s
+//! "LC Raw Data Chromatogram Ch5/Ch6 decode" section.
 
 #![cfg_attr(not(test), warn(clippy::unwrap_used, clippy::expect_used))]
 

crates/openszraw/src/raw/lc_chrom.rs

@@ -0,0 +1,334 @@
+//! Parsing of `.lcd` `LC Raw Data/Chromatogram ChN` streams - a
+//! conventional LC detector channel (UV/RID-style), structurally
+//! unrelated to `PDA 3D Raw Data` despite sharing the same outer
+//! `RC\x00\x00` segment header.
+//!
+//! See `docs/format/04-lcd-chromatogram-pda.md`'s "LC Raw Data
+//! Chromatogram Ch5/Ch6 decode" session (Sigilweaver/OpenSZRaw#21) for the
+//! full derivation and evidence this implements.
+
+use byteorder::{ByteOrder, LittleEndian};
+
+/// Magic for the 24-byte `RC\x00\x00` segment header shared by
+/// `PDA 3D Raw Data` and `LC Raw Data` streams (see
+/// `docs/format/04-lcd-chromatogram-pda.md`'s "Segment Header" section).
+const SEGMENT_MAGIC: u32 = 17234;
+const SEGMENT_HEADER_SIZE: usize = 24;
+
+/// Byte value at/above which a sample token widens from 1 byte to 2
+/// bytes. The unique zero-exception result of an exhaustive
+/// `(threshold in 0..=255) x (wide_width in 2..=4)` sweep requiring every
+/// page of a stream to decode to exactly its declared point count with no
+/// leftover bytes - see docs/format/04. Confirmed with zero exceptions
+/// across every page of `Chromatogram Ch5`/`Ch6` in all 5 locally
+/// available `PXD020792` files.
+const WIDE_TOKEN_THRESHOLD: u8 = 0x20;
+
+/// Nominal LC channel sample interval, in seconds (2 Hz). Derived from
+/// cross-referencing the fixed 7200-sample stream length against
+/// `TTFL Raw Data/Retention Time`'s own max value (consistently just
+/// under, never over, 3600s / 60 minutes across all 5 corpus files) - see
+/// docs/format/04. Not itself read from any field in the `LC Raw Data`
+/// stream; a corpus-derived constant, like `docs/format/03`'s RLE
+/// decoder constants.
+pub const SAMPLE_INTERVAL_SEC: f64 = 0.5;
+
+/// One decoded `LC Raw Data/Chromatogram ChN` stream: a dense,
+/// evenly-spaced (RT, intensity) time series.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct LcChromatogram {
+    pub time_sec: Vec<f32>,
+    pub intensity: Vec<f32>,
+}
+
+/// Split a `LC Raw Data/Chromatogram ChN` payload into its internal
+/// "pages": each page is a `u16` LE byte-length prefix, that many data
+/// bytes, then a `u16` LE suffix equal to the prefix - back to back, no
+/// gap between one page's suffix and the next page's prefix. This is the
+/// internal sub-segment structure the PDA payload doc's "LC Raw Data"
+/// section flagged as not yet characterized: the whole stream is one
+/// giant `RC\x00\x00` segment (`u32[3]` spans the entire rest of the
+/// stream), but *within* that segment's body, this length-prefix/suffix
+/// framing repeats - the same "first/last length word" wrapper style
+/// already documented for the PDA payload's "symmetric" envelope form,
+/// just applied per-page rather than per-segment. Verified zero-exception
+/// (every page's prefix equals its suffix, every stream fully consumed)
+/// against every one of the 5 `PXD020792` corpus files' `Ch5` and `Ch6`
+/// streams (290 pages total).
+///
+/// Returns `None` if the payload is not an exact, zero-leftover sequence
+/// of such pages.
+fn parse_pages(payload: &[u8]) -> Option<Vec<&[u8]>> {
+    let mut pages = Vec::new();
+    let mut off = 0usize;
+    let n = payload.len();
+    while off < n {
+        if off + 2 > n {
+            return None;
+        }
+        let plen = LittleEndian::read_u16(&payload[off..off + 2]) as usize;
+        let start = off + 2;
+        let end = start.checked_add(plen)?;
+        if end + 2 > n {
+            return None;
+        }
+        let suffix = LittleEndian::read_u16(&payload[end..end + 2]) as usize;
+        if suffix != plen {
+            return None;
+        }
+        pages.push(&payload[start..end]);
+        off = end + 2;
+    }
+    Some(pages)
+}
+
+/// Decode one page's bytes into signed per-sample deltas.
+///
+/// Tokenization: a byte `b < 0x20` is a 1-byte literal, read as a signed
+/// 5-bit two's-complement value (`0..=15` -> `+0..=+15`, `16..=31` ->
+/// `-16..=-1`). A byte `b >= 0x20` (`WIDE_TOKEN_THRESHOLD`) starts a
+/// 2-byte "wide" token: `b`'s low 5 bits become the high bits of a signed
+/// 13-bit value, the following byte the low 8 bits. This
+/// `(threshold, width) = (0x20, 2)` rule is the *only* combination (of an
+/// exhaustive 256x3 sweep) that decodes every page of every locally
+/// available `Ch5`/`Ch6` stream to exactly its expected sample count with
+/// zero leftover bytes - a fully verified structural fact.
+///
+/// The signed-value interpretation (5-bit literal, 13-bit wide) is a
+/// strong but not byte-exact-certain hypothesis: cumulative-summing the
+/// decoded deltas produces a smooth, low-magnitude, physically plausible
+/// chromatogram trace (baseline near zero, a single broad rise, then a
+/// stable plateau of similar absolute magnitude) consistently across all
+/// 5 corpus files, clearly outperforming (by an order of magnitude in
+/// mean per-sample delta size) every alternative byte-order/bit-width
+/// layout tried - see docs/format/04 for the comparison. This is a soft,
+/// physical-plausibility validator (the kind docs/format/04's own
+/// "Further avenues" section recommends), not a byte-exact proof.
+fn decode_page_tokens(page: &[u8]) -> Option<Vec<i32>> {
+    let mut out = Vec::new();
+    let mut i = 0usize;
+    let n = page.len();
+    while i < n {
+        let b = page[i];
+        if b >= WIDE_TOKEN_THRESHOLD {
+            if i + 1 >= n {
+                return None; // truncated wide token
+            }
+            let b1 = page[i + 1];
+            let raw = (i32::from(b & 0x1F) << 8) | i32::from(b1);
+            let signed = if raw >= 4096 { raw - 8192 } else { raw };
+            out.push(signed);
+            i += 2;
+        } else {
+            let signed = if b < 16 {
+                i32::from(b)
+            } else {
+                i32::from(b) - 32
+            };
+            out.push(signed);
+            i += 1;
+        }
+    }
+    Some(out)
+}
+
+/// Decode a full `LC Raw Data/Chromatogram ChN` stream.
+///
+/// Returns `None` when:
+/// - the stream is too short or its 24-byte header magic doesn't match
+///   (not a `RC\x00\x00` segment at all);
+/// - the page framing or tokenization doesn't cleanly consume the whole
+///   payload (a malformed or differently-encoded stream);
+/// - the decoded sample count doesn't match the segment header's own
+///   declared point count (`u32[2]`);
+/// - **every decoded delta is identical** (fewer than 2 distinct decoded
+///   token values across the whole stream). This is deliberate, not a
+///   parsing failure: every locally available file's `Chromatogram Ch5`
+///   decodes cleanly under the rule above but is a single repeated token
+///   for its entire ~1-hour run, which makes it impossible to tell
+///   whether the delta+cumulative-sum interpretation established from
+///   `Ch6`'s real, varying signal actually applies to it, or whether it
+///   is some other convention (e.g. a plain absolute reading) that just
+///   happens to look identical when the value never changes - see
+///   docs/format/04. Emitting a delta-integrated trace for such a stream
+///   risks shipping a fabricated, possibly-wrong unbounded ramp instead
+///   of the flat channel the source data almost certainly represents;
+///   skipping it is the honest choice per `CONTRIBUTING.md`'s clean-room
+///   policy. A future file whose `Ch5` (or any other channel) actually
+///   varies would decode normally under this same rule.
+pub fn decode_stream(data: &[u8]) -> Option<LcChromatogram> {
+    if data.len() < SEGMENT_HEADER_SIZE {
+        return None;
+    }
+    let magic = LittleEndian::read_u32(&data[0..4]);
+    if magic != SEGMENT_MAGIC {
+        return None;
+    }
+    let npts = LittleEndian::read_u32(&data[8..12]) as usize;
+    let blocksz = LittleEndian::read_u32(&data[12..16]) as usize;
+    if blocksz < SEGMENT_HEADER_SIZE || blocksz > data.len() {
+        return None;
+    }
+    let payload = &data[SEGMENT_HEADER_SIZE..blocksz];
+    let pages = parse_pages(payload)?;
+
+    let mut cumulative: i64 = 0;
+    let mut time_sec = Vec::with_capacity(npts);
+    let mut intensity = Vec::with_capacity(npts);
+    let mut distinct_deltas = std::collections::HashSet::new();
+    let mut sample_idx: usize = 0;
+    for page in pages {
+        let deltas = decode_page_tokens(page)?;
+        for d in deltas {
+            distinct_deltas.insert(d);
+            cumulative += i64::from(d);
+            #[allow(clippy::cast_precision_loss)]
+            let t = sample_idx as f64 * SAMPLE_INTERVAL_SEC;
+            time_sec.push(t as f32);
+            #[allow(clippy::cast_precision_loss)]
+            intensity.push(cumulative as f32);
+            sample_idx += 1;
+        }
+    }
+    if sample_idx != npts {
+        return None; // didn't consume exactly the declared point count
+    }
+    if distinct_deltas.len() < 2 {
+        return None; // single repeated value: grammar untestable, see doc comment above
+    }
+    Some(LcChromatogram {
+        time_sec,
+        intensity,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn write_header(buf: &mut Vec<u8>, npts: u32, blocksz: u32) {
+        buf.extend_from_slice(&SEGMENT_MAGIC.to_le_bytes());
+        buf.extend_from_slice(&1u32.to_le_bytes()); // version
+        buf.extend_from_slice(&npts.to_le_bytes());
+        buf.extend_from_slice(&blocksz.to_le_bytes());
+        buf.extend_from_slice(&0u32.to_le_bytes()); // padding
+        buf.extend_from_slice(&0u32.to_le_bytes()); // padding
+    }
+
+    fn write_page(buf: &mut Vec<u8>, page_bytes: &[u8]) {
+        let len = page_bytes.len() as u16;
+        buf.extend_from_slice(&len.to_le_bytes());
+        buf.extend_from_slice(page_bytes);
+        buf.extend_from_slice(&len.to_le_bytes());
+    }
+
+    #[test]
+    fn parse_pages_splits_back_to_back_length_wrapped_pages() {
+        let mut payload = Vec::new();
+        write_page(&mut payload, &[1, 2, 3]);
+        write_page(&mut payload, &[4, 5]);
+        let pages = parse_pages(&payload).expect("parse");
+        assert_eq!(pages, vec![&[1u8, 2, 3][..], &[4u8, 5][..]]);
+    }
+
+    #[test]
+    fn parse_pages_rejects_mismatched_suffix() {
+        let mut payload = Vec::new();
+        payload.extend_from_slice(&3u16.to_le_bytes());
+        payload.extend_from_slice(&[1, 2, 3]);
+        payload.extend_from_slice(&4u16.to_le_bytes()); // wrong suffix
+        assert!(parse_pages(&payload).is_none());
+    }
+
+    #[test]
+    fn decode_page_tokens_reads_signed_5bit_literals() {
+        // 0 -> 0, 1 -> +1, 15 -> +15, 16 -> -16, 31 -> -1.
+        let page = [0u8, 1, 15, 16, 31];
+        let out = decode_page_tokens(&page).expect("decode");
+        assert_eq!(out, vec![0, 1, 15, -16, -1]);
+    }
+
+    #[test]
+    fn decode_page_tokens_reads_signed_13bit_wide_tokens() {
+        // b=0x20 (low 5 bits 0), b1=0x01 -> raw 1 -> +1.
+        // b=0x3F (low 5 bits 0x1F), b1=0xFF -> raw 0x1FFF (8191) -> -1.
+        let page = [0x20u8, 0x01, 0x3F, 0xFF];
+        let out = decode_page_tokens(&page).expect("decode");
+        assert_eq!(out, vec![1, -1]);
+    }
+
+    #[test]
+    fn decode_page_tokens_rejects_truncated_wide_token() {
+        let page = [0x20u8];
+        assert!(decode_page_tokens(&page).is_none());
+    }
+
+    #[test]
+    fn decode_stream_rejects_bad_magic() {
+        let mut data = vec![0u8; SEGMENT_HEADER_SIZE];
+        LittleEndian::write_u32(&mut data[0..4], 0xDEAD_BEEF);
+        assert!(decode_stream(&data).is_none());
+    }
+
+    #[test]
+    fn decode_stream_rejects_too_short_input() {
+        assert!(decode_stream(&[0u8; 4]).is_none());
+    }
+
+    #[test]
+    fn decode_stream_skips_a_single_valued_stream() {
+        // Reproduces the real Ch5 signature: every sample is the same
+        // wide token (here, delta +1 repeated) for the whole stream -
+        // grammar untestable, must be skipped rather than integrated
+        // into a fabricated ramp.
+        let mut payload = Vec::new();
+        let page: Vec<u8> = std::iter::repeat_n([0x20u8, 0x01], 4).flatten().collect();
+        write_page(&mut payload, &page);
+
+        let mut data = Vec::new();
+        let blocksz = (SEGMENT_HEADER_SIZE + payload.len()) as u32;
+        write_header(&mut data, 4, blocksz);
+        data.extend_from_slice(&payload);
+
+        assert!(decode_stream(&data).is_none());
+    }
+
+    #[test]
+    fn decode_stream_round_trips_a_varying_two_page_stream() {
+        // Page 1: deltas +2, +2, -1 (all literal). Page 2: deltas +1,
+        // wide +100. Total 5 samples, npts must match.
+        let mut payload = Vec::new();
+        write_page(&mut payload, &[2, 2, 31]); // +2, +2, -1
+        let wide = 100i32;
+        let b0 = 0x20 | ((wide >> 8) as u8 & 0x1F);
+        let b1 = (wide & 0xFF) as u8;
+        write_page(&mut payload, &[1, b0, b1]); // +1, +100
+
+        let mut data = Vec::new();
+        let blocksz = (SEGMENT_HEADER_SIZE + payload.len()) as u32;
+        write_header(&mut data, 5, blocksz);
+        data.extend_from_slice(&payload);
+
+        let chrom = decode_stream(&data).expect("decode");
+        assert_eq!(chrom.intensity.len(), 5);
+        assert_eq!(chrom.time_sec.len(), 5);
+        // cumulative sum: 2, 4, 3, 4, 104
+        assert_eq!(chrom.intensity, vec![2.0, 4.0, 3.0, 4.0, 104.0]);
+        assert_eq!(chrom.time_sec[1], SAMPLE_INTERVAL_SEC as f32);
+        assert_eq!(chrom.time_sec[4], 4.0 * SAMPLE_INTERVAL_SEC as f32);
+    }
+
+    #[test]
+    fn decode_stream_rejects_point_count_mismatch() {
+        let mut payload = Vec::new();
+        write_page(&mut payload, &[2, 2, 31]);
+
+        let mut data = Vec::new();
+        let blocksz = (SEGMENT_HEADER_SIZE + payload.len()) as u32;
+        // Declare 5 points but only 3 are present.
+        write_header(&mut data, 5, blocksz);
+        data.extend_from_slice(&payload);
+
+        assert!(decode_stream(&data).is_none());
+    }
+}

crates/openszraw/src/raw/mod.rs

@@ -1,5 +1,6 @@
 //! Low-level parsing modules for `.qgd` and `.lcd` files.
 
+pub mod lc_chrom;
 pub mod qgd;
 pub mod qtfl;
 pub mod timestamp;