Opus review

docs/src/content/docs/reference/test-coverage.md

@@ -226,26 +226,39 @@ Test files: `tests/{c,cpp,py,ts,js,csharp,rust}/test_negative.*`
 
 See `tests/NEGATIVE_TESTS.md` for full scenario descriptions.
 
-The 20 scenarios in the table below are registered in every language's `test_negative.*` file. Individual languages carry additional language-specific scenarios: C/C++/TS/JS/C# (24 each) add bulk `pkg_id`/`msg_id` corruption, cross-package rejection, and network `pkg_id` corruption; Python (34) adds those plus diagnostic-counter and status-machine tests; Rust (20) currently matches the uniform set.
+The 33 scenarios in the table below are registered in every language's `test_negative.*` file, covering corruption handling, the `tryNext` drain contract, diagnostic counters (unified semantics in buffer and stream mode), minimal-profile resync, and a chunk-boundary split sweep. All seven languages additionally carry the four package-corruption scenarios (bulk `pkg_id`/`msg_id` corruption, cross-package rejection, network `pkg_id` corruption) for 37 scenarios each; Python (42) adds status-machine and buffer-mode diagnostic extras.
 
 | Error Scenario (test name) | C | C++ | Python | TS | JS | C# | Rust |
 |--------|--------|--------|--------|--------|--------|--------|--------|
+| Buffer mode: CRC failure counters | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Buffer mode: Sequence gap counted | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Buffer mode: garbage prefix partial recovers | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Buffer mode: oversized length recovers | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Buffer mode: recovers after CRC failure | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Buffer reader: skips CRC-failed frame | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Bulk profile: Corrupted CRC | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Corrupted CRC detection | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Corrupted length field detection | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Diagnostics: CRC failure counter | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Diagnostics: Length error counter | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Diagnostics: Reset diagnostics | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Diagnostics: Sequence gap counter | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Diagnostics: Sync recovery counter | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Invalid message ID rejection | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Invalid start bytes detection | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| IPC buffer: unknown msg_id advances one byte | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Minimal profile: Truncated frame | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Multiple frames: CRC error then valid frame | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Multiple frames: Corrupted middle frame | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Network profile: SysId/CompId corruption | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Partial frame across buffer boundary | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Sensor buffer: unknown msg_id resync | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Split sweep: two frames at every boundary | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Split-buffer: CRC error status preserved | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Stream mode: recovers after garbage prefix | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Streaming: Corrupted CRC detection | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Streaming: Garbage data handling | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Streaming: two frames byte-by-byte | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | TryNext drain: CRC/resync + valid | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | TryNext partial pending contract | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Truncated frame detection | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |

src/struct_frame/boilerplate/c/frame_profiles.h

@@ -535,7 +535,9 @@ static inline frame_msg_info_t buffer_reader_next(buffer_reader_t* reader)
                 reader->offset = reader->size;
             }
         } else {
-            reader->offset = reader->size;
+            /* No start bytes (e.g. IPC profile): advance one byte and retry so a
+             * single unknown msg_id doesn't discard the rest of the buffer. */
+            reader->offset += 1;
         }
         result.status = FRAME_MSG_STATUS_SYNC_RECOVERY;
         result.frame_size = reader->offset - old_offset;
@@ -679,6 +681,12 @@ typedef struct accumulating_reader {
     size_t bytes_appended_to_internal; /* bytes copied from current buffer into internal buffer this add_data() call */
     accumulating_reader_state_t state;
 
+    /* Cached per-profile constants so the per-byte push path doesn't recompute
+     * them on every call. */
+    uint8_t cached_start_byte1;
+    uint8_t cached_start_byte2;
+    uint8_t cached_header_size;
+
     const uint8_t* current_buffer;
     size_t current_size;
     size_t current_offset;
@@ -704,6 +712,17 @@ static inline void accumulating_reader_init(
     reader->expected_frame_size = 0;
     reader->bytes_appended_to_internal = 0;
     reader->state = ACC_STATE_IDLE;
+
+    /* Precompute the per-profile constants used on every push_byte() call. */
+    reader->cached_start_byte1 = config->header.start_byte1;
+    reader->cached_start_byte2 = config->header.start_byte2;
+    if (config->header.header_type == HEADER_TINY && config->header.encodes_payload_type) {
+        reader->cached_start_byte1 = get_tiny_start_byte(config->payload.payload_type);
+    }
+    if (config->header.header_type == HEADER_BASIC && config->header.encodes_payload_type) {
+        reader->cached_start_byte2 = get_basic_second_start_byte(config->payload.payload_type);
+    }
+    reader->cached_header_size = profile_header_size(config);
     reader->current_buffer = NULL;
     reader->current_size = 0;
     reader->current_offset = 0;
@@ -739,6 +758,49 @@ static inline frame_msg_info_t _acc_parse_buffer(
     const uint8_t* buffer,
     size_t size);
 
+/* Record per-frame diagnostics for a complete frame consumed in buffer mode,
+ * matching the stream-mode (push_byte) counter semantics:
+ *  - cnt_len_errors when the header length is outside [min_size, size]
+ *  - cnt_seq_gaps on valid frames for profiles that carry a sequence number
+ * frame_start points at the first byte of the frame (start bytes included). */
+static inline void _acc_record_frame_diagnostics(
+    accumulating_reader_t* reader, const uint8_t* frame_start, bool valid)
+{
+    const profile_config_t* config = reader->config;
+    if (config->payload.has_length && reader->get_message_info) {
+        size_t len_offset = config->header.num_start_bytes;
+        if (config->payload.has_seq) len_offset++;
+        if (config->payload.has_sys_id) len_offset++;
+        if (config->payload.has_comp_id) len_offset++;
+        size_t payload_len;
+        if (config->payload.length_bytes == 1) {
+            payload_len = frame_start[len_offset];
+        } else {
+            payload_len = frame_start[len_offset] | ((size_t)frame_start[len_offset + 1] << 8);
+        }
+        uint8_t header_size = profile_header_size(config);
+        uint16_t full_msg_id = 0;
+        if (config->payload.has_pkg_id) {
+            full_msg_id = (uint16_t)frame_start[header_size - 2] << 8;
+        }
+        full_msg_id |= frame_start[header_size - 1];
+        message_info_t info;
+        if (reader->get_message_info(full_msg_id, &info) &&
+            (payload_len > info.size || payload_len < info.min_size)) {
+            reader->diagnostics.cnt_len_errors++;
+        }
+    }
+    if (valid && config->payload.has_seq) {
+        uint8_t seq = frame_start[config->header.num_start_bytes];
+        if (reader->last_seq_valid) {
+            uint8_t expected_seq = (uint8_t)(reader->last_seq + 1);
+            if (seq != expected_seq) reader->diagnostics.cnt_seq_gaps++;
+        }
+        reader->last_seq = seq;
+        reader->last_seq_valid = true;
+    }
+}
+
 static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* reader)
 {
     frame_msg_info_t result = {false, 0, 0, NULL, FRAME_MSG_STATUS_NONE, 0, NULL};
@@ -758,11 +820,26 @@ static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* r
     /* First, try to complete a partial message from the internal buffer.
      * partial_len = bytes that came from a PREVIOUS add_data call (before
      * bytes_appended_to_internal were added this call). */
+    if (reader->internal_data_len > 0 && reader->current_offset == 0 &&
+        reader->bytes_appended_to_internal == 0 &&
+        reader->current_size > reader->current_offset) {
+        /* Full-wedge escape: new data arrived but nothing could be appended
+         * because the internal buffer is full. The buffered bytes can never
+         * complete — discard them so parsing continues from the current
+         * buffer instead of stalling forever. */
+        reader->diagnostics.cnt_sync_recoveries++;
+        reader->diagnostics.cnt_failed_bytes += (uint32_t)reader->internal_data_len;
+        reader->internal_data_len = 0;
+        reader->expected_frame_size = 0;
+        /* fall through to current-buffer parsing below */
+    }
+
     if (reader->internal_data_len > 0 && reader->current_offset == 0) {
         size_t partial_len = reader->internal_data_len - reader->bytes_appended_to_internal;
         result = _acc_parse_buffer(reader, reader->internal_buffer, reader->internal_data_len);
 
         if (result.valid) {
+            _acc_record_frame_diagnostics(reader, reader->internal_buffer, true);
             size_t bytes_from_current = result.frame_size > partial_len ? result.frame_size - partial_len : 0;
             reader->current_offset = bytes_from_current;
             reader->internal_data_len = 0;
@@ -775,6 +852,8 @@ static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* r
             /* Complete frame but CRC failed — count it and skip */
             reader->diagnostics.cnt_crc_failures++;
             reader->diagnostics.cnt_failed_bytes += (uint32_t)result.frame_size;
+            reader->diagnostics.cnt_sync_recoveries++;
+            _acc_record_frame_diagnostics(reader, reader->internal_buffer, false);
             size_t bytes_from_current = result.frame_size > partial_len ? result.frame_size - partial_len : 0;
             reader->current_offset = bytes_from_current;
             reader->internal_data_len = 0;
@@ -783,6 +862,41 @@ static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* r
             return result;
         }
 
+        /* Garbage prefix saved as a partial (WAITING_FOR_START), or a frame
+         * that can never complete because its claimed size exceeds the
+         * internal buffer (still collecting with the buffer full): resync
+         * inside the internal buffer instead of waiting forever. */
+        if (result.status == FRAME_MSG_STATUS_WAITING_FOR_START ||
+            reader->internal_data_len >= reader->buffer_size) {
+            size_t discard = reader->internal_data_len;
+            if (reader->config->header.num_start_bytes >= 1) {
+                if (reader->internal_data_len > 1) {
+                    const void* p = memchr(reader->internal_buffer + 1, (int)sb1,
+                                           reader->internal_data_len - 1);
+                    if (p) discard = (size_t)((const uint8_t*)p - reader->internal_buffer);
+                }
+            } else {
+                /* No start bytes (e.g. IPC): advance one byte and retry. */
+                discard = 1;
+            }
+            reader->diagnostics.cnt_sync_recoveries++;
+            reader->diagnostics.cnt_failed_bytes += (uint32_t)discard;
+            size_t keep = reader->internal_data_len - discard;
+            if (keep > 0) {
+                memmove(reader->internal_buffer, reader->internal_buffer + discard, keep);
+            }
+            /* Only the portion of the discard that reached into this cycle's
+             * appended bytes reduces the appended count. */
+            if (discard > partial_len) {
+                reader->bytes_appended_to_internal -= (discard - partial_len);
+            }
+            reader->internal_data_len = keep;
+            result.valid = false;
+            result.status = FRAME_MSG_STATUS_SYNC_RECOVERY;
+            result.frame_size = discard;
+            return result;
+        }
+
         /* Still not enough data for a complete frame — wait for next add_data() */
         return result;
     }
@@ -797,6 +911,7 @@ static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* r
     result = _acc_parse_buffer(reader, cur, cur_remaining);
 
     if (result.valid && result.frame_size > 0) {
+        _acc_record_frame_diagnostics(reader, cur, true);
         reader->current_offset += result.frame_size;
         return result;
     }
@@ -805,6 +920,8 @@ static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* r
         /* Complete frame with bad CRC — count it, skip it */
         reader->diagnostics.cnt_crc_failures++;
         reader->diagnostics.cnt_failed_bytes += (uint32_t)result.frame_size;
+        reader->diagnostics.cnt_sync_recoveries++;
+        _acc_record_frame_diagnostics(reader, cur, false);
         reader->current_offset += result.frame_size;
         return result;
     }
@@ -826,7 +943,9 @@ static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* r
                 reader->current_offset = reader->current_size;
             }
         } else {
-            reader->current_offset = reader->current_size;
+            /* No start bytes (e.g. IPC profile): advance one byte and retry so a
+             * single unknown msg_id doesn't discard the rest of the buffer. */
+            reader->current_offset += 1;
         }
         result.status = FRAME_MSG_STATUS_SYNC_RECOVERY;
         result.frame_size = reader->current_offset - old_current_offset;
@@ -842,6 +961,11 @@ static inline frame_msg_info_t accumulating_reader_next(accumulating_reader_t* r
         reader->internal_data_len = remaining;
         reader->bytes_appended_to_internal = 0;
         reader->current_offset = reader->current_size;
+    } else if (remaining >= reader->buffer_size) {
+        /* Partial too large to buffer — discard with diagnostics. */
+        reader->diagnostics.cnt_sync_recoveries++;
+        reader->diagnostics.cnt_failed_bytes += (uint32_t)remaining;
+        reader->current_offset = reader->current_size;
     }
 
     return result;
@@ -851,20 +975,13 @@ static inline frame_msg_info_t accumulating_reader_push_byte(accumulating_reader
 {
     frame_msg_info_t result = {false, 0, 0, NULL, FRAME_MSG_STATUS_NONE, 0, NULL};
     result.diagnostics = &reader->diagnostics;
-
-    uint8_t header_size = profile_header_size(reader->config);
-    uint8_t start_byte1 = reader->config->header.start_byte1;
-    uint8_t start_byte2 = reader->config->header.start_byte2;
-
-    /* For tiny headers, start byte encodes payload type */
-    if (reader->config->header.header_type == HEADER_TINY && reader->config->header.encodes_payload_type) {
-        start_byte1 = get_tiny_start_byte(reader->config->payload.payload_type);
-    }
-    /* For basic headers, second start byte encodes payload type */
-    if (reader->config->header.header_type == HEADER_BASIC && reader->config->header.encodes_payload_type) {
-        start_byte2 = get_basic_second_start_byte(reader->config->payload.payload_type);
-    }
-
+
+    /* Use the constants precomputed at init instead of re-deriving them on
+     * every pushed byte. */
+    uint8_t header_size = reader->cached_header_size;
+    uint8_t start_byte1 = reader->cached_start_byte1;
+    uint8_t start_byte2 = reader->cached_start_byte2;
+
     if (reader->state == ACC_STATE_IDLE || reader->state == ACC_STATE_BUFFER_MODE) {
         reader->state = ACC_STATE_LOOKING_FOR_START1;
         reader->internal_data_len = 0;

src/struct_frame/boilerplate/cpp/frame_base.hpp

@@ -16,11 +16,15 @@ struct MessageInfo {
   uint8_t magic2;
   bool valid;        // True if message info is valid
   size_t base_size;  // Non-extension portion size (== size when no extensions)
-
-  MessageInfo() : size(0), magic1(0), magic2(0), valid(false), base_size(0) {}
-  MessageInfo(size_t s, uint8_t m1, uint8_t m2) : size(s), magic1(m1), magic2(m2), valid(true), base_size(s) {}
-  MessageInfo(size_t s, uint8_t m1, uint8_t m2, size_t bs) : size(s), magic1(m1), magic2(m2), valid(true), base_size(bs) {}
-
+  size_t min_size;   // Minimum valid payload size: base_size for fixed messages,
+                     // MIN_SIZE for variable messages
+
+  MessageInfo() : size(0), magic1(0), magic2(0), valid(false), base_size(0), min_size(0) {}
+  MessageInfo(size_t s, uint8_t m1, uint8_t m2) : size(s), magic1(m1), magic2(m2), valid(true), base_size(s), min_size(s) {}
+  MessageInfo(size_t s, uint8_t m1, uint8_t m2, size_t bs) : size(s), magic1(m1), magic2(m2), valid(true), base_size(bs), min_size(bs) {}
+  MessageInfo(size_t s, uint8_t m1, uint8_t m2, size_t bs, size_t ms)
+      : size(s), magic1(m1), magic2(m2), valid(true), base_size(bs), min_size(ms) {}
+
   explicit operator bool() const { return valid; }
 };