Fixing wire evolution

docs/src/content/docs/basic-usage/message-definitions.md

@@ -818,6 +818,7 @@ The generator enforces:
 - Envelope oneof using `msgid` discriminator must have messages with msgid
 - `extensions_start` must be ≥ 2 (at least one non-extension base field required)
 - `extensions_start` must equal an existing field number in the same scope
+- Message-level `extensions_start` cannot be combined with `oneof` fields — the wire serializes fields before oneofs, so extension fields would not be trailing. Use `option extensions_start` *inside* the `oneof` to add extension variants instead
 
 ## Wire Evolution (Extension Fields)
 
@@ -834,7 +835,7 @@ The generator enforces:
 
 ### Message-level extensions
 
-Declare `option extensions_start = N;` anywhere in the message body, then list extension fields with numbers `>= N`:
+Declare `option extensions_start = N;` anywhere in the message body, then list extension fields with numbers `>= N`. Messages containing a `oneof` cannot use message-level extension fields (the wire serializes fields before oneofs, so the extension bytes would not be trailing) — use [oneof-level extensions](#oneof-level-extensions-extension-variants) instead:
 
 ```proto
 message StatusReport {

src/struct_frame/c_gen.py

@@ -642,61 +642,79 @@ def _flush_fixed_run():
                         "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1}
         for key, field in msg.fields.items():
             var_name = field.name
+            field_lines = []
+            min_prefix = 0
             if field.is_array and field.max_size is not None:
                 # Variable array: count stored as uint8_t or uint16_t on the wire
                 count_bytes = 2 if field.max_size > 255 else 1
+                min_prefix = count_bytes
                 if field.field_type in ("string", "bytes"):
                     element_size = field.element_size if field.element_size else 1
-                    result += f'    // {var_name}: variable string array\n'
+                    field_lines.append(f'// {var_name}: variable string array')
                     if count_bytes == 2:
-                        result += f'    if (offset + 2 > buffer_size) return 0;\n'
-                        result += f'    memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;\n'
-                        result += f'    if (msg->{var_name}.count > {field.max_size}) return 0;\n'
+                        field_lines.append(f'if (offset + 2 > buffer_size) return 0;')
+                        field_lines.append(f'memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;')
+                        field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;')
                     else:
-                        result += f'    if (offset >= buffer_size) return 0;\n'
-                        result += f'    msg->{var_name}.count = buffer[offset++];\n'
-                        result += f'    if (msg->{var_name}.count > {field.max_size}) return 0;\n'
-                    result += f'    if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;\n'
-                    result += f'    memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});\n'
-                    result += f'    offset += msg->{var_name}.count * {element_size};\n'
+                        field_lines.append(f'if (offset >= buffer_size) return 0;')
+                        field_lines.append(f'msg->{var_name}.count = buffer[offset++];')
+                        field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;')
+                    field_lines.append(f'if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;')
+                    field_lines.append(f'memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});')
+                    field_lines.append(f'offset += msg->{var_name}.count * {element_size};')
                 else:
                     if field.field_type in _type_sizes2:
                         element_size = _type_sizes2[field.field_type]
                     else:
                         element_size = (field.size - count_bytes) // field.max_size
-                    result += f'    // {var_name}: variable array\n'
+                    field_lines.append(f'// {var_name}: variable array')
                     if count_bytes == 2:
-                        result += f'    if (offset + 2 > buffer_size) return 0;\n'
-                        result += f'    memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;\n'
-                        result += f'    if (msg->{var_name}.count > {field.max_size}) return 0;\n'
+                        field_lines.append(f'if (offset + 2 > buffer_size) return 0;')
+                        field_lines.append(f'memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;')
+                        field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;')
                     else:
-                        result += f'    if (offset >= buffer_size) return 0;\n'
-                        result += f'    msg->{var_name}.count = buffer[offset++];\n'
-                        result += f'    if (msg->{var_name}.count > {field.max_size}) return 0;\n'
-                    result += f'    if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;\n'
-                    result += f'    memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});\n'
-                    result += f'    offset += msg->{var_name}.count * {element_size};\n'
+                        field_lines.append(f'if (offset >= buffer_size) return 0;')
+                        field_lines.append(f'msg->{var_name}.count = buffer[offset++];')
+                        field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;')
+                    field_lines.append(f'if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;')
+                    field_lines.append(f'memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});')
+                    field_lines.append(f'offset += msg->{var_name}.count * {element_size};')
             elif field.field_type in ("string", "bytes") and field.max_size is not None:
                 # Variable string: length stored as uint8_t or uint16_t on the wire
                 length_bytes = 2 if field.max_size > 255 else 1
-                result += f'    // {var_name}: variable string\n'
+                min_prefix = length_bytes
+                field_lines.append(f'// {var_name}: variable string')
                 if length_bytes == 2:
-                    result += f'    if (offset + 2 > buffer_size) return 0;\n'
-                    result += f'    memcpy(&msg->{var_name}.length, buffer + offset, 2); offset += 2;\n'
-                    result += f'    if (msg->{var_name}.length > {field.max_size}) return 0;\n'
+                    field_lines.append(f'if (offset + 2 > buffer_size) return 0;')
+                    field_lines.append(f'memcpy(&msg->{var_name}.length, buffer + offset, 2); offset += 2;')
+                    field_lines.append(f'if (msg->{var_name}.length > {field.max_size}) return 0;')
                 else:
-                    result += f'    if (offset >= buffer_size) return 0;\n'
-                    result += f'    msg->{var_name}.length = buffer[offset++];\n'
-                    result += f'    if (msg->{var_name}.length > {field.max_size}) return 0;\n'
-                result += f'    if (offset + msg->{var_name}.length > buffer_size) return 0;\n'
-                result += f'    memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.length);\n'
-                result += f'    offset += msg->{var_name}.length;\n'
+                    field_lines.append(f'if (offset >= buffer_size) return 0;')
+                    field_lines.append(f'msg->{var_name}.length = buffer[offset++];')
+                    field_lines.append(f'if (msg->{var_name}.length > {field.max_size}) return 0;')
+                field_lines.append(f'if (offset + msg->{var_name}.length > buffer_size) return 0;')
+                field_lines.append(f'memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.length);')
+                field_lines.append(f'offset += msg->{var_name}.length;')
             else:
                 # Fixed-size field
-                result += f'    // {var_name}: fixed size ({field.size} bytes)\n'
-                result += f'    if (offset + {field.size} > buffer_size) return 0;\n'
-                result += f'    memcpy(&msg->{var_name}, buffer + offset, {field.size});\n'
-                result += f'    offset += {field.size};\n'
+                min_prefix = field.size
+                field_lines.append(f'// {var_name}: fixed size ({field.size} bytes)')
+                field_lines.append(f'if (offset + {field.size} > buffer_size) return 0;')
+                field_lines.append(f'memcpy(&msg->{var_name}, buffer + offset, {field.size});')
+                field_lines.append(f'offset += {field.size};')
+
+            if getattr(field, 'is_extension', False):
+                # Extension field: older senders may omit it entirely. Only attempt
+                # the read if enough bytes remain; otherwise leave the field at its
+                # zero-initialized default (wire evolution, no caller padding needed).
+                result += f'    // {var_name}: extension field, tolerate a short buffer\n'
+                result += f'    if (offset + {min_prefix} <= buffer_size) {{\n'
+                for line in field_lines:
+                    result += f'    {line}\n'
+                result += f'    }}\n'
+            else:
+                for line in field_lines:
+                    result += f'    {line}\n'
 
         # Oneofs: read discriminator then union bytes (or length-prefix + variant bytes for variable oneof)
         for oneof_name, oneof in msg.oneofs.items():
@@ -799,17 +817,22 @@ def _generate_unified_unpack(msg, structName, defineName):
         if not msg.variable:
             result += f'\n/**\n'
             result += f' * Deserialize function for {structName}.\n'
-            result += f' * For fixed-size messages: uses memcpy with size validation\n'
+            result += f' * For fixed-size messages: uses memcpy with size validation.\n'
+            result += f' * Wire evolution: a buffer shorter than MAX_SIZE (older sender, base fields\n'
+            result += f' * only) is zero-filled for the missing extension fields; a buffer longer\n'
+            result += f' * than MAX_SIZE (newer sender) has its trailing extension bytes ignored.\n'
+            result += f' * Callers never need to pad or truncate the buffer themselves.\n'
             result += f' * @param buffer Input buffer\n'
             result += f' * @param buffer_size Size of the input buffer\n'
             result += f' * @param msg Pointer to the message to deserialize into\n'
-            result += f' * @return The number of bytes read, or 0 if buffer is invalid\n'
+            result += f' * @return The number of bytes copied from the buffer\n'
             result += f' */\n'
             result += f'static inline size_t {structName}_deserialize(const uint8_t* buffer, size_t buffer_size, {structName}* msg) {{\n'
-            result += f'    /* Fixed-size message - use direct copy */\n'
-            result += f'    if (buffer_size < {defineName}_MAX_SIZE) return 0;\n'
-            result += f'    memcpy(msg, buffer, {defineName}_MAX_SIZE);\n'
-            result += f'    return {defineName}_MAX_SIZE;\n'
+            result += f'    /* Fixed-size message - zero-fill any bytes the sender omitted (wire evolution) */\n'
+            result += f'    size_t copy_len = buffer_size < {defineName}_MAX_SIZE ? buffer_size : {defineName}_MAX_SIZE;\n'
+            result += f'    memset(msg, 0, sizeof({structName}));\n'
+            result += f'    if (copy_len > 0) memcpy(msg, buffer, copy_len);\n'
+            result += f'    return copy_len;\n'
             result += f'}}\n'
 
             # Also add serialize() for non-variable messages
@@ -1079,9 +1102,11 @@ def _generate_field_init(field, prefix="msg", index=0):
             if field.size_option is not None:
                 out += f'    strncpy({prefix}.{var_name}, "test_string", sizeof({prefix}.{var_name}) - 1);\n'
             elif field.max_size is not None:
-                test_str = "test_string"
+                # Clamp the test string to the field's max_size so length stays
+                # within capacity (a length > max_size is rejected by decoders).
+                test_str = "test_string"[:field.max_size]
                 out += f'    {prefix}.{var_name}.length = {len(test_str)};\n'
-                out += f'    strncpy({prefix}.{var_name}.data, "{test_str}", sizeof({prefix}.{var_name}.data) - 1);\n'
+                out += f'    memcpy({prefix}.{var_name}.data, "{test_str}", {len(test_str)});\n'
         else:
             dummy = TestCGen._dummy_value(field, index)
             if dummy is not None:

src/struct_frame/cpp_gen.py

@@ -601,29 +601,46 @@ def _flush_fixed_run():
 
         for key, field in msg.fields.items():
             var_name = field.name
+            field_lines = []
+            min_prefix = 0
             if field.is_array and field.max_size is not None:
                 type_sizes = {"uint8": 1, "int8": 1, "uint16": 2, "int16": 2, "uint32": 4, "int32": 4, "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1}
                 if field.field_type in ("string", "bytes"):
                     element_size = field.element_size if field.element_size else 1
                 else:
                     element_size = type_sizes.get(field.field_type, (field.size - 1) // field.max_size)
-                result += f'        if (offset >= buffer_size) return 0;\n'
-                result += f'        {var_name}.count = buffer[offset++];\n'
-                result += f'        if ({var_name}.count > {field.max_size}) {var_name}.count = {field.max_size};\n'
-                result += f'        if (offset + {var_name}.count * {element_size} > buffer_size) return 0;\n'
-                result += f'        std::memcpy({var_name}.data, buffer + offset, {var_name}.count * {element_size});\n'
-                result += f'        offset += {var_name}.count * {element_size};\n'
+                min_prefix = 1
+                field_lines.append(f'if (offset >= buffer_size) return 0;')
+                field_lines.append(f'{var_name}.count = buffer[offset++];')
+                field_lines.append(f'if ({var_name}.count > {field.max_size}) {var_name}.count = {field.max_size};')
+                field_lines.append(f'if (offset + {var_name}.count * {element_size} > buffer_size) return 0;')
+                field_lines.append(f'std::memcpy({var_name}.data, buffer + offset, {var_name}.count * {element_size});')
+                field_lines.append(f'offset += {var_name}.count * {element_size};')
             elif field.field_type in ("string", "bytes") and field.max_size is not None:
-                result += f'        if (offset >= buffer_size) return 0;\n'
-                result += f'        {var_name}.length = buffer[offset++];\n'
-                result += f'        if ({var_name}.length > {field.max_size}) {var_name}.length = {field.max_size};\n'
-                result += f'        if (offset + {var_name}.length > buffer_size) return 0;\n'
-                result += f'        std::memcpy({var_name}.data, buffer + offset, {var_name}.length);\n'
-                result += f'        offset += {var_name}.length;\n'
+                min_prefix = 1
+                field_lines.append(f'if (offset >= buffer_size) return 0;')
+                field_lines.append(f'{var_name}.length = buffer[offset++];')
+                field_lines.append(f'if ({var_name}.length > {field.max_size}) {var_name}.length = {field.max_size};')
+                field_lines.append(f'if (offset + {var_name}.length > buffer_size) return 0;')
+                field_lines.append(f'std::memcpy({var_name}.data, buffer + offset, {var_name}.length);')
+                field_lines.append(f'offset += {var_name}.length;')
+            else:
+                min_prefix = field.size
+                field_lines.append(f'if (offset + {field.size} > buffer_size) return 0;')
+                field_lines.append(f'std::memcpy(&{var_name}, buffer + offset, {field.size});')
+                field_lines.append(f'offset += {field.size};')
+
+            if getattr(field, 'is_extension', False):
+                # Extension field: older senders may omit it entirely. Only attempt
+                # the read if enough bytes remain; otherwise leave the field at its
+                # zero-initialized default (wire evolution, no caller padding needed).
+                result += f'        if (offset + {min_prefix} <= buffer_size) {{\n'
+                for line in field_lines:
+                    result += f'        {line}\n'
+                result += f'        }}\n'
             else:
-                result += f'        if (offset + {field.size} > buffer_size) return 0;\n'
-                result += f'        std::memcpy(&{var_name}, buffer + offset, {field.size});\n'
-                result += f'        offset += {field.size};\n'
+                for line in field_lines:
+                    result += f'        {line}\n'
 
         # Oneofs: read discriminator then union bytes (or length-prefix + variant bytes for variable oneof)
         for oneof_name, oneof in msg.oneofs.items():
@@ -711,11 +728,15 @@ def _generate_unified_unpack(msg, structName):
                 result += f'            return _deserialize_variable(buffer, buffer_size);\n'
                 result += f'        }}\n'
         else:
-            # Non-variable message: simple memcpy with size check
-            result += f'        // Fixed-size message - use direct copy\n'
-            result += f'        if (buffer_size < MAX_SIZE) return 0;\n'
-            result += f'        std::memcpy(this, buffer, MAX_SIZE);\n'
-            result += f'        return MAX_SIZE;\n'
+            # Non-variable message: zero-fill any bytes the sender omitted (wire
+            # evolution). A buffer shorter than MAX_SIZE (older sender, base
+            # fields only) leaves the missing extension fields at their default;
+            # a buffer longer than MAX_SIZE (newer sender) has its trailing
+            # extension bytes ignored. Callers never need to pad or truncate.
+            result += f'        size_t copy_len = buffer_size < MAX_SIZE ? buffer_size : MAX_SIZE;\n'
+            result += f'        std::memset(this, 0, sizeof(*this));\n'
+            result += f'        if (copy_len > 0) std::memcpy(this, buffer, copy_len);\n'
+            result += f'        return copy_len;\n'
 
         result += f'    }}\n'
 
@@ -1117,10 +1138,12 @@ def _generate_field_init(field, use_namespace=False, prefix="msg", index=0):
                 # Fixed string
                 result += f'    std::strncpy({prefix}.{var_name}, "test_string", sizeof({prefix}.{var_name}) - 1);\n'
             elif field.max_size is not None:
-                # Variable string
-                test_str = "test_string"
+                # Variable string. Clamp the test string to the field's max_size
+                # so length stays within capacity (length > max_size is rejected
+                # by decoders).
+                test_str = "test_string"[:field.max_size]
                 result += f'    {prefix}.{var_name}.length = {len(test_str)};\n'
-                result += f'    std::strncpy({prefix}.{var_name}.data, "{test_str}", sizeof({prefix}.{var_name}.data) - 1);\n'
+                result += f'    std::memcpy({prefix}.{var_name}.data, "{test_str}", {len(test_str)});\n'
         else:
             # Regular field
             result += f"    {prefix}.{var_name} = {TestCppGen._get_dummy_value(field, use_namespace, index)};\n"