diff --git a/docs/src/content/docs/basic-usage/message-definitions.md b/docs/src/content/docs/basic-usage/message-definitions.md index 4580776b..5eefbc3d 100644 --- a/docs/src/content/docs/basic-usage/message-definitions.md +++ b/docs/src/content/docs/basic-usage/message-definitions.md @@ -818,6 +818,7 @@ The generator enforces: - Envelope oneof using `msgid` discriminator must have messages with msgid - `extensions_start` must be ≥ 2 (at least one non-extension base field required) - `extensions_start` must equal an existing field number in the same scope +- Message-level `extensions_start` cannot be combined with `oneof` fields — the wire serializes fields before oneofs, so extension fields would not be trailing. Use `option extensions_start` *inside* the `oneof` to add extension variants instead ## Wire Evolution (Extension Fields) @@ -834,7 +835,7 @@ The generator enforces: ### Message-level extensions -Declare `option extensions_start = N;` anywhere in the message body, then list extension fields with numbers `>= N`: +Declare `option extensions_start = N;` anywhere in the message body, then list extension fields with numbers `>= N`. Messages containing a `oneof` cannot use message-level extension fields (the wire serializes fields before oneofs, so the extension bytes would not be trailing) — use [oneof-level extensions](#oneof-level-extensions-extension-variants) instead: ```proto message StatusReport { diff --git a/src/struct_frame/c_gen.py b/src/struct_frame/c_gen.py index 1bd7f3ad..c85a6fab 100644 --- a/src/struct_frame/c_gen.py +++ b/src/struct_frame/c_gen.py @@ -642,61 +642,79 @@ def _flush_fixed_run(): "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1} for key, field in msg.fields.items(): var_name = field.name + field_lines = [] + min_prefix = 0 if field.is_array and field.max_size is not None: # Variable array: count stored as uint8_t or uint16_t on the wire count_bytes = 2 if field.max_size > 255 else 1 + min_prefix = count_bytes if field.field_type in ("string", "bytes"): element_size = field.element_size if field.element_size else 1 - result += f' // {var_name}: variable string array\n' + field_lines.append(f'// {var_name}: variable string array') if count_bytes == 2: - result += f' if (offset + 2 > buffer_size) return 0;\n' - result += f' memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;\n' - result += f' if (msg->{var_name}.count > {field.max_size}) return 0;\n' + field_lines.append(f'if (offset + 2 > buffer_size) return 0;') + field_lines.append(f'memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;') + field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;') else: - result += f' if (offset >= buffer_size) return 0;\n' - result += f' msg->{var_name}.count = buffer[offset++];\n' - result += f' if (msg->{var_name}.count > {field.max_size}) return 0;\n' - result += f' if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;\n' - result += f' memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});\n' - result += f' offset += msg->{var_name}.count * {element_size};\n' + field_lines.append(f'if (offset >= buffer_size) return 0;') + field_lines.append(f'msg->{var_name}.count = buffer[offset++];') + field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;') + field_lines.append(f'if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;') + field_lines.append(f'memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});') + field_lines.append(f'offset += msg->{var_name}.count * {element_size};') else: if field.field_type in _type_sizes2: element_size = _type_sizes2[field.field_type] else: element_size = (field.size - count_bytes) // field.max_size - result += f' // {var_name}: variable array\n' + field_lines.append(f'// {var_name}: variable array') if count_bytes == 2: - result += f' if (offset + 2 > buffer_size) return 0;\n' - result += f' memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;\n' - result += f' if (msg->{var_name}.count > {field.max_size}) return 0;\n' + field_lines.append(f'if (offset + 2 > buffer_size) return 0;') + field_lines.append(f'memcpy(&msg->{var_name}.count, buffer + offset, 2); offset += 2;') + field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;') else: - result += f' if (offset >= buffer_size) return 0;\n' - result += f' msg->{var_name}.count = buffer[offset++];\n' - result += f' if (msg->{var_name}.count > {field.max_size}) return 0;\n' - result += f' if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;\n' - result += f' memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});\n' - result += f' offset += msg->{var_name}.count * {element_size};\n' + field_lines.append(f'if (offset >= buffer_size) return 0;') + field_lines.append(f'msg->{var_name}.count = buffer[offset++];') + field_lines.append(f'if (msg->{var_name}.count > {field.max_size}) return 0;') + field_lines.append(f'if (offset + msg->{var_name}.count * {element_size} > buffer_size) return 0;') + field_lines.append(f'memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.count * {element_size});') + field_lines.append(f'offset += msg->{var_name}.count * {element_size};') elif field.field_type in ("string", "bytes") and field.max_size is not None: # Variable string: length stored as uint8_t or uint16_t on the wire length_bytes = 2 if field.max_size > 255 else 1 - result += f' // {var_name}: variable string\n' + min_prefix = length_bytes + field_lines.append(f'// {var_name}: variable string') if length_bytes == 2: - result += f' if (offset + 2 > buffer_size) return 0;\n' - result += f' memcpy(&msg->{var_name}.length, buffer + offset, 2); offset += 2;\n' - result += f' if (msg->{var_name}.length > {field.max_size}) return 0;\n' + field_lines.append(f'if (offset + 2 > buffer_size) return 0;') + field_lines.append(f'memcpy(&msg->{var_name}.length, buffer + offset, 2); offset += 2;') + field_lines.append(f'if (msg->{var_name}.length > {field.max_size}) return 0;') else: - result += f' if (offset >= buffer_size) return 0;\n' - result += f' msg->{var_name}.length = buffer[offset++];\n' - result += f' if (msg->{var_name}.length > {field.max_size}) return 0;\n' - result += f' if (offset + msg->{var_name}.length > buffer_size) return 0;\n' - result += f' memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.length);\n' - result += f' offset += msg->{var_name}.length;\n' + field_lines.append(f'if (offset >= buffer_size) return 0;') + field_lines.append(f'msg->{var_name}.length = buffer[offset++];') + field_lines.append(f'if (msg->{var_name}.length > {field.max_size}) return 0;') + field_lines.append(f'if (offset + msg->{var_name}.length > buffer_size) return 0;') + field_lines.append(f'memcpy(msg->{var_name}.data, buffer + offset, msg->{var_name}.length);') + field_lines.append(f'offset += msg->{var_name}.length;') else: # Fixed-size field - result += f' // {var_name}: fixed size ({field.size} bytes)\n' - result += f' if (offset + {field.size} > buffer_size) return 0;\n' - result += f' memcpy(&msg->{var_name}, buffer + offset, {field.size});\n' - result += f' offset += {field.size};\n' + min_prefix = field.size + field_lines.append(f'// {var_name}: fixed size ({field.size} bytes)') + field_lines.append(f'if (offset + {field.size} > buffer_size) return 0;') + field_lines.append(f'memcpy(&msg->{var_name}, buffer + offset, {field.size});') + field_lines.append(f'offset += {field.size};') + + if getattr(field, 'is_extension', False): + # Extension field: older senders may omit it entirely. Only attempt + # the read if enough bytes remain; otherwise leave the field at its + # zero-initialized default (wire evolution, no caller padding needed). + result += f' // {var_name}: extension field, tolerate a short buffer\n' + result += f' if (offset + {min_prefix} <= buffer_size) {{\n' + for line in field_lines: + result += f' {line}\n' + result += f' }}\n' + else: + for line in field_lines: + result += f' {line}\n' # Oneofs: read discriminator then union bytes (or length-prefix + variant bytes for variable oneof) for oneof_name, oneof in msg.oneofs.items(): @@ -799,17 +817,22 @@ def _generate_unified_unpack(msg, structName, defineName): if not msg.variable: result += f'\n/**\n' result += f' * Deserialize function for {structName}.\n' - result += f' * For fixed-size messages: uses memcpy with size validation\n' + result += f' * For fixed-size messages: uses memcpy with size validation.\n' + result += f' * Wire evolution: a buffer shorter than MAX_SIZE (older sender, base fields\n' + result += f' * only) is zero-filled for the missing extension fields; a buffer longer\n' + result += f' * than MAX_SIZE (newer sender) has its trailing extension bytes ignored.\n' + result += f' * Callers never need to pad or truncate the buffer themselves.\n' result += f' * @param buffer Input buffer\n' result += f' * @param buffer_size Size of the input buffer\n' result += f' * @param msg Pointer to the message to deserialize into\n' - result += f' * @return The number of bytes read, or 0 if buffer is invalid\n' + result += f' * @return The number of bytes copied from the buffer\n' result += f' */\n' result += f'static inline size_t {structName}_deserialize(const uint8_t* buffer, size_t buffer_size, {structName}* msg) {{\n' - result += f' /* Fixed-size message - use direct copy */\n' - result += f' if (buffer_size < {defineName}_MAX_SIZE) return 0;\n' - result += f' memcpy(msg, buffer, {defineName}_MAX_SIZE);\n' - result += f' return {defineName}_MAX_SIZE;\n' + result += f' /* Fixed-size message - zero-fill any bytes the sender omitted (wire evolution) */\n' + result += f' size_t copy_len = buffer_size < {defineName}_MAX_SIZE ? buffer_size : {defineName}_MAX_SIZE;\n' + result += f' memset(msg, 0, sizeof({structName}));\n' + result += f' if (copy_len > 0) memcpy(msg, buffer, copy_len);\n' + result += f' return copy_len;\n' result += f'}}\n' # Also add serialize() for non-variable messages @@ -1079,9 +1102,11 @@ def _generate_field_init(field, prefix="msg", index=0): if field.size_option is not None: out += f' strncpy({prefix}.{var_name}, "test_string", sizeof({prefix}.{var_name}) - 1);\n' elif field.max_size is not None: - test_str = "test_string" + # Clamp the test string to the field's max_size so length stays + # within capacity (a length > max_size is rejected by decoders). + test_str = "test_string"[:field.max_size] out += f' {prefix}.{var_name}.length = {len(test_str)};\n' - out += f' strncpy({prefix}.{var_name}.data, "{test_str}", sizeof({prefix}.{var_name}.data) - 1);\n' + out += f' memcpy({prefix}.{var_name}.data, "{test_str}", {len(test_str)});\n' else: dummy = TestCGen._dummy_value(field, index) if dummy is not None: diff --git a/src/struct_frame/cpp_gen.py b/src/struct_frame/cpp_gen.py index 487feed0..257cc0e3 100644 --- a/src/struct_frame/cpp_gen.py +++ b/src/struct_frame/cpp_gen.py @@ -601,29 +601,46 @@ def _flush_fixed_run(): for key, field in msg.fields.items(): var_name = field.name + field_lines = [] + min_prefix = 0 if field.is_array and field.max_size is not None: type_sizes = {"uint8": 1, "int8": 1, "uint16": 2, "int16": 2, "uint32": 4, "int32": 4, "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1} if field.field_type in ("string", "bytes"): element_size = field.element_size if field.element_size else 1 else: element_size = type_sizes.get(field.field_type, (field.size - 1) // field.max_size) - result += f' if (offset >= buffer_size) return 0;\n' - result += f' {var_name}.count = buffer[offset++];\n' - result += f' if ({var_name}.count > {field.max_size}) {var_name}.count = {field.max_size};\n' - result += f' if (offset + {var_name}.count * {element_size} > buffer_size) return 0;\n' - result += f' std::memcpy({var_name}.data, buffer + offset, {var_name}.count * {element_size});\n' - result += f' offset += {var_name}.count * {element_size};\n' + min_prefix = 1 + field_lines.append(f'if (offset >= buffer_size) return 0;') + field_lines.append(f'{var_name}.count = buffer[offset++];') + field_lines.append(f'if ({var_name}.count > {field.max_size}) {var_name}.count = {field.max_size};') + field_lines.append(f'if (offset + {var_name}.count * {element_size} > buffer_size) return 0;') + field_lines.append(f'std::memcpy({var_name}.data, buffer + offset, {var_name}.count * {element_size});') + field_lines.append(f'offset += {var_name}.count * {element_size};') elif field.field_type in ("string", "bytes") and field.max_size is not None: - result += f' if (offset >= buffer_size) return 0;\n' - result += f' {var_name}.length = buffer[offset++];\n' - result += f' if ({var_name}.length > {field.max_size}) {var_name}.length = {field.max_size};\n' - result += f' if (offset + {var_name}.length > buffer_size) return 0;\n' - result += f' std::memcpy({var_name}.data, buffer + offset, {var_name}.length);\n' - result += f' offset += {var_name}.length;\n' + min_prefix = 1 + field_lines.append(f'if (offset >= buffer_size) return 0;') + field_lines.append(f'{var_name}.length = buffer[offset++];') + field_lines.append(f'if ({var_name}.length > {field.max_size}) {var_name}.length = {field.max_size};') + field_lines.append(f'if (offset + {var_name}.length > buffer_size) return 0;') + field_lines.append(f'std::memcpy({var_name}.data, buffer + offset, {var_name}.length);') + field_lines.append(f'offset += {var_name}.length;') + else: + min_prefix = field.size + field_lines.append(f'if (offset + {field.size} > buffer_size) return 0;') + field_lines.append(f'std::memcpy(&{var_name}, buffer + offset, {field.size});') + field_lines.append(f'offset += {field.size};') + + if getattr(field, 'is_extension', False): + # Extension field: older senders may omit it entirely. Only attempt + # the read if enough bytes remain; otherwise leave the field at its + # zero-initialized default (wire evolution, no caller padding needed). + result += f' if (offset + {min_prefix} <= buffer_size) {{\n' + for line in field_lines: + result += f' {line}\n' + result += f' }}\n' else: - result += f' if (offset + {field.size} > buffer_size) return 0;\n' - result += f' std::memcpy(&{var_name}, buffer + offset, {field.size});\n' - result += f' offset += {field.size};\n' + for line in field_lines: + result += f' {line}\n' # Oneofs: read discriminator then union bytes (or length-prefix + variant bytes for variable oneof) for oneof_name, oneof in msg.oneofs.items(): @@ -711,11 +728,15 @@ def _generate_unified_unpack(msg, structName): result += f' return _deserialize_variable(buffer, buffer_size);\n' result += f' }}\n' else: - # Non-variable message: simple memcpy with size check - result += f' // Fixed-size message - use direct copy\n' - result += f' if (buffer_size < MAX_SIZE) return 0;\n' - result += f' std::memcpy(this, buffer, MAX_SIZE);\n' - result += f' return MAX_SIZE;\n' + # Non-variable message: zero-fill any bytes the sender omitted (wire + # evolution). A buffer shorter than MAX_SIZE (older sender, base + # fields only) leaves the missing extension fields at their default; + # a buffer longer than MAX_SIZE (newer sender) has its trailing + # extension bytes ignored. Callers never need to pad or truncate. + result += f' size_t copy_len = buffer_size < MAX_SIZE ? buffer_size : MAX_SIZE;\n' + result += f' std::memset(this, 0, sizeof(*this));\n' + result += f' if (copy_len > 0) std::memcpy(this, buffer, copy_len);\n' + result += f' return copy_len;\n' result += f' }}\n' @@ -1117,10 +1138,12 @@ def _generate_field_init(field, use_namespace=False, prefix="msg", index=0): # Fixed string result += f' std::strncpy({prefix}.{var_name}, "test_string", sizeof({prefix}.{var_name}) - 1);\n' elif field.max_size is not None: - # Variable string - test_str = "test_string" + # Variable string. Clamp the test string to the field's max_size + # so length stays within capacity (length > max_size is rejected + # by decoders). + test_str = "test_string"[:field.max_size] result += f' {prefix}.{var_name}.length = {len(test_str)};\n' - result += f' std::strncpy({prefix}.{var_name}.data, "{test_str}", sizeof({prefix}.{var_name}.data) - 1);\n' + result += f' std::memcpy({prefix}.{var_name}.data, "{test_str}", {len(test_str)});\n' else: # Regular field result += f" {prefix}.{var_name} = {TestCppGen._get_dummy_value(field, use_namespace, index)};\n" diff --git a/src/struct_frame/csharp_gen.py b/src/struct_frame/csharp_gen.py index 3f12925e..ece769a6 100644 --- a/src/struct_frame/csharp_gen.py +++ b/src/struct_frame/csharp_gen.py @@ -657,7 +657,20 @@ def generate(msg, package=None, equality=False): result += ' /// \n' result += f' private static {structName} _DeserializeMaxSize(ReadOnlySpan data)\n' result += ' {\n' - + else: + # Non-variable message: zero-fill any bytes the sender omitted (wire + # evolution). A buffer shorter than MaxSize (older sender, base fields + # only) leaves the missing extension fields at their default; a buffer + # longer than MaxSize (newer sender) has its trailing extension bytes + # ignored. Callers never need to pad or truncate the buffer themselves. + result += ' if (data.Length != MaxSize)\n' + result += ' {\n' + result += ' byte[] _padded = new byte[MaxSize];\n' + result += ' int _n = Math.Min(data.Length, MaxSize);\n' + result += ' data.Slice(0, _n).CopyTo(_padded);\n' + result += ' data = _padded;\n' + result += ' }\n' + result += f' var msg = new {structName}();\n' offset = 0 @@ -1025,6 +1038,8 @@ def _generate_variable_methods(msg, structName, renamed_enums=None): for key, f in msg.fields.items(): var_name = pascal_case(f.name) type_name = f.field_type + field_lines = [] + min_prefix = 0 if f.is_array and f.max_size is not None: type_sizes = {"uint8": 1, "int8": 1, "uint16": 2, "int16": 2, "uint32": 4, "int32": 4, "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1} if normalize_bytes_type(type_name) == "string": @@ -1032,81 +1047,101 @@ def _generate_variable_methods(msg, structName, renamed_enums=None): else: count_size = 2 if f.max_size > 255 else 1 element_size = type_sizes.get(type_name, (f.size - count_size) // f.max_size) - result += f' // {f.name}: variable array\n' + min_prefix = 2 if f.max_size > 255 else 1 + field_lines.append(f'// {f.name}: variable array') if f.max_size > 255: - result += f' if (offset + 2 > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} count");\n' - result += f' msg.{var_name}Count = Math.Min(BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset, 2)), (ushort){f.max_size});\n' - result += f' offset += 2;\n' + field_lines.append(f'if (offset + 2 > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} count");') + field_lines.append(f'msg.{var_name}Count = Math.Min(BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset, 2)), (ushort){f.max_size});') + field_lines.append(f'offset += 2;') else: - result += f' if (offset >= data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} count");\n' - result += f' msg.{var_name}Count = Math.Min(data[offset++], (byte){f.max_size});\n' + field_lines.append(f'if (offset >= data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} count");') + field_lines.append(f'msg.{var_name}Count = Math.Min(data[offset++], (byte){f.max_size});') if type_name in type_sizes: base_type = csharp_types.get(type_name, type_name) - result += f' msg.{var_name}Data = new {base_type}[{f.max_size}];\n' - result += f' if (offset + msg.{var_name}Count * {element_size} > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} array");\n' - result += f' MemoryMarshal.Cast(data.Slice(offset, msg.{var_name}Count * {element_size})).CopyTo(msg.{var_name}Data.AsSpan());\n' - result += f' offset += msg.{var_name}Count * {element_size};\n' + field_lines.append(f'msg.{var_name}Data = new {base_type}[{f.max_size}];') + field_lines.append(f'if (offset + msg.{var_name}Count * {element_size} > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} array");') + field_lines.append(f'MemoryMarshal.Cast(data.Slice(offset, msg.{var_name}Count * {element_size})).CopyTo(msg.{var_name}Data.AsSpan());') + field_lines.append(f'offset += msg.{var_name}Count * {element_size};') elif f.is_enum: - result += f' msg.{var_name}Data = new byte[{f.max_size}];\n' - result += f' if (offset + msg.{var_name}Count > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} array");\n' - result += f' data.Slice(offset, msg.{var_name}Count).CopyTo(msg.{var_name}Data.AsSpan());\n' - result += f' offset += msg.{var_name}Count;\n' + field_lines.append(f'msg.{var_name}Data = new byte[{f.max_size}];') + field_lines.append(f'if (offset + msg.{var_name}Count > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} array");') + field_lines.append(f'data.Slice(offset, msg.{var_name}Count).CopyTo(msg.{var_name}Data.AsSpan());') + field_lines.append(f'offset += msg.{var_name}Count;') else: type_pkg = f.type_package if f.type_package else f.package nested_type = type_name - result += f' msg.{var_name}Data = new {nested_type}[{f.max_size}];\n' - result += f' if (offset + msg.{var_name}Count * {element_size} > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} array");\n' - result += f' for (int i = 0; i < msg.{var_name}Count; i++)\n' - result += f' msg.{var_name}Data[i] = {nested_type}.Deserialize(data[(offset + i * {element_size})..(offset + (i + 1) * {element_size})]);\n' - result += f' offset += msg.{var_name}Count * {element_size};\n' + field_lines.append(f'msg.{var_name}Data = new {nested_type}[{f.max_size}];') + field_lines.append(f'if (offset + msg.{var_name}Count * {element_size} > data.Length) throw new System.IO.InvalidDataException("Truncated data reading {f.name} array");') + field_lines.append(f'for (int i = 0; i < msg.{var_name}Count; i++)') + field_lines.append(f' msg.{var_name}Data[i] = {nested_type}.Deserialize(data[(offset + i * {element_size})..(offset + (i + 1) * {element_size})]);') + field_lines.append(f'offset += msg.{var_name}Count * {element_size};') elif normalize_bytes_type(type_name) == "string" and f.max_size is not None: - result += f' // {f.name}: variable string\n' + min_prefix = 2 if f.max_size > 255 else 1 + field_lines.append(f'// {f.name}: variable string') if f.max_size > 255: - result += f' msg.{var_name}Length = Math.Min(BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset, 2)), (ushort){f.max_size});\n' - result += f' offset += 2;\n' + field_lines.append(f'msg.{var_name}Length = Math.Min(BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset, 2)), (ushort){f.max_size});') + field_lines.append(f'offset += 2;') else: - result += f' msg.{var_name}Length = Math.Min(data[offset++], (byte){f.max_size});\n' - result += f' msg.{var_name}Data = new byte[{f.max_size}];\n' - result += f' data.Slice(offset, msg.{var_name}Length).CopyTo(msg.{var_name}Data.AsSpan());\n' - result += f' offset += msg.{var_name}Length;\n' + field_lines.append(f'msg.{var_name}Length = Math.Min(data[offset++], (byte){f.max_size});') + field_lines.append(f'msg.{var_name}Data = new byte[{f.max_size}];') + field_lines.append(f'data.Slice(offset, msg.{var_name}Length).CopyTo(msg.{var_name}Data.AsSpan());') + field_lines.append(f'offset += msg.{var_name}Length;') else: # Fixed field if type_name in csharp_type_sizes: + min_prefix = csharp_type_sizes[type_name] if type_name == "uint8": - result += f' msg.{var_name} = data[offset++];\n' + field_lines.append(f'msg.{var_name} = data[offset++];') elif type_name == "int8": - result += f' msg.{var_name} = (sbyte)data[offset++];\n' + field_lines.append(f'msg.{var_name} = (sbyte)data[offset++];') elif type_name == "uint16": - result += f' msg.{var_name} = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset, 2)); offset += 2;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset, 2)); offset += 2;') elif type_name == "int16": - result += f' msg.{var_name} = BinaryPrimitives.ReadInt16LittleEndian(data.Slice(offset, 2)); offset += 2;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadInt16LittleEndian(data.Slice(offset, 2)); offset += 2;') elif type_name == "uint32": - result += f' msg.{var_name} = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(offset, 4)); offset += 4;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadUInt32LittleEndian(data.Slice(offset, 4)); offset += 4;') elif type_name == "int32": - result += f' msg.{var_name} = BinaryPrimitives.ReadInt32LittleEndian(data.Slice(offset, 4)); offset += 4;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadInt32LittleEndian(data.Slice(offset, 4)); offset += 4;') elif type_name == "uint64": - result += f' msg.{var_name} = BinaryPrimitives.ReadUInt64LittleEndian(data.Slice(offset, 8)); offset += 8;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadUInt64LittleEndian(data.Slice(offset, 8)); offset += 8;') elif type_name == "int64": - result += f' msg.{var_name} = BinaryPrimitives.ReadInt64LittleEndian(data.Slice(offset, 8)); offset += 8;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadInt64LittleEndian(data.Slice(offset, 8)); offset += 8;') elif type_name == "float": - result += f' msg.{var_name} = BinaryPrimitives.ReadSingleLittleEndian(data.Slice(offset, 4)); offset += 4;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadSingleLittleEndian(data.Slice(offset, 4)); offset += 4;') elif type_name == "double": - result += f' msg.{var_name} = BinaryPrimitives.ReadDoubleLittleEndian(data.Slice(offset, 8)); offset += 8;\n' + field_lines.append(f'msg.{var_name} = BinaryPrimitives.ReadDoubleLittleEndian(data.Slice(offset, 8)); offset += 8;') elif type_name == "bool": - result += f' msg.{var_name} = data[offset++] != 0;\n' + field_lines.append(f'msg.{var_name} = data[offset++] != 0;') elif normalize_bytes_type(type_name) == "string" and f.size_option is not None: # Fixed string - copy into byte array - result += f' msg.{var_name} = new byte[{f.size}];\n' - result += f' data.Slice(offset, {f.size}).CopyTo(msg.{var_name}.AsSpan());\n' - result += f' offset += {f.size};\n' + min_prefix = f.size + field_lines.append(f'msg.{var_name} = new byte[{f.size}];') + field_lines.append(f'data.Slice(offset, {f.size}).CopyTo(msg.{var_name}.AsSpan());') + field_lines.append(f'offset += {f.size};') elif f.is_enum: + min_prefix = 1 type_pkg = f.type_package if f.type_package else f.package enum_type = renamed_enums.get(type_name, type_name) if renamed_enums else type_name - result += f' msg.{var_name} = ({enum_type})data[offset++];\n' + field_lines.append(f'msg.{var_name} = ({enum_type})data[offset++];') else: type_pkg = f.type_package if f.type_package else f.package nested_type = type_name - result += f' msg.{var_name} = {nested_type}.Deserialize(data[offset..(offset + {nested_type}.MaxSize)]); offset += {nested_type}.MaxSize;\n' + min_prefix = f.size + field_lines.append(f'msg.{var_name} = {nested_type}.Deserialize(data[offset..(offset + {nested_type}.MaxSize)]); offset += {nested_type}.MaxSize;') + + if getattr(f, 'is_extension', False): + # Extension field: older senders may omit it entirely. Only attempt + # the read if enough bytes remain; otherwise leave it at its default + # (wire evolution, no caller padding needed). + result += f' // {f.name}: extension field, tolerate a short buffer\n' + result += f' if (offset + {min_prefix} <= data.Length)\n' + result += ' {\n' + for line in field_lines: + result += f' {line}\n' + result += ' }\n' + else: + for line in field_lines: + result += f' {line}\n' # Oneofs: read discriminator then union payload (or length-prefix + variant bytes for variable oneof) for oneof_name, oneof in msg.oneofs.items(): @@ -1988,7 +2023,9 @@ def _generate_field_init(field, prefix="msg", index=0): if field.size_option is not None: out += f' {prefix}.{var_name} = {TestCSharpGen._bytes_literal("test_string", field.size_option)};\n' elif field.max_size is not None: - test_str = "test_string" + # Clamp the test string to the field's max_size so length stays + # within capacity (length > max_size is rejected by decoders). + test_str = "test_string"[:field.max_size] out += f' {prefix}.{var_name}Length = {len(test_str)};\n' out += f' {prefix}.{var_name}Data = {TestCSharpGen._bytes_literal(test_str)};\n' else: diff --git a/src/struct_frame/generate.py b/src/struct_frame/generate.py index 8052cd4e..4ed8fbc9 100644 --- a/src/struct_frame/generate.py +++ b/src/struct_frame/generate.py @@ -1148,6 +1148,19 @@ def validate(self, current_package, packages, debug=False): # Extension semantics apply only to top-level message fields; oneof # fields are always treated as base. if self.extensions_start is not None: + # Extension fields must form the trailing bytes of the wire payload, + # but the wire order is all plain fields followed by all oneofs. A + # message-level extensions_start combined with a oneof would place + # extension bytes BEFORE the oneof, breaking the base-prefix CRC + # split and cross-version interop. Use `option extensions_start` + # inside the oneof instead to add extension variants. + if self.oneofs: + print( + f"Message {self.name}: message-level `option extensions_start` " + f"cannot be combined with oneof fields (extension fields would " + f"not be trailing on the wire). Declare `option extensions_start` " + f"inside the oneof to add extension variants instead.") + return False field_numbers = [f.number for f in self.fields.values()] if self.extensions_start not in field_numbers: print( @@ -1284,10 +1297,14 @@ def validate(self, current_package, packages, debug=False): self.magic_bytes = calculate_magic_numbers(self) # Calculate minimum size for variable messages - # min_size is the size when all variable-length fields are at their minimum + # min_size is the size when all variable-length fields are at their minimum. + # Extension fields contribute nothing: an older (extension-unaware) sender + # omits them entirely, so the smallest valid payload is base fields only. if self.variable: self.min_size = 0 for key, value in self.fields.items(): + if value.is_extension: + continue if value.is_array and value.max_size is not None: # Bounded array: only the count bytes (1 or 2, no data when empty) count_bytes = 2 if value.max_size > 255 else 1 diff --git a/src/struct_frame/js_gen.py b/src/struct_frame/js_gen.py index 3a89e596..50a079f4 100644 --- a/src/struct_frame/js_gen.py +++ b/src/struct_frame/js_gen.py @@ -648,6 +648,8 @@ def _generate_variable_methods(msg, package_msg_name): for key, field in msg.fields.items(): name = to_camel_case(field.name) field_type = field.field_type + field_lines = [] + min_prefix = 0 if field.is_array and field.max_size is not None: type_sizes = {"uint8": 1, "int8": 1, "uint16": 2, "int16": 2, "uint32": 4, "int32": 4, "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1} count_bytes = 2 if field.max_size > 255 else 1 @@ -655,32 +657,48 @@ def _generate_variable_methods(msg, package_msg_name): element_size = field.element_size if field.element_size else 1 else: element_size = type_sizes.get(field_type, (field.size - count_bytes) // field.max_size) - result += f' // {name}: variable array\n' + min_prefix = count_bytes + field_lines.append(f'// {name}: variable array') if count_bytes == 2: - result += f' const {name}Count = Math.min(buffer.readUInt16LE(offset), {field.max_size}); offset += 2;\n' - result += f' msg._buffer.writeUInt16LE({name}Count, {msg_offset});\n' + field_lines.append(f'const {name}Count = Math.min(buffer.readUInt16LE(offset), {field.max_size}); offset += 2;') + field_lines.append(f'msg._buffer.writeUInt16LE({name}Count, {msg_offset});') else: - result += f' const {name}Count = Math.min(buffer.readUInt8(offset++), {field.max_size});\n' - result += f' msg._buffer.writeUInt8({name}Count, {msg_offset});\n' - result += f' for (let i = 0; i < {name}Count; i++) {{\n' - result += f' buffer.copy(msg._buffer, {msg_offset + count_bytes} + i * {element_size}, offset, offset + {element_size});\n' - result += f' offset += {element_size};\n' - result += f' }}\n' + field_lines.append(f'const {name}Count = Math.min(buffer.readUInt8(offset++), {field.max_size});') + field_lines.append(f'msg._buffer.writeUInt8({name}Count, {msg_offset});') + field_lines.append(f'for (let i = 0; i < {name}Count; i++) {{') + field_lines.append(f' buffer.copy(msg._buffer, {msg_offset + count_bytes} + i * {element_size}, offset, offset + {element_size});') + field_lines.append(f' offset += {element_size};') + field_lines.append(f'}}') elif field_type in ("string", "bytes") and field.max_size is not None: length_bytes = 2 if field.max_size > 255 else 1 - result += f' // {name}: variable string\n' + min_prefix = length_bytes + field_lines.append(f'// {name}: variable string') if length_bytes == 2: - result += f' const {name}Len = Math.min(buffer.readUInt16LE(offset), {field.max_size}); offset += 2;\n' - result += f' msg._buffer.writeUInt16LE({name}Len, {msg_offset});\n' + field_lines.append(f'const {name}Len = Math.min(buffer.readUInt16LE(offset), {field.max_size}); offset += 2;') + field_lines.append(f'msg._buffer.writeUInt16LE({name}Len, {msg_offset});') else: - result += f' const {name}Len = Math.min(buffer.readUInt8(offset++), {field.max_size});\n' - result += f' msg._buffer.writeUInt8({name}Len, {msg_offset});\n' - result += f' buffer.copy(msg._buffer, {msg_offset + length_bytes}, offset, offset + {name}Len);\n' - result += f' offset += {name}Len;\n' + field_lines.append(f'const {name}Len = Math.min(buffer.readUInt8(offset++), {field.max_size});') + field_lines.append(f'msg._buffer.writeUInt8({name}Len, {msg_offset});') + field_lines.append(f'buffer.copy(msg._buffer, {msg_offset + length_bytes}, offset, offset + {name}Len);') + field_lines.append(f'offset += {name}Len;') else: - result += f' // {name}: fixed size ({field.size} bytes)\n' - result += f' buffer.copy(msg._buffer, {msg_offset}, offset, offset + {field.size});\n' - result += f' offset += {field.size};\n' + min_prefix = field.size + field_lines.append(f'// {name}: fixed size ({field.size} bytes)') + field_lines.append(f'buffer.copy(msg._buffer, {msg_offset}, offset, offset + {field.size});') + field_lines.append(f'offset += {field.size};') + + if getattr(field, 'is_extension', False): + # Extension field: older senders may omit it entirely. Only attempt + # the read if enough bytes remain; otherwise leave the field at its + # zero default in msg._buffer (wire evolution, no caller padding needed). + result += f' // {name}: extension field, tolerate a short buffer\n' + result += f' if (offset + {min_prefix} <= buffer.length) {{\n' + for line in field_lines: + result += f' {line}\n' + result += f' }}\n' + else: + for line in field_lines: + result += f' {line}\n' msg_offset += field.size # Oneofs: read discriminator bytes + union payload (or length-prefix + variant bytes for variable oneof) @@ -1130,7 +1148,9 @@ def _generate_field_init(field, prefix="msg", index=0): if field.size_option is not None: result += f' {prefix}.{var_name} = "test_string";\n' elif field.max_size is not None: - test_str = "test_string" + # Clamp the test string to the field's max_size so length stays + # within capacity (length > max_size is rejected by decoders). + test_str = "test_string"[:field.max_size] result += f' {prefix}.{var_name}Length = {len(test_str)};\n' result += f' {prefix}.{var_name}Data = "{test_str}";\n' else: diff --git a/src/struct_frame/py_gen.py b/src/struct_frame/py_gen.py index d392f829..a5b88c83 100644 --- a/src/struct_frame/py_gen.py +++ b/src/struct_frame/py_gen.py @@ -426,7 +426,15 @@ def generate_unpack_method(msg): """Generate the _deserialize_fixed() class method""" result = '\n @classmethod\n' result += ' def _deserialize_fixed(cls, data: bytes):\n' - result += ' """Deserialize binary data into a message instance (fixed-size format)"""\n' + result += ' """Deserialize binary data into a message instance (fixed-size format).\n' + result += '\n' + result += ' Wire evolution: a buffer shorter than MAX_SIZE (older sender, base fields\n' + result += ' only) is zero-filled for the missing extension fields; a buffer longer\n' + result += ' than MAX_SIZE (newer sender) has its trailing extension bytes ignored.\n' + result += ' Callers never need to pad or truncate the buffer themselves.\n' + result += ' """\n' + result += ' if len(data) < cls.MAX_SIZE:\n' + result += ' data = data + b"\\x00" * (cls.MAX_SIZE - len(data))\n' result += ' offset = 0\n' result += ' fields = {}\n' @@ -1014,108 +1022,131 @@ def generate_variable_methods(msg): result += ' fields = {}\n' for key, f in msg.fields.items(): + field_lines = [] + min_prefix = 0 if f.is_array and f.max_size is not None: # Variable array type_sizes = {"uint8": 1, "int8": 1, "uint16": 2, "int16": 2, "uint32": 4, "int32": 4, "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1} # Count prefix is 2 bytes when max_size > 255, else 1 (must match the writer) count_fmt = "H" if f.max_size > 255 else "B" count_size = 2 if f.max_size > 255 else 1 + min_prefix = count_size if f.field_type in ("string", "bytes"): element_size = f.element_size if f.element_size else 1 - result += f' # {f.name}: variable string array\n' - result += f' count = struct.unpack_from("<{count_fmt}", data, offset)[0]\n' - result += f' offset += {count_size}\n' - result += f' fields["{f.name}"] = []\n' - result += f' for i in range(min(count, {f.max_size})):\n' - result += f' s = struct.unpack_from("<{element_size}s", data, offset)[0]\n' - result += f' fields["{f.name}"].append(s)\n' - result += f' offset += {element_size}\n' + field_lines.append(f'# {f.name}: variable string array') + field_lines.append(f'count = struct.unpack_from("<{count_fmt}", data, offset)[0]') + field_lines.append(f'offset += {count_size}') + field_lines.append(f'fields["{f.name}"] = []') + field_lines.append(f'for i in range(min(count, {f.max_size})):') + field_lines.append(f' s = struct.unpack_from("<{element_size}s", data, offset)[0]') + field_lines.append(f' fields["{f.name}"].append(s)') + field_lines.append(f' offset += {element_size}') elif f.is_enum: - result += f' # {f.name}: variable enum array\n' - result += f' count = struct.unpack_from("<{count_fmt}", data, offset)[0]\n' - result += f' offset += {count_size}\n' - result += f' _n = min(count, {f.max_size})\n' - result += f' fields["{f.name}"] = list(struct.unpack_from("<%dB" % _n, data, offset))\n' - result += f' offset += _n\n' + field_lines.append(f'# {f.name}: variable enum array') + field_lines.append(f'count = struct.unpack_from("<{count_fmt}", data, offset)[0]') + field_lines.append(f'offset += {count_size}') + field_lines.append(f'_n = min(count, {f.max_size})') + field_lines.append(f'fields["{f.name}"] = list(struct.unpack_from("<%dB" % _n, data, offset))') + field_lines.append(f'offset += _n') elif f.field_type in type_sizes: element_size = type_sizes[f.field_type] fmt = py_struct_format.get(f.field_type, 'B') - result += f' # {f.name}: variable {f.field_type} array\n' - result += f' count = struct.unpack_from("<{count_fmt}", data, offset)[0]\n' - result += f' offset += {count_size}\n' - result += f' _n = min(count, {f.max_size})\n' - result += f' fields["{f.name}"] = list(struct.unpack_from("<%d{fmt}" % _n, data, offset))\n' - result += f' offset += _n * {element_size}\n' + field_lines.append(f'# {f.name}: variable {f.field_type} array') + field_lines.append(f'count = struct.unpack_from("<{count_fmt}", data, offset)[0]') + field_lines.append(f'offset += {count_size}') + field_lines.append(f'_n = min(count, {f.max_size})') + field_lines.append(f'fields["{f.name}"] = list(struct.unpack_from("<%d{fmt}" % _n, data, offset))') + field_lines.append(f'offset += _n * {element_size}') else: # Nested message array type_name = f.field_type element_size = (f.size - 1) // f.max_size - result += f' # {f.name}: variable nested message array\n' - result += f' count = struct.unpack_from("<{count_fmt}", data, offset)[0]\n' - result += f' offset += {count_size}\n' - result += f' fields["{f.name}"] = []\n' - result += f' for i in range(min(count, {f.max_size})):\n' - result += f' msg = {type_name}._deserialize_fixed(data[offset:offset+{type_name}.MAX_SIZE])\n' - result += f' fields["{f.name}"].append(msg)\n' - result += f' offset += {type_name}.MAX_SIZE\n' + field_lines.append(f'# {f.name}: variable nested message array') + field_lines.append(f'count = struct.unpack_from("<{count_fmt}", data, offset)[0]') + field_lines.append(f'offset += {count_size}') + field_lines.append(f'fields["{f.name}"] = []') + field_lines.append(f'for i in range(min(count, {f.max_size})):') + field_lines.append(f' msg = {type_name}._deserialize_fixed(data[offset:offset+{type_name}.MAX_SIZE])') + field_lines.append(f' fields["{f.name}"].append(msg)') + field_lines.append(f' offset += {type_name}.MAX_SIZE') elif f.field_type in ("string", "bytes") and f.max_size is not None: # Variable string count_fmt = "H" if f.max_size > 255 else "B" count_size = 2 if f.max_size > 255 else 1 - result += f' # {f.name}: variable string\n' - result += f' str_len = struct.unpack_from("<{count_fmt}", data, offset)[0]\n' - result += f' offset += {count_size}\n' - result += f' str_len = min(str_len, {f.max_size})\n' - result += f' fields["{f.name}"] = data[offset:offset+str_len]\n' - result += f' offset += str_len\n' + min_prefix = count_size + field_lines.append(f'# {f.name}: variable string') + field_lines.append(f'str_len = struct.unpack_from("<{count_fmt}", data, offset)[0]') + field_lines.append(f'offset += {count_size}') + field_lines.append(f'str_len = min(str_len, {f.max_size})') + field_lines.append(f'fields["{f.name}"] = data[offset:offset+str_len]') + field_lines.append(f'offset += str_len') elif f.field_type in ("string", "bytes") and f.size_option is not None: # Fixed string - result += f' # {f.name}: fixed string\n' - result += f' fields["{f.name}"] = struct.unpack_from("<{f.size_option}s", data, offset)[0]\n' - result += f' offset += {f.size_option}\n' + min_prefix = f.size_option + field_lines.append(f'# {f.name}: fixed string') + field_lines.append(f'fields["{f.name}"] = struct.unpack_from("<{f.size_option}s", data, offset)[0]') + field_lines.append(f'offset += {f.size_option}') elif f.is_array and f.size_option is not None: # Fixed array if f.is_enum: - result += f' # {f.name}: fixed enum array\n' - result += f' fields["{f.name}"] = []\n' - result += f' for i in range({f.size_option}):\n' - result += f' val = struct.unpack_from(" 0 and getattr(field, 'is_extension', False) and not emitted_ext_pad: + result += f'{indent}let mut _ext_padded = [0u8; {ext_width}];\n' + result += f'{indent}let _ext_n = buf.len().saturating_sub(_pos).min({ext_width});\n' + result += f'{indent}_ext_padded[.._ext_n].copy_from_slice(&buf[_pos.._pos + _ext_n]);\n' + result += f'{indent}let buf = &_ext_padded[..];\n' + result += f'{indent}let mut _pos = 0usize;\n' + emitted_ext_pad = True unpack_code = _generate_unpack_field(field, variable=variable_mode) if unpack_code: result += unpack_code + '\n' @@ -964,10 +982,14 @@ def _build_unpack_body(result, variable_mode, indent=' '): result += f'{indent}}})\n' return result - # MIN_SIZE constant for variable messages + # MIN_SIZE constant for variable messages. Extension fields + # contribute nothing: an older sender omits them entirely, so the + # smallest valid payload is base fields only (matches msg.min_size). result += '\n /// Calculate minimum serialized size (all variable fields empty).\n' min_size = 0 for field in msg.fields.values(): + if getattr(field, 'is_extension', False): + continue if field.is_array: if field.max_size is not None: min_size += 2 if field.max_size > 255 else 1 # just the count @@ -1011,8 +1033,16 @@ def _build_unpack_body(result, variable_mode, indent=' '): result = _build_unpack_body(result, variable_mode=True, indent=' ') result += ' }\n' else: - # Fixed message: simple unpack - result += f' if buf.len() < Self::SIZE {{ return None; }}\n' + # Fixed message: simple unpack. + # Wire evolution: zero-fill any bytes an older sender omitted (a shorter + # buffer); ignore any trailing bytes a newer sender appended (a longer + # buffer). Padding into a fixed Self::SIZE-length array up front means + # every field read below always has the bytes it expects, so callers + # never need to pad or truncate the buffer themselves. + result += ' let mut _padded = [0u8; Self::SIZE];\n' + result += ' let _n = buf.len().min(Self::SIZE);\n' + result += ' _padded[.._n].copy_from_slice(&buf[.._n]);\n' + result += ' let buf = &_padded[..];\n' result += ' let mut _pos = 0usize;\n' result += ' #[allow(unused_variables)]\n' result += ' let _ = _pos;\n' diff --git a/src/struct_frame/ts_gen.py b/src/struct_frame/ts_gen.py index c75c41ef..7f140cc9 100644 --- a/src/struct_frame/ts_gen.py +++ b/src/struct_frame/ts_gen.py @@ -684,6 +684,8 @@ def _generate_variable_methods(msg, package_msg_name, fields): for key, field in msg.fields.items(): name = to_camel_case(field.name) field_type = field.field_type + field_lines = [] + min_prefix = 0 if field.is_array and field.max_size is not None: # Variable array type_sizes = {"uint8": 1, "int8": 1, "uint16": 2, "int16": 2, "uint32": 4, "int32": 4, "uint64": 8, "int64": 8, "float": 4, "double": 8, "bool": 1} @@ -693,66 +695,60 @@ def _generate_variable_methods(msg, package_msg_name, fields): else: element_size = type_sizes.get(field_type, (field.size - count_bytes) // field.max_size) max_len = field.max_size - result += f' // {name}: variable array\n' + min_prefix = count_bytes + field_lines.append(f'// {name}: variable array') if count_bytes == 2: - result += f' const {name}Count = Math.min(buffer.readUInt16LE(offset), {max_len}); offset += 2;\n' + field_lines.append(f'const {name}Count = Math.min(buffer.readUInt16LE(offset), {max_len}); offset += 2;') else: - result += f' const {name}Count = Math.min(buffer.readUInt8(offset++), {max_len});\n' + field_lines.append(f'const {name}Count = Math.min(buffer.readUInt8(offset++), {max_len});') - if field_type not in type_sizes and field_type != "string" and not field.is_enum: - # Nested struct array - need to set the internal buffer array elements - nested_type = '%s%s' % (pascal_case(field.package), field_type) - result += f' // Write count to internal buffer\n' - if count_bytes == 2: - result += f' msg._buffer.writeUInt16LE({name}Count, {msg_offset});\n' - else: - result += f' msg._buffer.writeUInt8({name}Count, {msg_offset});\n' - result += f' for (let i = 0; i < {name}Count; i++) {{\n' - result += f' buffer.copy(msg._buffer, {msg_offset + count_bytes} + i * {element_size}, offset, offset + {element_size});\n' - result += f' offset += {element_size};\n' - result += f' }}\n' - elif field_type in ("string", "bytes"): - result += f' // Write count to internal buffer\n' - if count_bytes == 2: - result += f' msg._buffer.writeUInt16LE({name}Count, {msg_offset});\n' - else: - result += f' msg._buffer.writeUInt8({name}Count, {msg_offset});\n' - result += f' for (let i = 0; i < {name}Count; i++) {{\n' - result += f' buffer.copy(msg._buffer, {msg_offset + count_bytes} + i * {element_size}, offset, offset + {element_size});\n' - result += f' offset += {element_size};\n' - result += f' }}\n' + # Count write + element copy loop is identical for nested + # struct, string, and primitive arrays. + field_lines.append(f'// Write count to internal buffer') + if count_bytes == 2: + field_lines.append(f'msg._buffer.writeUInt16LE({name}Count, {msg_offset});') else: - # Primitive array - result += f' // Write count to internal buffer\n' - if count_bytes == 2: - result += f' msg._buffer.writeUInt16LE({name}Count, {msg_offset});\n' - else: - result += f' msg._buffer.writeUInt8({name}Count, {msg_offset});\n' - result += f' for (let i = 0; i < {name}Count; i++) {{\n' - result += f' buffer.copy(msg._buffer, {msg_offset + count_bytes} + i * {element_size}, offset, offset + {element_size});\n' - result += f' offset += {element_size};\n' - result += f' }}\n' + field_lines.append(f'msg._buffer.writeUInt8({name}Count, {msg_offset});') + field_lines.append(f'for (let i = 0; i < {name}Count; i++) {{') + field_lines.append(f' buffer.copy(msg._buffer, {msg_offset + count_bytes} + i * {element_size}, offset, offset + {element_size});') + field_lines.append(f' offset += {element_size};') + field_lines.append(f'}}') elif field_type in ("string", "bytes") and field.max_size is not None: # Variable string max_len = field.max_size length_bytes = 2 if field.max_size > 255 else 1 - result += f' // {name}: variable string\n' + min_prefix = length_bytes + field_lines.append(f'// {name}: variable string') if length_bytes == 2: - result += f' const {name}Len = Math.min(buffer.readUInt16LE(offset), {max_len}); offset += 2;\n' + field_lines.append(f'const {name}Len = Math.min(buffer.readUInt16LE(offset), {max_len}); offset += 2;') else: - result += f' const {name}Len = Math.min(buffer.readUInt8(offset++), {max_len});\n' - result += f' // Write length to internal buffer\n' + field_lines.append(f'const {name}Len = Math.min(buffer.readUInt8(offset++), {max_len});') + field_lines.append(f'// Write length to internal buffer') if length_bytes == 2: - result += f' msg._buffer.writeUInt16LE({name}Len, {msg_offset});\n' + field_lines.append(f'msg._buffer.writeUInt16LE({name}Len, {msg_offset});') else: - result += f' msg._buffer.writeUInt8({name}Len, {msg_offset});\n' - result += f' buffer.copy(msg._buffer, {msg_offset + length_bytes}, offset, offset + {name}Len);\n' - result += f' offset += {name}Len;\n' + field_lines.append(f'msg._buffer.writeUInt8({name}Len, {msg_offset});') + field_lines.append(f'buffer.copy(msg._buffer, {msg_offset + length_bytes}, offset, offset + {name}Len);') + field_lines.append(f'offset += {name}Len;') else: # Fixed field - result += f' // {name}: fixed size ({field.size} bytes)\n' - result += f' buffer.copy(msg._buffer, {msg_offset}, offset, offset + {field.size});\n' - result += f' offset += {field.size};\n' + min_prefix = field.size + field_lines.append(f'// {name}: fixed size ({field.size} bytes)') + field_lines.append(f'buffer.copy(msg._buffer, {msg_offset}, offset, offset + {field.size});') + field_lines.append(f'offset += {field.size};') + + if getattr(field, 'is_extension', False): + # Extension field: older senders may omit it entirely. Only attempt + # the read if enough bytes remain; otherwise leave the field at its + # zero default in msg._buffer (wire evolution, no caller padding needed). + result += f' // {name}: extension field, tolerate a short buffer\n' + result += f' if (offset + {min_prefix} <= buffer.length) {{\n' + for line in field_lines: + result += f' {line}\n' + result += f' }}\n' + else: + for line in field_lines: + result += f' {line}\n' msg_offset += field.size # Oneofs: read discriminator bytes + union payload (or length-prefix + variant bytes for variable oneof) @@ -1265,7 +1261,9 @@ def _generate_field_init(field, prefix="msg", index=0): if field.size_option is not None: result += f' {prefix}.{var_name} = "test_string";\n' elif field.max_size is not None: - test_str = "test_string" + # Clamp the test string to the field's max_size so length stays + # within capacity (length > max_size is rejected by decoders). + test_str = "test_string"[:field.max_size] result += f' {prefix}.{var_name}Length = {len(test_str)};\n' result += f' {prefix}.{var_name}Data = "{test_str}";\n' else: diff --git a/tests/c/test_wire_evolution_interop.c b/tests/c/test_wire_evolution_interop.c index 797c51f2..0c8801d4 100644 --- a/tests/c/test_wire_evolution_interop.c +++ b/tests/c/test_wire_evolution_interop.c @@ -120,12 +120,9 @@ static void scenario_2(void) { check(r.valid, "[S2] v1->v2 base-only frame validates CRC"); if (r.valid && r.msg_data) { - /* Newer receiver zero-fills the missing extension bytes before decoding. */ - uint8_t padded[WIRE_EVOLUTION_V2_BASE_EXTENSION_MESSAGE_MAX_SIZE] = {0}; - size_t copy_len = r.msg_len < sizeof(padded) ? r.msg_len : sizeof(padded); - memcpy(padded, r.msg_data, copy_len); + /* deserialize() zero-fills the missing extension bytes internally. */ WireEvolutionV2BaseExtensionMessage d = {0}; - WireEvolutionV2BaseExtensionMessage_deserialize(padded, sizeof(padded), &d); + WireEvolutionV2BaseExtensionMessage_deserialize(r.msg_data, r.msg_len, &d); check(d.header == 0x1234 && d.seq == 7, "[S2] v2 decodes base fields correctly"); check(d.crc_seed == 0, "[S2] v2 extension field zero-filled to default"); } @@ -217,11 +214,8 @@ static void scenario_5(void) { frame_msg_info_t r = parse_with(&PROFILE_STANDARD_CONFIG, buffer, fs, wire_evolution_v2_get_message_info); check(r.valid, "[S5] v1 base-variant -> v2 frame validates CRC"); if (r.valid && r.msg_data) { - uint8_t padded[WIRE_EVOLUTION_V2_ONE_OF_EXTENSION_MESSAGE_MAX_SIZE] = {0}; - size_t copy_len = r.msg_len < sizeof(padded) ? r.msg_len : sizeof(padded); - memcpy(padded, r.msg_data, copy_len); WireEvolutionV2OneOfExtensionMessage d = {0}; - WireEvolutionV2OneOfExtensionMessage_deserialize(padded, sizeof(padded), &d); + WireEvolutionV2OneOfExtensionMessage_deserialize(r.msg_data, r.msg_len, &d); check(d.command_discriminator == WIRE_EVOLUTION_V2_ONE_OF_EXTENSION_MESSAGE_COMMAND_FIELD_CMD_B && d.command.cmd_b.value_b == -1234, "[S5] v2 decodes the older base oneof variant correctly"); @@ -418,17 +412,16 @@ static void scenario_10(void) { wire_evolution_v2_get_message_info); check(r2.valid, "[S10] v1 variable (base-only) -> v2 validates CRC"); if (r2.valid && r2.msg_data) { - uint8_t padded[WIRE_EVOLUTION_V2_VARIABLE_EXTENSION_MESSAGE_MAX_SIZE] = {0}; - size_t copy_len = r2.msg_len < sizeof(padded) ? r2.msg_len : sizeof(padded); - memcpy(padded, r2.msg_data, copy_len); WireEvolutionV2VariableExtensionMessage d2 = {0}; - WireEvolutionV2VariableExtensionMessage_deserialize(padded, sizeof(padded), &d2); + WireEvolutionV2VariableExtensionMessage_deserialize(r2.msg_data, r2.msg_len, &d2); check(d2.node_id == 9 && d2.readings.count == 2 && d2.readings.data[0] == 1 && d2.readings.data[1] == 2, "[S10] v2 locates variable base after cross-version decode"); check(d2.ext_timestamp == 0, "[S10] v2 zero-fills the trailing extension field"); + check(d2.ext_note.length == 0, + "[S10] v2 zero-fills the count-prefixed (string) extension field"); } } diff --git a/tests/cpp/test_wire_evolution_interop.cpp b/tests/cpp/test_wire_evolution_interop.cpp index b3ac9d7d..b83ce64f 100644 --- a/tests/cpp/test_wire_evolution_interop.cpp +++ b/tests/cpp/test_wire_evolution_interop.cpp @@ -51,17 +51,6 @@ static void check(bool cond, const char* msg) { else { printf(" [FAIL] %s\n", msg); ++g_fail; } } -// Copy a (possibly shorter, base-only) payload into a MAX_SIZE zero buffer so a -// newer receiver can zero-fill the missing extension bytes before decoding. -template -static void decode_zerofill(Msg& out, const uint8_t* data, size_t len) { - uint8_t padded[Msg::MAX_SIZE]; - std::memset(padded, 0, sizeof(padded)); - if (len > Msg::MAX_SIZE) len = Msg::MAX_SIZE; - std::memcpy(padded, data, len); - out.deserialize(padded, sizeof(padded)); -} - // --------------------------------------------------------------------------- // Scenario 1: newer sender -> older receiver over length-bearing profiles // --------------------------------------------------------------------------- @@ -108,7 +97,7 @@ static void scenario_2() { if (result.valid && result.msg_data) { v2::BaseExtensionMessage d{}; - decode_zerofill(d, result.msg_data, result.msg_len); + d.deserialize(result.msg_data, result.msg_len); check(d.header == 0x1234 && d.seq == 7, "[S2] v2 decodes base fields correctly"); check(d.crc_seed == 0, "[S2] v2 extension field zero-filled to default"); } @@ -186,7 +175,7 @@ static void scenario_5() { if (result.valid && result.msg_data) { v2::OneOfExtensionMessage d{}; - decode_zerofill(d, result.msg_data, result.msg_len); + d.deserialize(result.msg_data, result.msg_len); check(d.command_discriminator == v2::OneOfExtensionMessage::CommandField::CMD_B && d.command.cmd_b.value_b == -1234, "[S5] v2 decodes the older base oneof variant correctly"); @@ -332,7 +321,9 @@ static void scenario_10() { check(result2.valid, "[S10] v1 variable (base-only) -> v2 validates CRC"); if (result2.valid && result2.msg_data) { v2::VariableExtensionMessage d2{}; - decode_zerofill(d2, result2.msg_data, result2.msg_len); + d2.deserialize(result2.msg_data, result2.msg_len); + check(d2.ext_note.length == 0, + "[S10] v2 zero-fills the count-prefixed (string) extension field"); check(d2.node_id == 9 && d2.readings.count == 2 && d2.readings.data[0] == 1 diff --git a/tests/csharp/test_wire_evolution_interop.cs b/tests/csharp/test_wire_evolution_interop.cs index a84a6ec5..2719d4d1 100644 --- a/tests/csharp/test_wire_evolution_interop.cs +++ b/tests/csharp/test_wire_evolution_interop.cs @@ -51,17 +51,6 @@ private static void Check(bool condition, string testName) } } - // Copy a (possibly shorter, base-only) payload into a MaxSize zero buffer so - // a newer receiver can zero-fill the missing extension bytes before decoding. - private static byte[] PadPayload(FrameMsgInfo info, int maxSize) - { - var padded = new byte[maxSize]; - var span = info.GetPayloadSpan(); - int n = Math.Min(span.Length, maxSize); - span.Slice(0, n).CopyTo(padded.AsSpan()); - return padded; - } - // ------------------------------------------------------------------------- // Scenario 1: newer sender -> older receiver over length-bearing profiles // ------------------------------------------------------------------------- @@ -112,7 +101,7 @@ private static void Scenario2() Check(info.Valid, "[S2] v1->v2 base-only frame validates CRC"); if (info.Valid && info.MsgData != null) { - var d = V2.BaseExtensionMessage.Deserialize(PadPayload(info, V2.BaseExtensionMessage.MaxSize)); + var d = V2.BaseExtensionMessage.Deserialize(info); Check(d.Header == 0x1234 && d.Seq == 7, "[S2] v2 decodes base fields correctly"); Check(d.CrcSeed == 0, "[S2] v2 extension field zero-filled to default"); } @@ -194,7 +183,7 @@ private static void Scenario5() Check(info.Valid, "[S5] v1 base-variant -> v2 frame validates CRC"); if (info.Valid && info.MsgData != null) { - var d = V2.OneOfExtensionMessage.Deserialize(PadPayload(info, V2.OneOfExtensionMessage.MaxSize)); + var d = V2.OneOfExtensionMessage.Deserialize(info); Check(d.CommandDiscriminator == V2.OneOfExtensionMessageCommandField.CmdB && d.CmdB != null && d.CmdB.ValueB == -1234, "[S5] v2 decodes the older base oneof variant correctly"); @@ -344,8 +333,7 @@ private static void Scenario10() Check(info2.Valid, "[S10] v1 variable (base-only) -> v2 validates CRC"); if (info2.Valid && info2.MsgData != null) { - var d2 = V2.VariableExtensionMessage.Deserialize( - PadPayload(info2, V2.VariableExtensionMessage.MaxSize)); + var d2 = V2.VariableExtensionMessage.Deserialize(info2); Check(d2.NodeId == 9 && d2.ReadingsCount == 2 && d2.ReadingsData != null @@ -353,6 +341,8 @@ private static void Scenario10() && d2.ReadingsData[1] == 2, "[S10] v2 locates variable base after cross-version decode"); Check(d2.ExtTimestamp == 0, "[S10] v2 zero-fills the trailing extension field"); + Check(d2.ExtNoteLength == 0, + "[S10] v2 zero-fills the count-prefixed (string) extension field"); } } diff --git a/tests/js/test_wire_evolution_interop.js b/tests/js/test_wire_evolution_interop.js index f7820792..dd848e72 100644 --- a/tests/js/test_wire_evolution_interop.js +++ b/tests/js/test_wire_evolution_interop.js @@ -324,6 +324,8 @@ function scenario10() { && d2.readingsData[1] === 2, '[S10] v2 locates variable base after cross-version decode'); check(d2.extTimestamp === 0, '[S10] v2 zero-fills the trailing extension field'); + check(d2.extNoteLength === 0, + '[S10] v2 zero-fills the count-prefixed (string) extension field'); } } diff --git a/tests/proto/wire_evolution_v2.sf b/tests/proto/wire_evolution_v2.sf index b99225d4..0d2fb3ec 100644 --- a/tests/proto/wire_evolution_v2.sf +++ b/tests/proto/wire_evolution_v2.sf @@ -54,7 +54,10 @@ message BaseExtensionMessage { } // ============================================================================ -// Scenario 2 — variable-length message with top-level extension field +// Scenario 2 — variable-length message with top-level extension fields. +// ext_note is deliberately a count-prefixed (variable string) extension AFTER +// a fixed extension: a base-only v1 frame forces receivers to skip multiple +// extension fields, including one whose length prefix is entirely absent. // ============================================================================ message VariableExtensionMessage { option msgid = 2; @@ -65,6 +68,7 @@ message VariableExtensionMessage { option extensions_start = 3; uint32 ext_timestamp = 3; + string ext_note = 4 [max_size=8]; } // ============================================================================ diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index 6309fb7e..8f703a27 100644 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -1221,10 +1221,8 @@ fn run_wire_evolution_interop_tests() -> ! { "[S2] v1->v2 base-only frame validates CRC" ); if let Some(f) = frame { - // Newer receiver zero-fills the missing extension bytes before decoding. - let mut padded = vec![0u8; v2::BaseExtensionMessage::MAX_SIZE]; - padded[..f.msg_data.len()].copy_from_slice(&f.msg_data); - let d = v2::BaseExtensionMessage::unpack(&padded); + // unpack() zero-fills the missing extension bytes internally. + let d = v2::BaseExtensionMessage::unpack(&f.msg_data); check!( d.as_ref().map_or(false, |x| x.header == 0x1234 && x.seq == 7), "[S2] v2 decodes base fields correctly" @@ -1306,9 +1304,7 @@ fn run_wire_evolution_interop_tests() -> ! { "[S5] v1 base-variant -> v2 frame validates CRC" ); if let Some(f) = frame { - let mut padded = vec![0u8; v2::OneOfExtensionMessage::MAX_SIZE]; - padded[..f.msg_data.len()].copy_from_slice(&f.msg_data); - let d = v2::OneOfExtensionMessage::unpack(&padded); + let d = v2::OneOfExtensionMessage::unpack(&f.msg_data); check!( d.map_or(false, |x| x.command_discriminator == 2 && x.get_cmd_b().map_or(false, |cb| cb.value_b == -1234)), @@ -1458,11 +1454,8 @@ fn run_wire_evolution_interop_tests() -> ! { "[S10] v1 variable (base-only) -> v2 validates CRC" ); if let Some(f2) = frame2 { - // Pad payload to MAX_SIZE so unpack() uses fixed-size read (zero-fills ext). - let mut padded = vec![0u8; v2::VariableExtensionMessage::MAX_SIZE]; - let copy_len = f2.msg_data.len().min(padded.len()); - padded[..copy_len].copy_from_slice(&f2.msg_data[..copy_len]); - let d2 = v2::VariableExtensionMessage::unpack(&padded); + // unpack() zero-fills the missing trailing extension field internally. + let d2 = v2::VariableExtensionMessage::unpack(&f2.msg_data); check!( d2.as_ref().map_or(false, |x| x.node_id == 9 @@ -1472,9 +1465,13 @@ fn run_wire_evolution_interop_tests() -> ! { "[S10] v2 locates variable base after cross-version decode" ); check!( - d2.map_or(false, |x| x.ext_timestamp == 0), + d2.as_ref().map_or(false, |x| x.ext_timestamp == 0), "[S10] v2 zero-fills the trailing extension field" ); + check!( + d2.map_or(false, |x| x.ext_note_length == 0), + "[S10] v2 zero-fills the count-prefixed (string) extension field" + ); } } diff --git a/tests/test_generator_validation.py b/tests/test_generator_validation.py index 3e4055fa..6aeeacd6 100644 --- a/tests/test_generator_validation.py +++ b/tests/test_generator_validation.py @@ -388,3 +388,71 @@ def test_field_number_zero() -> None: result = _run(str(sf)) _report("field_number_zero", result, expected_reject=True, expected_msg="must be numbered 1..N") + + +# --------------------------------------------------------------------------- +# Message-level extensions_start combined with a oneof must be rejected: +# wire order is fields-then-oneofs, so extension field bytes would sit BEFORE +# the oneof and not form the trailing region the base-prefix CRC split needs. +# --------------------------------------------------------------------------- + +def test_message_extensions_with_oneof_rejected() -> None: + """Message-level extensions_start + oneof breaks the trailing-extension wire model.""" + proto = """\ +package ext_oneof_test; + +message SubA { + uint8 a = 1; +} + +message Foo { + option msgid = 1; + uint8 head = 1; + + option extensions_start = 2; + uint32 ext_field = 2; + + oneof payload { + option discriminator = "field_order"; + SubA sub_a = 1; + } +} +""" + with tempfile.TemporaryDirectory() as tmp: + sf = Path(tmp) / "ext_oneof.sf" + sf.write_text(proto) + result = _run(str(sf)) + _report("message_extensions_with_oneof", result, expected_reject=True, + expected_msg="cannot be combined with oneof") + + +def test_oneof_level_extensions_still_allowed() -> None: + """extensions_start INSIDE a oneof (extension variants) must remain valid.""" + proto = """\ +package ext_oneof_ok_test; + +message SubA { + uint8 a = 1; +} + +message SubB { + uint16 b = 1; +} + +message Foo { + option msgid = 1; + uint8 head = 1; + + oneof payload { + option discriminator = "field_order"; + SubA sub_a = 1; + option extensions_start = 2; + SubB sub_b = 2; + } +} +""" + with tempfile.TemporaryDirectory() as tmp: + sf = Path(tmp) / "ext_oneof_ok.sf" + sf.write_text(proto) + result = _run(str(sf)) + _report("oneof_level_extensions_allowed", result, expected_reject=False) diff --git a/tests/test_wire_evolution_interop.py b/tests/test_wire_evolution_interop.py index 2162056b..877c4527 100644 --- a/tests/test_wire_evolution_interop.py +++ b/tests/test_wire_evolution_interop.py @@ -100,14 +100,6 @@ def header_size(self, profile: str) -> int: return self._configs[profile].header_size -def _decode_zerofill(cls, info): - """Decode a frame payload, zero-filling any extension bytes the sender omitted.""" - data = info.msg_data - if len(data) < cls.MAX_SIZE: - data = data + b"\x00" * (cls.MAX_SIZE - len(data)) - return cls.deserialize(data) - - # --------------------------------------------------------------------------- # Scenarios # --------------------------------------------------------------------------- @@ -134,7 +126,7 @@ def scenario_2_older_to_newer(v1, v2, fr: Framing) -> None: _check(info.valid, "[S2] v1->v2 base-only frame validates CRC") _check(info.msg_len == v1.BaseExtensionMessage.MAX_SIZE, "[S2] frame length carries shorter base-only payload") - decoded = _decode_zerofill(v2.BaseExtensionMessage, info) + decoded = v2.BaseExtensionMessage.deserialize(info) _check(decoded.header == 0x1234 and decoded.seq == 7, "[S2] v2 decodes base fields correctly") _check(decoded.crc_seed == 0, "[S2] v2 extension field zero-filled to default") @@ -178,7 +170,7 @@ def scenario_5_older_base_variant_to_newer(v1, v2, fr: Framing) -> None: buf = fr.encode("standard", orig) info = fr.parse("standard", buf, v2.get_message_info) _check(info.valid, "[S5] v1 base-variant -> v2 frame validates CRC") - decoded = _decode_zerofill(v2.OneOfExtensionMessage, info) + decoded = v2.OneOfExtensionMessage.deserialize(info) _check(decoded.command_which == "cmd_b" and decoded.command["cmd_b"].value_b == -1234, "[S5] v2 decodes the base oneof variant correctly") @@ -215,7 +207,7 @@ def scenario_6_multi_oneof(v1, v2, fr: Framing) -> None: buf2 = fr.encode("standard", orig2) info2 = fr.parse("standard", buf2, v2.get_message_info) _check(info2.valid, "[S6] v1 multi-oneof (base in 2nd union) -> v2 validates CRC") - decoded2 = _decode_zerofill(v2.MultiOneOfExtensionMessage, info2) + decoded2 = v2.MultiOneOfExtensionMessage.deserialize(info2) _check(decoded2.ext_union_which == "second_a" and decoded2.ext_union["second_a"].value_a == 77, "[S6] v2 decodes the older base variant in the ext oneof correctly") @@ -272,11 +264,13 @@ def scenario_10_variable(v1, v2, fr: Framing) -> None: buf2 = fr.encode("standard", orig2) info2 = fr.parse("standard", buf2, v2.get_message_info) _check(info2.valid, "[S10] v1 variable (base-only) -> v2 validates CRC") - decoded2 = _decode_zerofill(v2.VariableExtensionMessage, info2) + decoded2 = v2.VariableExtensionMessage.deserialize(info2) _check(decoded2.node_id == 9 and list(decoded2.readings) == [1, 2], "[S10] v2 locates variable base after cross-version decode") _check(decoded2.ext_timestamp == 0, "[S10] v2 zero-fills the trailing extension field") + _check(decoded2.ext_note == b"", + "[S10] v2 zero-fills the count-prefixed (string) extension field") # --------------------------------------------------------------------------- diff --git a/tests/ts/test_wire_evolution_interop.ts b/tests/ts/test_wire_evolution_interop.ts index 548dc810..97ffb841 100644 --- a/tests/ts/test_wire_evolution_interop.ts +++ b/tests/ts/test_wire_evolution_interop.ts @@ -313,6 +313,8 @@ function scenario10(): void { && d2.readingsData[1] === 2, '[S10] v2 locates variable base after cross-version decode'); check(d2.extTimestamp === 0, '[S10] v2 zero-fills the trailing extension field'); + check(d2.extNoteLength === 0, + '[S10] v2 zero-fills the count-prefixed (string) extension field'); } }