From 26c483f960a61d601d0c9607bb7b2586ac0649d5 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Wed, 3 Jun 2026 04:40:58 -0700 Subject: [PATCH 1/4] feat(csharp): make the driver AOT/trim-safe under net10.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add net10.0 to the driver's target frameworks and enable the trim, single-file, and NativeAOT Roslyn analyzers for it (IsAotCompatible). A normal `dotnet build` now surfaces IL2xxx/IL3xxx warnings — which, with TreatWarningsAsErrors, fail the build — so the driver is verified AOT-ready without yet producing a native build. Enabling the analyzers flagged one category of problem: reflection-based System.Text.Json. Fixes: - Route the concrete-DTO (de)serialization through source-generated JsonSerializerContexts: SEA request/response models, the DESC-EXTENDED result, the feature-flags response, the config dictionary, and the telemetry request envelope. Options mirror the prior runtime JsonSerializerOptions; unused options fields removed. - ComplexTypeSerializingStream serializes polymorphic Arrow value graphs (ARRAY/MAP/STRUCT), which source-gen can't model. Replace JsonSerializer.Serialize(object) with a recursive Utf8JsonWriter walk over the closed set of Arrow scalar/list/dict types — same output, no reflection. - TelemetryFrontendLog nests a protobuf type whose graph trips the source generator (SYSLIB1031: nested `Type` enum name collisions), even with a property-level [JsonConverter]. Serialize its small, stable envelope by hand (TelemetryPayloadWriter), reusing the proto converter for the leaf. The proto-free TelemetryRequest stays on source-gen; the reflection-based TelemetryJsonOptions is kept for the test suite. All four TFMs (netstandard2.0;net472;net8.0;net10.0) build clean with no IL/SYSLIB warnings; net10.0 goes from 28 IL warnings to 0. The 825 unit tests pass, including the telemetry tests that exercise the production serialization path. --- csharp/Directory.Packages.props | 4 + csharp/src/AdbcDrivers.Databricks.csproj | 16 ++- csharp/src/ComplexTypeSerializingStream.cs | 124 +++++++++++++++++- csharp/src/DatabricksConfigJsonContext.cs | 38 ++++++ csharp/src/DatabricksConfiguration.cs | 12 +- csharp/src/DatabricksStatement.cs | 2 +- csharp/src/FeatureFlagContext.cs | 2 +- csharp/src/FeatureFlagsJsonContext.cs | 29 ++++ csharp/src/Result/DescTableJsonContext.cs | 30 +++++ .../StatementExecutionClient.cs | 21 +-- .../StatementExecutionJsonContext.cs | 42 ++++++ .../StatementExecutionStatement.cs | 2 +- .../Telemetry/DatabricksTelemetryExporter.cs | 6 +- csharp/src/Telemetry/ProtoJsonConverter.cs | 11 ++ csharp/src/Telemetry/TelemetryJsonContext.cs | 41 ++++++ .../src/Telemetry/TelemetryPayloadWriter.cs | 113 ++++++++++++++++ 16 files changed, 459 insertions(+), 34 deletions(-) create mode 100644 csharp/src/DatabricksConfigJsonContext.cs create mode 100644 csharp/src/FeatureFlagsJsonContext.cs create mode 100644 csharp/src/Result/DescTableJsonContext.cs create mode 100644 csharp/src/StatementExecution/StatementExecutionJsonContext.cs create mode 100644 csharp/src/Telemetry/TelemetryJsonContext.cs create mode 100644 csharp/src/Telemetry/TelemetryPayloadWriter.cs diff --git a/csharp/Directory.Packages.props b/csharp/Directory.Packages.props index 21ddc682e..e70c60d7a 100644 --- a/csharp/Directory.Packages.props +++ b/csharp/Directory.Packages.props @@ -16,6 +16,7 @@ true + true @@ -33,6 +34,7 @@ + @@ -44,5 +46,7 @@ + + diff --git a/csharp/src/AdbcDrivers.Databricks.csproj b/csharp/src/AdbcDrivers.Databricks.csproj index 39ded3cc1..ba96d82cd 100644 --- a/csharp/src/AdbcDrivers.Databricks.csproj +++ b/csharp/src/AdbcDrivers.Databricks.csproj @@ -1,9 +1,21 @@ - netstandard2.0;net472;net8.0 - netstandard2.0;net8.0 + netstandard2.0;net472;net8.0;net10.0 + netstandard2.0;net8.0;net10.0 readme.md + + + + true + true + diff --git a/csharp/src/ComplexTypeSerializingStream.cs b/csharp/src/ComplexTypeSerializingStream.cs index d2b0b9868..f3cd248e4 100644 --- a/csharp/src/ComplexTypeSerializingStream.cs +++ b/csharp/src/ComplexTypeSerializingStream.cs @@ -16,6 +16,9 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Nodes; @@ -75,7 +78,7 @@ internal sealed class ComplexTypeSerializingStream : IArrowArrayStream // a double quote becomes \" (not ") and non-ASCII / < > & are emitted verbatim // rather than \uXXXX-escaped. The output is still valid JSON; "unsafe" only refers to // embedding directly in HTML, which is the consuming application's concern, not ours. - private static readonly JsonSerializerOptions JsonOptions = new JsonSerializerOptions + private static readonly JsonWriterOptions WriterOptions = new JsonWriterOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, }; @@ -139,16 +142,131 @@ private RecordBatch ConvertComplexColumns(RecordBatch batch) private static StringArray SerializeToStringArray(IArrowArray array) { StringArray.Builder builder = new StringArray.Builder(); + + // Write each value with a manual Utf8JsonWriter rather than JsonSerializer.Serialize. + // The value graph (ToObject) is a closed set of Arrow scalar types, lists, and + // dictionaries, so we can emit it without reflection — keeping this trim- and + // NativeAOT-safe. A single stream/writer pair is reused across rows to avoid + // per-row allocations. + using MemoryStream stream = new MemoryStream(); + using Utf8JsonWriter writer = new Utf8JsonWriter(stream, WriterOptions); for (int i = 0; i < array.Length; i++) { if (array.IsNull(i)) + { builder.AppendNull(); - else - builder.Append(JsonSerializer.Serialize(ToObject(array, i), JsonOptions)); + continue; + } + + stream.SetLength(0); + writer.Reset(stream); + WriteJsonValue(writer, ToObject(array, i)); + writer.Flush(); + builder.Append(Encoding.UTF8.GetString(stream.GetBuffer(), 0, (int)stream.Length)); } return builder.Build(); } + /// + /// Writes a value from the graph to . + /// The graph is a closed set of types: (raw numbers), strings, + /// the boxed CLR primitives produced by , + /// (struct/map), and lists. Output matches + /// what System.Text.Json's default converters would have produced for these types. + /// + private static void WriteJsonValue(Utf8JsonWriter writer, object? value) + { + switch (value) + { + case null: + writer.WriteNullValue(); + break; + case JsonNode node: + node.WriteTo(writer); + break; + case string s: + writer.WriteStringValue(s); + break; + case bool b: + writer.WriteBooleanValue(b); + break; + case sbyte v: + writer.WriteNumberValue(v); + break; + case byte v: + writer.WriteNumberValue(v); + break; + case short v: + writer.WriteNumberValue(v); + break; + case ushort v: + writer.WriteNumberValue(v); + break; + case int v: + writer.WriteNumberValue(v); + break; + case uint v: + writer.WriteNumberValue(v); + break; + case long v: + writer.WriteNumberValue(v); + break; + case ulong v: + writer.WriteNumberValue(v); + break; + case float v: + writer.WriteNumberValue(v); + break; + case double v: + writer.WriteNumberValue(v); + break; + case decimal v: + writer.WriteNumberValue(v); + break; + case DateTime v: + writer.WriteStringValue(v); + break; + case DateTimeOffset v: + writer.WriteStringValue(v); + break; + case Guid v: + writer.WriteStringValue(v); + break; + case byte[] bytes: + writer.WriteBase64StringValue(bytes); + break; +#if NET6_0_OR_GREATER + case TimeOnly t: + // System.Text.Json renders TimeOnly as an ISO 8601 time string. + writer.WriteStringValue(t.ToString("O", CultureInfo.InvariantCulture)); + break; +#endif + case TimeSpan ts: + writer.WriteStringValue(ts.ToString("c", CultureInfo.InvariantCulture)); + break; + case IReadOnlyDictionary dict: + writer.WriteStartObject(); + foreach (KeyValuePair kvp in dict) + { + writer.WritePropertyName(kvp.Key); + WriteJsonValue(writer, kvp.Value); + } + writer.WriteEndObject(); + break; + case IEnumerable list: + writer.WriteStartArray(); + foreach (object? item in list) + WriteJsonValue(writer, item); + writer.WriteEndArray(); + break; + default: + // Rare Arrow scalar types not enumerated above (e.g. native interval structs): + // fall back to their invariant string form to keep the output valid JSON. + writer.WriteStringValue(Convert.ToString(value, CultureInfo.InvariantCulture)); + break; + } + } + /// /// Detects complex columns by inspecting the Spark:DataType:SqlName metadata /// on each field. This works for all result paths because they all expose the manifest diff --git a/csharp/src/DatabricksConfigJsonContext.cs b/csharp/src/DatabricksConfigJsonContext.cs new file mode 100644 index 000000000..483191af7 --- /dev/null +++ b/csharp/src/DatabricksConfigJsonContext.cs @@ -0,0 +1,38 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AdbcDrivers.Databricks +{ + /// + /// System.Text.Json source-generated metadata for the free-form configuration file, which is + /// read as a flat of string-to-string. Using the + /// generated context keeps the deserialization trim- and NativeAOT-safe. The options mirror + /// the prior runtime options: case-insensitive property names, comments skipped, and trailing + /// commas allowed. + /// + [JsonSourceGenerationOptions( + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true)] + [JsonSerializable(typeof(Dictionary))] + internal partial class DatabricksConfigJsonContext : JsonSerializerContext + { + } +} diff --git a/csharp/src/DatabricksConfiguration.cs b/csharp/src/DatabricksConfiguration.cs index 4789e534b..7dded0229 100644 --- a/csharp/src/DatabricksConfiguration.cs +++ b/csharp/src/DatabricksConfiguration.cs @@ -81,15 +81,11 @@ public static DatabricksConfiguration FromFile(string filePath) try { string json = File.ReadAllText(filePath); - var options = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true, - ReadCommentHandling = JsonCommentHandling.Skip, - AllowTrailingCommas = true - }; - // Deserialize as flat dictionary (free-form JSON) - var properties = JsonSerializer.Deserialize>(json, options); + // Deserialize as flat dictionary (free-form JSON). The source-generated context + // (case-insensitive, comments skipped, trailing commas allowed) keeps this + // trim- and NativeAOT-safe. + var properties = JsonSerializer.Deserialize(json, DatabricksConfigJsonContext.Default.DictionaryStringString); if (properties == null) { throw new InvalidOperationException($"Failed to deserialize configuration from {filePath}"); diff --git a/csharp/src/DatabricksStatement.cs b/csharp/src/DatabricksStatement.cs index d0b56ec0e..d7bb574b4 100644 --- a/csharp/src/DatabricksStatement.cs +++ b/csharp/src/DatabricksStatement.cs @@ -1080,7 +1080,7 @@ protected override async Task GetColumnsExtendedAsync(CancellationT { throw new FormatException($"Invalid json result of {query}: result is null or empty"); } - var result = JsonSerializer.Deserialize(resultJson!); + var result = JsonSerializer.Deserialize(resultJson!, DescTableJsonContext.Default.DescTableExtendedResult); if (result == null) { throw new FormatException($"Invalid json result of {query}.Result={resultJson}"); diff --git a/csharp/src/FeatureFlagContext.cs b/csharp/src/FeatureFlagContext.cs index a4e3f7174..69ce6cceb 100644 --- a/csharp/src/FeatureFlagContext.cs +++ b/csharp/src/FeatureFlagContext.cs @@ -377,7 +377,7 @@ private void ProcessResponse(string content, Activity? activity) { try { - var response = JsonSerializer.Deserialize(content); + var response = JsonSerializer.Deserialize(content, FeatureFlagsJsonContext.Default.FeatureFlagsResponse); if (response?.Flags != null) { diff --git a/csharp/src/FeatureFlagsJsonContext.cs b/csharp/src/FeatureFlagsJsonContext.cs new file mode 100644 index 000000000..f459f2983 --- /dev/null +++ b/csharp/src/FeatureFlagsJsonContext.cs @@ -0,0 +1,29 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Text.Json.Serialization; + +namespace AdbcDrivers.Databricks +{ + /// + /// System.Text.Json source-generated metadata for so the + /// feature-flag response is deserialized without reflection (trim- and NativeAOT-safe). + /// + [JsonSerializable(typeof(FeatureFlagsResponse))] + internal partial class FeatureFlagsJsonContext : JsonSerializerContext + { + } +} diff --git a/csharp/src/Result/DescTableJsonContext.cs b/csharp/src/Result/DescTableJsonContext.cs new file mode 100644 index 000000000..68205b86d --- /dev/null +++ b/csharp/src/Result/DescTableJsonContext.cs @@ -0,0 +1,30 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Text.Json.Serialization; + +namespace AdbcDrivers.Databricks.Result +{ + /// + /// System.Text.Json source-generated metadata for + /// (the DESC EXTENDED … AS JSON response). Used so the deserialization is trim- and + /// NativeAOT-safe. Default options match the prior parameterless deserialize calls. + /// + [JsonSerializable(typeof(DescTableExtendedResult))] + internal partial class DescTableJsonContext : JsonSerializerContext + { + } +} diff --git a/csharp/src/StatementExecution/StatementExecutionClient.cs b/csharp/src/StatementExecution/StatementExecutionClient.cs index d47969218..7ed5a25ae 100644 --- a/csharp/src/StatementExecution/StatementExecutionClient.cs +++ b/csharp/src/StatementExecution/StatementExecutionClient.cs @@ -19,7 +19,6 @@ using System.Net.Http; using System.Text; using System.Text.Json; -using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using Apache.Arrow.Adbc; @@ -101,12 +100,6 @@ internal class StatementExecutionClient : IStatementExecutionClient private const string SessionsEndpoint = "/api/2.0/sql/sessions"; private const string StatementsEndpoint = "/api/2.0/sql/statements"; - // JSON serialization options - ignore null values when writing - private static readonly JsonSerializerOptions s_jsonOptions = new JsonSerializerOptions - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - /// /// Initializes a new instance of the class. /// @@ -147,7 +140,7 @@ public async Task CreateSessionAsync( } var url = $"{_baseUrl}{SessionsEndpoint}"; - var jsonContent = JsonSerializer.Serialize(request, s_jsonOptions); + var jsonContent = JsonSerializer.Serialize(request, StatementExecutionJsonContext.Default.CreateSessionRequest); var content = new StringContent(jsonContent, Encoding.UTF8, "application/json"); var httpRequest = new HttpRequestMessage(HttpMethod.Post, url) @@ -160,7 +153,7 @@ public async Task CreateSessionAsync( await EnsureSuccessStatusCodeAsync(response).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - var sessionResponse = JsonSerializer.Deserialize(responseContent, s_jsonOptions); + var sessionResponse = JsonSerializer.Deserialize(responseContent, StatementExecutionJsonContext.Default.CreateSessionResponse); if (sessionResponse == null) { @@ -214,7 +207,7 @@ public async Task ExecuteStatementAsync( } var url = $"{_baseUrl}{StatementsEndpoint}"; - var jsonContent = JsonSerializer.Serialize(request, s_jsonOptions); + var jsonContent = JsonSerializer.Serialize(request, StatementExecutionJsonContext.Default.ExecuteStatementRequest); var content = new StringContent(jsonContent, Encoding.UTF8, "application/json"); var httpRequest = new HttpRequestMessage(HttpMethod.Post, url) @@ -232,7 +225,7 @@ public async Task ExecuteStatementAsync( await EnsureSuccessStatusCodeAsync(response).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - var executeResponse = JsonSerializer.Deserialize(responseContent, s_jsonOptions); + var executeResponse = JsonSerializer.Deserialize(responseContent, StatementExecutionJsonContext.Default.ExecuteStatementResponse); if (executeResponse == null) { @@ -276,7 +269,7 @@ public async Task GetStatementAsync( await EnsureSuccessStatusCodeAsync(response).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - var getResponse = JsonSerializer.Deserialize(responseContent, s_jsonOptions); + var getResponse = JsonSerializer.Deserialize(responseContent, StatementExecutionJsonContext.Default.GetStatementResponse); if (getResponse == null) { @@ -316,7 +309,7 @@ public async Task GetResultChunkAsync( await EnsureSuccessStatusCodeAsync(response).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - var resultData = JsonSerializer.Deserialize(responseContent, s_jsonOptions); + var resultData = JsonSerializer.Deserialize(responseContent, StatementExecutionJsonContext.Default.ResultData); if (resultData == null) { @@ -388,7 +381,7 @@ private async Task EnsureSuccessStatusCodeAsync(HttpResponseMessage response) // Try to parse error details from JSON response try { - var errorResponse = JsonSerializer.Deserialize(errorContent, s_jsonOptions); + var errorResponse = JsonSerializer.Deserialize(errorContent, StatementExecutionJsonContext.Default.ServiceError); if (errorResponse?.ErrorCode != null || errorResponse?.Message != null) { errorMessage = $"{errorMessage}. Error Code: {errorResponse.ErrorCode ?? "Unknown"}, Message: {errorResponse.Message ?? "Unknown"}"; diff --git a/csharp/src/StatementExecution/StatementExecutionJsonContext.cs b/csharp/src/StatementExecution/StatementExecutionJsonContext.cs new file mode 100644 index 000000000..921190d07 --- /dev/null +++ b/csharp/src/StatementExecution/StatementExecutionJsonContext.cs @@ -0,0 +1,42 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Text.Json.Serialization; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + /// + /// System.Text.Json source-generated metadata for the Statement Execution API (SEA) + /// request and response models. Routing the (de)serialization calls through this context + /// makes them trim- and NativeAOT-safe (no reflection-based serialization), which is why + /// the driver builds cleanly under IsAotCompatible on net10.0. + /// + /// The options mirror the previously-used runtime JsonSerializerOptions: null + /// properties are omitted on write. Nested model types (status, manifest, schema, chunks, + /// links, parameters, …) are pulled in automatically by the generator. + /// + [JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] + [JsonSerializable(typeof(CreateSessionRequest))] + [JsonSerializable(typeof(CreateSessionResponse))] + [JsonSerializable(typeof(ExecuteStatementRequest))] + [JsonSerializable(typeof(ExecuteStatementResponse))] + [JsonSerializable(typeof(GetStatementResponse))] + [JsonSerializable(typeof(ResultData))] + [JsonSerializable(typeof(ServiceError))] + internal partial class StatementExecutionJsonContext : JsonSerializerContext + { + } +} diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 55784d932..0f8932686 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1371,7 +1371,7 @@ private async Task GetColumnsExtendedViaDescTableAsync(string? cata if (string.IsNullOrEmpty(resultJson)) throw new FormatException($"Empty result from {query}"); - var descResult = System.Text.Json.JsonSerializer.Deserialize(resultJson!); + var descResult = System.Text.Json.JsonSerializer.Deserialize(resultJson!, DescTableJsonContext.Default.DescTableExtendedResult); if (descResult == null) throw new FormatException($"Failed to parse JSON result from {query}"); diff --git a/csharp/src/Telemetry/DatabricksTelemetryExporter.cs b/csharp/src/Telemetry/DatabricksTelemetryExporter.cs index ca24a848b..a70a32bda 100644 --- a/csharp/src/Telemetry/DatabricksTelemetryExporter.cs +++ b/csharp/src/Telemetry/DatabricksTelemetryExporter.cs @@ -64,8 +64,6 @@ internal sealed class DatabricksTelemetryExporter : ITelemetryExporter private readonly TelemetryConfiguration _config; private readonly ResiliencePipeline _retryPipeline; - private static readonly JsonSerializerOptions s_jsonOptions = TelemetryJsonOptions.Default; - /// /// Gets the host URL for the telemetry endpoint. /// @@ -197,7 +195,7 @@ internal TelemetryRequest CreateTelemetryRequest(IReadOnlyList internal string SerializeRequest(TelemetryRequest request) { - return JsonSerializer.Serialize(request, s_jsonOptions); + return JsonSerializer.Serialize(request, TelemetryJsonContext.Default.TelemetryRequest); } /// diff --git a/csharp/src/Telemetry/ProtoJsonConverter.cs b/csharp/src/Telemetry/ProtoJsonConverter.cs index 76666c13b..65f74150e 100644 --- a/csharp/src/Telemetry/ProtoJsonConverter.cs +++ b/csharp/src/Telemetry/ProtoJsonConverter.cs @@ -54,6 +54,17 @@ public override void Write( Utf8JsonWriter writer, OssSqlDriverTelemetryLog value, JsonSerializerOptions options) + { + WriteValue(writer, value); + } + + /// + /// Writes the snake_case proto JSON for directly to + /// . Exposed so the AOT-safe manual frontend-log serializer + /// () can reuse the exact same proto formatting as the + /// reflection-based converter path, keeping a single source of truth. + /// + internal static void WriteValue(Utf8JsonWriter writer, OssSqlDriverTelemetryLog? value) { if (value == null) { diff --git a/csharp/src/Telemetry/TelemetryJsonContext.cs b/csharp/src/Telemetry/TelemetryJsonContext.cs new file mode 100644 index 000000000..aa9e77c67 --- /dev/null +++ b/csharp/src/Telemetry/TelemetryJsonContext.cs @@ -0,0 +1,41 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Text.Json.Serialization; +using AdbcDrivers.Databricks.Telemetry.Models; + +namespace AdbcDrivers.Databricks.Telemetry +{ + /// + /// System.Text.Json source-generated metadata for the telemetry request envelope so it is + /// serialized without reflection (trim- and NativeAOT-safe). Options mirror the prior runtime + /// : camelCase property names and null properties omitted. + /// + /// + /// Only lives here. TelemetryFrontendLog nests a + /// protobuf-generated type whose graph the source generator can't process (SYSLIB1031 name + /// collisions among its nested Type enums), so it is serialized by hand in + /// instead. + /// + /// + [JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] + [JsonSerializable(typeof(TelemetryRequest))] + internal partial class TelemetryJsonContext : JsonSerializerContext + { + } +} diff --git a/csharp/src/Telemetry/TelemetryPayloadWriter.cs b/csharp/src/Telemetry/TelemetryPayloadWriter.cs new file mode 100644 index 000000000..ab53204a3 --- /dev/null +++ b/csharp/src/Telemetry/TelemetryPayloadWriter.cs @@ -0,0 +1,113 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.IO; +using System.Text; +using System.Text.Json; +using AdbcDrivers.Databricks.Telemetry.Models; + +namespace AdbcDrivers.Databricks.Telemetry +{ + /// + /// AOT- and trim-safe serializer for . + /// + /// + /// The frontend log nests a protobuf-generated OssSqlDriverTelemetryLog, whose type + /// graph contains many nested enums sharing the simple name Type. The System.Text.Json + /// source generator walks that graph during type discovery and fails with SYSLIB1031 (name + /// collision) even when the proto member carries a custom [JsonConverter]. So rather + /// than route this type through a , we write its small, + /// stable envelope by hand with and delegate the proto leaf to + /// — the same formatter the + /// reflection-based path uses. + /// + /// + /// + /// Output matches the reflection serializer configured by : + /// the default encoder, explicit snake_case property names, and null properties omitted. + /// + /// + internal static class TelemetryPayloadWriter + { + internal static string SerializeFrontendLog(TelemetryFrontendLog log) + { + using MemoryStream stream = new MemoryStream(); + using (Utf8JsonWriter writer = new Utf8JsonWriter(stream)) + { + WriteFrontendLog(writer, log); + } + return Encoding.UTF8.GetString(stream.GetBuffer(), 0, (int)stream.Length); + } + + private static void WriteFrontendLog(Utf8JsonWriter writer, TelemetryFrontendLog log) + { + writer.WriteStartObject(); + + writer.WriteNumber("workspace_id", log.WorkspaceId); + if (log.FrontendLogEventId != null) + { + writer.WriteString("frontend_log_event_id", log.FrontendLogEventId); + } + + if (log.Context != null) + { + writer.WritePropertyName("context"); + WriteContext(writer, log.Context); + } + + if (log.Entry != null) + { + writer.WritePropertyName("entry"); + WriteEntry(writer, log.Entry); + } + + writer.WriteEndObject(); + } + + private static void WriteContext(Utf8JsonWriter writer, FrontendLogContext context) + { + writer.WriteStartObject(); + + if (context.ClientContext != null) + { + writer.WritePropertyName("client_context"); + writer.WriteStartObject(); + if (context.ClientContext.UserAgent != null) + { + writer.WriteString("user_agent", context.ClientContext.UserAgent); + } + writer.WriteEndObject(); + } + + writer.WriteNumber("timestamp_millis", context.TimestampMillis); + + writer.WriteEndObject(); + } + + private static void WriteEntry(Utf8JsonWriter writer, FrontendLogEntry entry) + { + writer.WriteStartObject(); + + if (entry.SqlDriverLog != null) + { + writer.WritePropertyName("sql_driver_log"); + OssSqlDriverTelemetryLogJsonConverter.WriteValue(writer, entry.SqlDriverLog); + } + + writer.WriteEndObject(); + } + } +} From 6654fdf437fb33b9a414516e7afe0b45d976c1a2 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Fri, 5 Jun 2026 21:24:52 -0700 Subject: [PATCH 2/4] refactor(csharp): streamline complex-type JSON writer and cover the AOT serializers Follow-up to the net10.0 AOT work (26c483f). - Rewrite ComplexTypeSerializingStream to emit ARRAY/MAP/STRUCT values directly from the Arrow arrays with Utf8JsonWriter, instead of boxing through an intermediate object graph. Same AOT/trim-safety, fewer allocations. MAP keys are written in Arrow source order. - Add ComplexTypeSerializingStream unit tests: STRUCT field order and null fields, nested array-in-struct, MAP ordering and non-string keys, null elements, the relaxed string encoder, multi-row buffer reuse, and non-complex column passthrough. The fractional-double assertion is framework-aware (.NET Core emits "1.1", .NET Framework "1.1000000000000001"). - Guard the telemetry serialization paths against drift: a round-trip test asserts the hand-written TelemetryPayloadWriter matches the reflection-based TelemetryJsonOptions, plus a reflection-based completeness check asserts the fixture populates every envelope property, so adding a property fails the build rather than silently dropping a field. Co-Authored-By: Claude Opus 4.8 (1M context) --- csharp/src/ComplexTypeSerializingStream.cs | 249 ++++++++--------- .../src/Telemetry/TelemetryPayloadWriter.cs | 4 +- .../Unit/ComplexTypeSerializingStreamTests.cs | 256 +++++++++++++++++- .../Telemetry/TelemetryPayloadWriterTests.cs | 170 ++++++++++++ 4 files changed, 536 insertions(+), 143 deletions(-) create mode 100644 csharp/test/Unit/Telemetry/TelemetryPayloadWriterTests.cs diff --git a/csharp/src/ComplexTypeSerializingStream.cs b/csharp/src/ComplexTypeSerializingStream.cs index f3cd248e4..c3508c4b4 100644 --- a/csharp/src/ComplexTypeSerializingStream.cs +++ b/csharp/src/ComplexTypeSerializingStream.cs @@ -16,19 +16,17 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Text; using System.Text.Encodings.Web; using System.Text.Json; -using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; using Apache.Arrow; using Apache.Arrow.Adbc.Extensions; using Apache.Arrow.Ipc; using Apache.Arrow.Types; -using AdbcDrivers.Databricks.StatementExecution; namespace AdbcDrivers.Databricks { @@ -160,113 +158,13 @@ private static StringArray SerializeToStringArray(IArrowArray array) stream.SetLength(0); writer.Reset(stream); - WriteJsonValue(writer, ToObject(array, i)); + SerializeStructuredValue(writer, array, i); writer.Flush(); builder.Append(Encoding.UTF8.GetString(stream.GetBuffer(), 0, (int)stream.Length)); } return builder.Build(); } - /// - /// Writes a value from the graph to . - /// The graph is a closed set of types: (raw numbers), strings, - /// the boxed CLR primitives produced by , - /// (struct/map), and lists. Output matches - /// what System.Text.Json's default converters would have produced for these types. - /// - private static void WriteJsonValue(Utf8JsonWriter writer, object? value) - { - switch (value) - { - case null: - writer.WriteNullValue(); - break; - case JsonNode node: - node.WriteTo(writer); - break; - case string s: - writer.WriteStringValue(s); - break; - case bool b: - writer.WriteBooleanValue(b); - break; - case sbyte v: - writer.WriteNumberValue(v); - break; - case byte v: - writer.WriteNumberValue(v); - break; - case short v: - writer.WriteNumberValue(v); - break; - case ushort v: - writer.WriteNumberValue(v); - break; - case int v: - writer.WriteNumberValue(v); - break; - case uint v: - writer.WriteNumberValue(v); - break; - case long v: - writer.WriteNumberValue(v); - break; - case ulong v: - writer.WriteNumberValue(v); - break; - case float v: - writer.WriteNumberValue(v); - break; - case double v: - writer.WriteNumberValue(v); - break; - case decimal v: - writer.WriteNumberValue(v); - break; - case DateTime v: - writer.WriteStringValue(v); - break; - case DateTimeOffset v: - writer.WriteStringValue(v); - break; - case Guid v: - writer.WriteStringValue(v); - break; - case byte[] bytes: - writer.WriteBase64StringValue(bytes); - break; -#if NET6_0_OR_GREATER - case TimeOnly t: - // System.Text.Json renders TimeOnly as an ISO 8601 time string. - writer.WriteStringValue(t.ToString("O", CultureInfo.InvariantCulture)); - break; -#endif - case TimeSpan ts: - writer.WriteStringValue(ts.ToString("c", CultureInfo.InvariantCulture)); - break; - case IReadOnlyDictionary dict: - writer.WriteStartObject(); - foreach (KeyValuePair kvp in dict) - { - writer.WritePropertyName(kvp.Key); - WriteJsonValue(writer, kvp.Value); - } - writer.WriteEndObject(); - break; - case IEnumerable list: - writer.WriteStartArray(); - foreach (object? item in list) - WriteJsonValue(writer, item); - writer.WriteEndArray(); - break; - default: - // Rare Arrow scalar types not enumerated above (e.g. native interval structs): - // fall back to their invariant string form to keep the output valid JSON. - writer.WriteStringValue(Convert.ToString(value, CultureInfo.InvariantCulture)); - break; - } - } - /// /// Detects complex columns by inspecting the Spark:DataType:SqlName metadata /// on each field. This works for all result paths because they all expose the manifest @@ -293,40 +191,106 @@ private static HashSet DetectComplexColumns(Schema schema) // --- JSON serialization helpers --- - private static object? ToObject(IArrowArray array, int index) + private static void SerializeStructuredValue(Utf8JsonWriter writer, IArrowArray array, int index) { if (array.IsNull(index)) - return null; + { + writer.WriteNullValue(); + return; + } - // Handle complex types with recursive traversal, and types needing specific - // string formatting. All other primitives delegate to ValueAt(). - return array switch + switch (array.Data.DataType.TypeId) { - ListArray la => ToListOrMap(la, index), - StructArray sa => ToDict(sa, index), + case ArrowTypeId.List: + case ArrowTypeId.Map: SerializeListOrMap(writer, (ListArray)array, index); break; + case ArrowTypeId.Struct: SerializeDict(writer, (StructArray)array, index); break; // DECIMAL: emit as a bare JSON number (not a quoted string) so the output matches // the JDBC driver and is valid JSON. The decimal's string form is written raw so // values beyond C# decimal's ~28-digit range (DECIMAL(38, …)) keep full precision. - Decimal128Array dec => RawNumber(dec.GetString(index)), - Decimal256Array dec => RawNumber(dec.GetString(index)), - Date32Array d32 => d32.GetDateTime(index)?.ToString("yyyy-MM-dd"), + case ArrowTypeId.Decimal32: writer.WriteRawValue(((Decimal32Array)array).GetString(index)); break; + case ArrowTypeId.Decimal64: writer.WriteRawValue(((Decimal64Array)array).GetString(index)); break; + case ArrowTypeId.Decimal128: writer.WriteRawValue(((Decimal128Array)array).GetString(index)); break; + case ArrowTypeId.Decimal256: writer.WriteRawValue(((Decimal256Array)array).GetString(index)); break; + case ArrowTypeId.Date32: writer.WriteStringValue(((Date32Array)array).GetDateTime(index)!.Value.ToString("yyyy-MM-dd")); break; + case ArrowTypeId.Date64: writer.WriteStringValue(((Date64Array)array).GetDateTime(index)!.Value.ToString("yyyy-MM-dd")); break; // INTERVAL: native YearMonth/Duration arrays serialize to {} via System.Text.Json // (no public properties). Render the same "Y-M" / "D HH:MM:SS.nnnnnnnnn" strings // IntervalSerializingStream produces for top-level interval columns. - YearMonthIntervalArray ym => IntervalSerializingStream.FormatYearMonth(ym.GetValue(index)!.Value.Months), - DurationArray dur => IntervalSerializingStream.FormatDuration(dur.GetValue(index)!.Value, ((DurationType)dur.Data.DataType).Unit), - _ => array.ValueAt(index, StructResultType.Object) // int, long, double, float, bool, string, timestamp, etc. - }; - } + case ArrowTypeId.Interval: + switch (((IntervalType)array.Data.DataType).Unit) + { + case IntervalUnit.DayTime: + var dayTime = ((DayTimeIntervalArray)array).GetValue(index)!.Value; + var timeSpan = TimeSpan.FromDays(dayTime.Days) + TimeSpan.FromMilliseconds(dayTime.Milliseconds); + writer.WriteStringValue(timeSpan.ToString()); + break; + case IntervalUnit.MonthDayNanosecond: + var monthDayNano = ((MonthDayNanosecondIntervalArray)array).GetValue(index)!.Value; + timeSpan = TimeSpan.FromDays(monthDayNano.Days) + TimeSpan.FromTicks(monthDayNano.Nanoseconds / 100); + writer.WriteStringValue(IntervalSerializingStream.FormatYearMonth(monthDayNano.Months) + " " + timeSpan.ToString()); + break; + case IntervalUnit.YearMonth: + writer.WriteStringValue(IntervalSerializingStream.FormatYearMonth( + ((YearMonthIntervalArray)array).GetValue(index)!.Value.Months)); + break; + default: writer.WriteNullValue(); break; + } + break; + case ArrowTypeId.Duration: + DurationArray dur = (DurationArray)array; + writer.WriteStringValue(IntervalSerializingStream.FormatDuration(dur.GetValue(index)!.Value, ((DurationType)dur.Data.DataType).Unit)); + break; - /// - /// Wraps a numeric string as a raw JSON number node so emits it - /// unquoted (e.g. 1, not "1") with full precision. - /// - private static JsonNode? RawNumber(string? numericText) => - numericText == null ? null : JsonNode.Parse(numericText); + case ArrowTypeId.Boolean: writer.WriteBooleanValue(((BooleanArray)array).GetValue(index)!.Value); break; + case ArrowTypeId.Double: writer.WriteNumberValue(((DoubleArray)array).GetValue(index)!.Value); break; + case ArrowTypeId.Float: writer.WriteNumberValue(((FloatArray)array).GetValue(index)!.Value); break; +#if NET5_0_OR_GREATER + case ArrowTypeId.HalfFloat: + var halfValue = ((HalfFloatArray)array).GetValue(index)!.Value; + writer.WriteNumberValue((double)halfValue); break; +#endif + case ArrowTypeId.Int8: writer.WriteNumberValue(((Int8Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.Int16: writer.WriteNumberValue(((Int16Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.Int32: writer.WriteNumberValue(((Int32Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.Int64: writer.WriteNumberValue(((Int64Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.String: writer.WriteStringValue(((StringArray)array).GetString(index)); break; + case ArrowTypeId.LargeString: writer.WriteStringValue(((LargeStringArray)array).GetString(index)); break; +#if NET6_0_OR_GREATER + case ArrowTypeId.Time32: writer.WriteRawValue(((Time32Array)array).GetTime(index)!.Value.ToString("HH:mm:ss.ffffff")); break; + case ArrowTypeId.Time64: writer.WriteRawValue(((Time64Array)array).GetTime(index)!.Value.ToString("HH:mm:ss.ffffff")); break; +#else + case ArrowTypeId.Time32: + Time32Array time32Array = (Time32Array)array; + int time32 = time32Array.GetValue(index)!.Value; + switch (((Time32Type)time32Array.Data.DataType).Unit) + { + case TimeUnit.Second: writer.WriteStringValue(TimeSpan.FromSeconds(time32).ToString()); break; + case TimeUnit.Millisecond: writer.WriteStringValue(TimeSpan.FromMilliseconds(time32).ToString()); break; + default: writer.WriteNullValue(); break; + }; + break; + case ArrowTypeId.Time64: + Time64Array time64Array = (Time64Array)array; + long time64 = time64Array.GetValue(index)!.Value; + switch (((Time64Type)time64Array.Data.DataType).Unit) + { + case TimeUnit.Microsecond: writer.WriteStringValue(TimeSpan.FromTicks(time64 * 10).ToString()); break; + case TimeUnit.Nanosecond: writer.WriteStringValue(TimeSpan.FromTicks(time64 / 100).ToString()); break; + default: writer.WriteNullValue(); break; + }; + break; +#endif + case ArrowTypeId.Timestamp: writer.WriteStringValue(((TimestampArray)array).GetTimestamp(index)!.Value); break; + case ArrowTypeId.UInt8: writer.WriteNumberValue(((UInt8Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.UInt16: writer.WriteNumberValue(((UInt16Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.UInt32: writer.WriteNumberValue(((UInt32Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.UInt64: writer.WriteNumberValue(((UInt64Array)array).GetValue(index)!.Value); break; + case ArrowTypeId.Binary: writer.WriteBase64StringValue(((BinaryArray)array).GetBytes(index)); break; + default: writer.WriteNullValue(); break; + } + } - private static object ToListOrMap(ListArray listArray, int index) + private static void SerializeListOrMap(Utf8JsonWriter writer, ListArray listArray, int index) { IArrowArray values = listArray.Values; int start = (int)listArray.ValueOffsets[index]; @@ -334,12 +298,16 @@ private static object ToListOrMap(ListArray listArray, int index) // Arrow MAP is stored as List> if (values is StructArray structValues && IsMapStruct(structValues)) - return ToMapDict(structValues, start, end); + { + SerializeMapDict(writer, structValues, start, end); + return; + } + writer.WriteStartArray(); List list = new List(); for (int i = start; i < end; i++) - list.Add(ToObject(values, i)); - return list; + SerializeStructuredValue(writer, values, i); + writer.WriteEndArray(); } private static bool IsMapStruct(StructArray structArray) @@ -350,28 +318,31 @@ private static bool IsMapStruct(StructArray structArray) type.Fields[1].Name == "value"; } - private static SortedDictionary ToMapDict(StructArray entries, int start, int end) + private static void SerializeMapDict(Utf8JsonWriter writer, StructArray entries, int start, int end) { IArrowArray keyArray = entries.Fields[0]; IArrowArray valueArray = entries.Fields[1]; - // Use SortedDictionary for deterministic key ordering in the JSON output - SortedDictionary result = new SortedDictionary(); + writer.WriteStartObject(); for (int i = start; i < end; i++) { // Convert any key type to its string representation; treat null keys as "null" - string key = ToObject(keyArray, i)?.ToString() ?? "null"; - result[key] = ToObject(valueArray, i); + string key = keyArray.ValueAt(i)?.ToString() ?? "null"; + writer.WritePropertyName(key); + SerializeStructuredValue(writer, valueArray, i); } - return result; + writer.WriteEndObject(); } - private static Dictionary ToDict(StructArray structArray, int index) + private static void SerializeDict(Utf8JsonWriter writer, StructArray structArray, int index) { StructType type = (StructType)structArray.Data.DataType; - Dictionary dict = new Dictionary(); + writer.WriteStartObject(); for (int i = 0; i < type.Fields.Count; i++) - dict[type.Fields[i].Name] = ToObject(structArray.Fields[i], index); - return dict; + { + writer.WritePropertyName(type.Fields[i].Name); + SerializeStructuredValue(writer, structArray.Fields[i], index); + } + writer.WriteEndObject(); } } } diff --git a/csharp/src/Telemetry/TelemetryPayloadWriter.cs b/csharp/src/Telemetry/TelemetryPayloadWriter.cs index ab53204a3..584397cf0 100644 --- a/csharp/src/Telemetry/TelemetryPayloadWriter.cs +++ b/csharp/src/Telemetry/TelemetryPayloadWriter.cs @@ -37,7 +37,9 @@ namespace AdbcDrivers.Databricks.Telemetry /// /// /// Output matches the reflection serializer configured by : - /// the default encoder, explicit snake_case property names, and null properties omitted. + /// the default encoder, explicit snake_case property names, and null properties omitted. The + /// snake_case names come from [JsonPropertyName] attrivutes on the proto-generated models, + /// not from a global naming policy. /// /// internal static class TelemetryPayloadWriter diff --git a/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs b/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs index e43ff7a87..b1d3d2a21 100644 --- a/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs +++ b/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs @@ -62,7 +62,15 @@ public async Task Array_Double_FractionalUnchanged() b.Append(); vb.Append(1.1); vb.Append(2.5); - Assert.Equal("[1.1,2.5]", await Serialize("ARRAY", b.Build())); + string? json = await Serialize("ARRAY", b.Build()); + // The exact text of a fractional double is framework-dependent: .NET Core emits the + // shortest round-trippable form ("1.1"), while .NET Framework's System.Text.Json emits + // the full "1.1000000000000001". Both are valid JSON that round-trip to the same value. +#if NETFRAMEWORK + Assert.Equal("[1.1000000000000001,2.5]", json); +#else + Assert.Equal("[1.1,2.5]", json); +#endif } [Fact] @@ -140,9 +148,235 @@ public async Task Map_StringValueWithQuotes_IsEscapedJson() Assert.Equal("{\"key1\":\"val \\\"quote\\\"\"}", await Serialize("MAP", mb.Build())); } + [Fact] + public async Task Array_Bool_IsBareBoolean() + { + ListArray.Builder b = new ListArray.Builder(BooleanType.Default); + BooleanArray.Builder vb = (BooleanArray.Builder)b.ValueBuilder; + b.Append(); + vb.Append(true); + vb.Append(false); + Assert.Equal("[true,false]", await Serialize("ARRAY", b.Build())); + } + + [Fact] + public async Task Array_Long_IsBareNumber() + { + ListArray.Builder b = new ListArray.Builder(Int64Type.Default); + Int64Array.Builder vb = (Int64Array.Builder)b.ValueBuilder; + b.Append(); + vb.Append(10L); + vb.Append(-20L); + Assert.Equal("[10,-20]", await Serialize("ARRAY", b.Build())); + } + + [Fact] + public async Task Array_NullElement_EmitsJsonNull() + { + // A null inside the list must serialize as a bare JSON null, not be dropped. + ListArray.Builder b = new ListArray.Builder(Int32Type.Default); + Int32Array.Builder vb = (Int32Array.Builder)b.ValueBuilder; + b.Append(); + vb.Append(1); + vb.AppendNull(); + vb.Append(3); + Assert.Equal("[1,null,3]", await Serialize("ARRAY", b.Build())); + } + + [Fact] + public async Task Array_String_UsesRelaxedEncoder() + { + // The relaxed encoder escapes a double quote as \" but emits <, >, &, and non-ASCII + // characters verbatim (not \uXXXX). This is the property the WriterOptions encoder must + // preserve for the output to remain compact, valid JSON that round-trips. + ListArray.Builder b = new ListArray.Builder(StringType.Default); + StringArray.Builder vb = (StringArray.Builder)b.ValueBuilder; + b.Append(); + vb.Append("a\"&é→"); + Assert.Equal("[\"a\\\"&é→\"]", await Serialize("ARRAY", b.Build())); + } + + [Fact] + public async Task Struct_SerializesAsObject_PreservingFieldOrder() + { + // STRUCT becomes a JSON object whose keys follow Arrow field order (not sorted). + Int32Array id = new Int32Array.Builder().Append(7).Build(); + StringArray name = new StringArray.Builder().Append("bob").Build(); + BooleanArray active = new BooleanArray.Builder().Append(true).Build(); + List fields = new List + { + new Field("id", Int32Type.Default, nullable: true), + new Field("name", StringType.Default, nullable: true), + new Field("active", BooleanType.Default, nullable: true), + }; + StructArray s = new StructArray(new StructType(fields), 1, + new IArrowArray[] { id, name, active }, AllValid(1)); + Assert.Equal("{\"id\":7,\"name\":\"bob\",\"active\":true}", + await Serialize("STRUCT", s)); + } + + [Fact] + public async Task Struct_NullField_EmitsJsonNull() + { + // A null struct field serializes as a JSON null rather than being omitted. + Int32Array a = new Int32Array.Builder().AppendNull().Build(); + StringArray b = new StringArray.Builder().Append("x").Build(); + List fields = new List + { + new Field("a", Int32Type.Default, nullable: true), + new Field("b", StringType.Default, nullable: true), + }; + StructArray s = new StructArray(new StructType(fields), 1, + new IArrowArray[] { a, b }, AllValid(1)); + Assert.Equal("{\"a\":null,\"b\":\"x\"}", await Serialize("STRUCT", s)); + } + + [Fact] + public async Task Struct_NestedArrayAndScalar_RecursesCorrectly() + { + // STRUCT, score: INT> — exercises recursion through a nested list. + ListArray.Builder tagsBuilder = new ListArray.Builder(StringType.Default); + StringArray.Builder tagValues = (StringArray.Builder)tagsBuilder.ValueBuilder; + tagsBuilder.Append(); + tagValues.Append("x"); + tagValues.Append("y"); + ListArray tags = tagsBuilder.Build(); + Int32Array score = new Int32Array.Builder().Append(42).Build(); + List fields = new List + { + new Field("tags", new ListType(StringType.Default), nullable: true), + new Field("score", Int32Type.Default, nullable: true), + }; + StructArray s = new StructArray(new StructType(fields), 1, + new IArrowArray[] { tags, score }, AllValid(1)); + Assert.Equal("{\"tags\":[\"x\",\"y\"],\"score\":42}", + await Serialize("STRUCT,score:INT>", s)); + } + + [Fact] + public async Task Map_MultipleKeys_PreserveSourceOrderAndQuoted() + { + // MAP serializes to a JSON object whose keys follow the Arrow entry order the server + // sent (which is already deterministic), not a re-sorted order. + MapArray.Builder mb = new MapArray.Builder(new MapType(StringType.Default, Int32Type.Default)); + StringArray.Builder kb = (StringArray.Builder)mb.KeyBuilder; + Int32Array.Builder vb = (Int32Array.Builder)mb.ValueBuilder; + mb.Append(); + kb.Append("banana"); + vb.Append(2); + kb.Append("apple"); + vb.Append(1); + kb.Append("cherry"); + vb.Append(3); + Assert.Equal("{\"banana\":2,\"apple\":1,\"cherry\":3}", + await Serialize("MAP", mb.Build())); + } + + [Fact] + public async Task Map_NonStringKeys_AreStringified() + { + // Non-string MAP keys are rendered as their string form, in source order. + MapArray.Builder mb = new MapArray.Builder(new MapType(Int32Type.Default, StringType.Default)); + Int32Array.Builder kb = (Int32Array.Builder)mb.KeyBuilder; + StringArray.Builder vb = (StringArray.Builder)mb.ValueBuilder; + mb.Append(); + kb.Append(3); + vb.Append("three"); + kb.Append(1); + vb.Append("one"); + kb.Append(2); + vb.Append("two"); + Assert.Equal("{\"3\":\"three\",\"1\":\"one\",\"2\":\"two\"}", + await Serialize("MAP", mb.Build())); + } + + [Fact] + public async Task Map_Empty_IsEmptyObject() + { + MapArray.Builder mb = new MapArray.Builder(new MapType(StringType.Default, Int32Type.Default)); + mb.Append(); // a row holding an empty map + Assert.Equal("{}", await Serialize("MAP", mb.Build())); + } + + [Fact] + public async Task NullComplexValue_ProducesNullStringEntry() + { + // A null complex value (the whole list/map/struct is null) becomes a null in the + // output StringArray, not the literal text "null". + ListArray.Builder b = new ListArray.Builder(Int32Type.Default); + b.AppendNull(); + string?[] rows = await SerializeRows("ARRAY", b.Build()); + Assert.Single(rows); + Assert.Null(rows[0]); + } + + [Fact] + public async Task MultipleRows_SerializeIndependently() + { + // The writer reuses a single stream/Utf8JsonWriter across rows; this guards that the + // buffer is reset between rows so values don't bleed together. Includes an empty list. + ListArray.Builder b = new ListArray.Builder(Int32Type.Default); + Int32Array.Builder vb = (Int32Array.Builder)b.ValueBuilder; + b.Append(); + vb.Append(1); + vb.Append(2); + b.Append(); + vb.Append(3); + b.Append(); // empty list row + string?[] rows = await SerializeRows("ARRAY", b.Build()); + Assert.Equal(new[] { "[1,2]", "[3]", "[]" }, rows); + } + + [Fact] + public async Task NonComplexColumn_PassesThrough_AndComplexColumnSchemaIsFlattened() + { + // A column without a complex SqlName must pass through untouched (same array instance, + // same Arrow type), while the complex column's schema is flattened to StringType. + ListArray.Builder listBuilder = new ListArray.Builder(Int32Type.Default); + Int32Array.Builder listValues = (Int32Array.Builder)listBuilder.ValueBuilder; + listBuilder.Append(); + listValues.Append(1); + listValues.Append(2); + listBuilder.Append(); + listValues.Append(3); + ListArray complexColumn = listBuilder.Build(); + Int32Array plainColumn = new Int32Array.Builder().Append(100).Append(200).Build(); + + Schema manifestSchema = new Schema.Builder() + .Field(new Field("c0", StringType.Default, nullable: true, + new Dictionary { ["Spark:DataType:SqlName"] = "ARRAY" })) + .Field(new Field("c1", Int32Type.Default, nullable: true, + new Dictionary { ["Spark:DataType:SqlName"] = "INT" })) + .Build(); + Schema nativeSchema = new Schema.Builder() + .Field(new Field("c0", complexColumn.Data.DataType, nullable: true)) + .Field(new Field("c1", Int32Type.Default, nullable: true)) + .Build(); + RecordBatch batch = new RecordBatch(nativeSchema, + new IArrowArray[] { complexColumn, plainColumn }, 2); + + using IArrowArrayStream inner = new StubArrowArrayStream(manifestSchema, new[] { batch }); + using ComplexTypeSerializingStream stream = new ComplexTypeSerializingStream(inner); + RecordBatch? result = await stream.ReadNextRecordBatchAsync(CancellationToken.None); + + Assert.NotNull(result); + Assert.IsType(result!.Schema.FieldsList[0].DataType); + Assert.IsType(result.Schema.FieldsList[1].DataType); + + StringArray converted = Assert.IsType(result.Column(0)); + Assert.Equal("[1,2]", converted.GetString(0)); + Assert.Equal("[3]", converted.GetString(1)); + + // The non-complex column is passed through as the very same instance. + Assert.Same(plainColumn, result.Column(1)); + } + // ----- helpers ----------------------------------------------------- - private static async Task Serialize(string sqlName, IArrowArray nativeArray) + private static async Task Serialize(string sqlName, IArrowArray nativeArray) => + (await SerializeRows(sqlName, nativeArray))[0]; + + private static async Task SerializeRows(string sqlName, IArrowArray nativeArray) { Schema manifestSchema = new Schema.Builder() .Field(new Field("c", StringType.Default, nullable: true, @@ -158,7 +392,23 @@ public async Task Map_StringValueWithQuotes_IsEscapedJson() RecordBatch? result = await stream.ReadNextRecordBatchAsync(CancellationToken.None); Assert.NotNull(result); StringArray outArray = Assert.IsType(result!.Column(0)); - return outArray.GetString(0); + string?[] rows = new string?[outArray.Length]; + for (int i = 0; i < outArray.Length; i++) + { + rows[i] = outArray.IsNull(i) ? null : outArray.GetString(i); + } + return rows; + } + + // A validity bitmap marking all rows non-null. + private static ArrowBuffer AllValid(int count) + { + ArrowBuffer.BitmapBuilder bitmap = new ArrowBuffer.BitmapBuilder(); + for (int i = 0; i < count; i++) + { + bitmap.Append(true); + } + return bitmap.Build(); } private sealed class StubArrowArrayStream : IArrowArrayStream diff --git a/csharp/test/Unit/Telemetry/TelemetryPayloadWriterTests.cs b/csharp/test/Unit/Telemetry/TelemetryPayloadWriterTests.cs new file mode 100644 index 000000000..2d6e46f5c --- /dev/null +++ b/csharp/test/Unit/Telemetry/TelemetryPayloadWriterTests.cs @@ -0,0 +1,170 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Text.Json; +using AdbcDrivers.Databricks.Telemetry; +using AdbcDrivers.Databricks.Telemetry.Models; +using AdbcDrivers.Databricks.Telemetry.Proto; +using ExecutionResultFormat = AdbcDrivers.Databricks.Telemetry.Proto.ExecutionResult.Types.Format; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.Telemetry +{ + /// + /// Guards the hand-written against drift from the + /// reflection-based serializer. + /// + /// + /// exists because the telemetry frontend log nests a + /// protobuf type the System.Text.Json source generator can't process. Because it emits the + /// envelope field-by-field by hand, a new property added to one of the envelope models would + /// be serialized by the reflection path () but + /// silently dropped by the manual writer — a quiet wire-format regression. + /// + /// + /// + /// Two checks close that gap: a round-trip test asserts the two paths produce identical JSON + /// for a fully-populated log, and a reflection-based completeness check asserts the shared + /// fixture actually populates every envelope property (so the round-trip stays meaningful when + /// a property is added). Adding an envelope property therefore forces both a fixture update and + /// a update, rather than shipping a missing field. + /// + /// + public class TelemetryPayloadWriterTests + { + /// + /// The envelope models that serializes by hand. The + /// proto leaf (OssSqlDriverTelemetryLog) is intentionally excluded — it is delegated + /// to the shared in both paths and is + /// treated as opaque here. + /// + private static readonly HashSet EnvelopeTypes = new HashSet + { + typeof(TelemetryFrontendLog), + typeof(FrontendLogContext), + typeof(FrontendLogEntry), + typeof(TelemetryClientContext), + }; + + [Fact] + public void SerializeFrontendLog_MatchesReflectionSerializer() + { + TelemetryFrontendLog log = CreateFullyPopulatedLog(); + + string reflectionJson = JsonSerializer.Serialize(log, TelemetryJsonOptions.Default); + string manualJson = TelemetryPayloadWriter.SerializeFrontendLog(log); + + Assert.Equal(reflectionJson, manualJson); + } + + [Fact] + public void FrontendLog_AllEnvelopePropertiesAreSerialized() + { + // The round-trip test above only catches a dropped field if the fixture populates that + // field (an unset property is omitted by both paths and stays equal). This asserts the + // shared fixture leaves no envelope property at its default, so a newly-added property + // forces a fixture update and the round-trip then catches a missing writer branch. + AssertFullyPopulated(CreateFullyPopulatedLog()); + } + + /// + /// Builds a with every envelope property set to a + /// non-default value. Shared by both tests so they cannot drift apart. + /// + private static TelemetryFrontendLog CreateFullyPopulatedLog() + { + return new TelemetryFrontendLog + { + WorkspaceId = 12345678901234L, + FrontendLogEventId = "550e8400-e29b-41d4-a716-446655440000", + Context = new FrontendLogContext + { + TimestampMillis = 1700000000000L, + ClientContext = new TelemetryClientContext + { + UserAgent = "AdbcDatabricksDriver/1.0.0 (.NET 8.0; Windows 10)" + } + }, + Entry = new FrontendLogEntry + { + SqlDriverLog = new OssSqlDriverTelemetryLog + { + SessionId = "session-uuid-123", + SqlStatementId = "statement-uuid-456", + OperationLatencyMs = 150, + SystemConfiguration = new DriverSystemConfiguration + { + DriverName = DatabricksConnection.DatabricksDriverName, + DriverVersion = "1.0.0", + OsName = "Windows", + RuntimeName = ".NET", + RuntimeVersion = "8.0.0" + }, + SqlOperation = new SqlExecutionEvent + { + ExecutionResult = ExecutionResultFormat.ExternalLinks, + IsCompressed = true + } + } + } + }; + } + + /// + /// Asserts every public readable property of is set to a + /// non-default value, recursing into nested . The proto leaf is + /// only required to be non-null (its fields are serialized by a shared converter, not the + /// hand-written writer). + /// + private static void AssertFullyPopulated(object instance) + { + Type type = instance.GetType(); + foreach (PropertyInfo prop in type.GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (!prop.CanRead || prop.GetIndexParameters().Length > 0) + continue; + + object? value = prop.GetValue(instance); + Assert.True( + value != null, + $"{type.Name}.{prop.Name} must be populated in the TelemetryPayloadWriter round-trip fixture. " + + "A new envelope property needs both a fixture value here and a matching branch in TelemetryPayloadWriter."); + + if (value is string s) + { + Assert.False( + string.IsNullOrEmpty(s), + $"{type.Name}.{prop.Name} must be a non-empty string in the fixture."); + } + else if (prop.PropertyType.IsValueType) + { + object defaultValue = Activator.CreateInstance(prop.PropertyType)!; + Assert.False( + value!.Equals(defaultValue), + $"{type.Name}.{prop.Name} must be set to a non-default value in the fixture."); + } + + if (EnvelopeTypes.Contains(prop.PropertyType)) + { + AssertFullyPopulated(value!); + } + } + } + } +} From 261b4f7786db2834f394a43ced0ec648360b95dc Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Tue, 9 Jun 2026 11:40:28 -0700 Subject: [PATCH 3/4] Update csharp/src/ComplexTypeSerializingStream.cs Co-authored-by: eric-wang-1990 <115501094+eric-wang-1990@users.noreply.github.com> --- csharp/src/ComplexTypeSerializingStream.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/src/ComplexTypeSerializingStream.cs b/csharp/src/ComplexTypeSerializingStream.cs index c3508c4b4..3af9f1656 100644 --- a/csharp/src/ComplexTypeSerializingStream.cs +++ b/csharp/src/ComplexTypeSerializingStream.cs @@ -256,8 +256,8 @@ private static void SerializeStructuredValue(Utf8JsonWriter writer, IArrowArray case ArrowTypeId.String: writer.WriteStringValue(((StringArray)array).GetString(index)); break; case ArrowTypeId.LargeString: writer.WriteStringValue(((LargeStringArray)array).GetString(index)); break; #if NET6_0_OR_GREATER - case ArrowTypeId.Time32: writer.WriteRawValue(((Time32Array)array).GetTime(index)!.Value.ToString("HH:mm:ss.ffffff")); break; - case ArrowTypeId.Time64: writer.WriteRawValue(((Time64Array)array).GetTime(index)!.Value.ToString("HH:mm:ss.ffffff")); break; + case ArrowTypeId.Time32: writer.WriteStringValue(((Time32Array)array).GetTime(index)!.Value.ToString("HH:mm:ss.ffffff")); break; + case ArrowTypeId.Time64: writer.WriteStringValue(((Time64Array)array).GetTime(index)!.Value.ToString("HH:mm:ss.ffffff")); break; #else case ArrowTypeId.Time32: Time32Array time32Array = (Time32Array)array; From 9230d5bab5cc7a4d74e49ee44174b596ae0f930b Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Wed, 10 Jun 2026 10:39:07 -0700 Subject: [PATCH 4/4] test(csharp): add complete test coverage for ComplexTypeSerializingStream JSON serialization Add 14 new unit tests covering previously untested scalar type serialization paths in the complex-type JSON serializer: - Time32 (seconds, milliseconds) and Time64 (microseconds, nanoseconds): regression tests for the WriteRawValue -> WriteStringValue fix that produced invalid JSON for time values nested in complex types - Float, Int8, Int16, UInt8, UInt16, UInt32, UInt64: bare JSON numbers - Date64: quoted yyyy-MM-dd format - Binary: base64-encoded quoted string - DayTime and MonthDayNanosecond intervals: quoted interval strings Time tests include conditional expectations for both .NET 6+ (TimeOnly) and .NET Framework (TimeSpan) code paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Unit/ComplexTypeSerializingStreamTests.cs | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs b/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs index b1d3d2a21..ae00ce695 100644 --- a/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs +++ b/csharp/test/Unit/ComplexTypeSerializingStreamTests.cs @@ -371,6 +371,190 @@ public async Task NonComplexColumn_PassesThrough_AndComplexColumnSchemaIsFlatten Assert.Same(plainColumn, result.Column(1)); } + // --- Time32 / Time64 (regression: these formerly used WriteRawValue, producing invalid JSON) --- + + [Fact] + public async Task Array_Time32_Seconds_IsQuotedString() + { + // 12:30:45 = 45045 seconds since midnight. + // Must serialize as a quoted JSON string, not a raw value. + ListArray.Builder b = new ListArray.Builder(new Time32Type(TimeUnit.Second)); + Time32Array.Builder vb = (Time32Array.Builder)b.ValueBuilder; + b.Append(); + vb.Append(45045); // 12*3600 + 30*60 + 45 +#if NETFRAMEWORK + Assert.Equal("[\"12:30:45\"]", await Serialize("ARRAY