Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions csharp/Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
<Project>
<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
<CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
</PropertyGroup>
<ItemGroup>
<!-- Compression -->
Expand All @@ -33,6 +34,7 @@
<PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
<!-- System -->
<PackageVersion Include="System.Collections.Immutable" Version="8.0.0" />
<PackageVersion Include="System.Memory" Version="4.6.3" />
<PackageVersion Include="System.Reflection.Metadata" Version="8.0.0" />
<PackageVersion Include="System.Net.Http" Version="4.3.4" />
<!-- Resilience -->
Expand All @@ -44,5 +46,7 @@
<!-- Protobuf (for telemetry schema testing) -->
<PackageVersion Include="Google.Protobuf" Version="3.25.1" />
<PackageVersion Include="Grpc.Tools" Version="2.60.0" />
<!-- Telemetry -->
<PackageVersion Include="OpenTelemetry.Api" Version="1.15.3" />
</ItemGroup>
</Project>
16 changes: 14 additions & 2 deletions csharp/src/AdbcDrivers.Databricks.csproj
Original file line number Diff line number Diff line change
@@ -1,9 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks Condition="'$(IsWindows)'=='true'">netstandard2.0;net472;net8.0</TargetFrameworks>
<TargetFrameworks Condition="'$(TargetFrameworks)'==''">netstandard2.0;net8.0</TargetFrameworks>
<TargetFrameworks Condition="'$(IsWindows)'=='true'">netstandard2.0;net472;net8.0;net10.0</TargetFrameworks>
<TargetFrameworks Condition="'$(TargetFrameworks)'==''">netstandard2.0;net8.0;net10.0</TargetFrameworks>
<PackageReadmeFile>readme.md</PackageReadmeFile>
</PropertyGroup>

<!--
Verify the driver is AOT/trim-safe under net10.0 without yet producing a
native build. IsAotCompatible turns on the trim, single-file, and AOT
Roslyn analyzers (and marks the assembly IsTrimmable) so a normal build
surfaces IL2xxx/IL3xxx warnings; with TreatWarningsAsErrors they fail the
build. The actual NativeAOT publish will live in a separate project.
-->
<PropertyGroup Condition="'$(TargetFramework)'=='net10.0'">
<IsAotCompatible>true</IsAotCompatible>
<EnableNETAnalyzers>true</EnableNETAnalyzers>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="K4os.Compression.LZ4" />
<PackageReference Include="K4os.Compression.LZ4.Streams" />
Expand Down
124 changes: 121 additions & 3 deletions csharp/src/ComplexTypeSerializingStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;
Expand Down Expand Up @@ -75,7 +78,7 @@ internal sealed class ComplexTypeSerializingStream : IArrowArrayStream
// a double quote becomes \" (not ") and non-ASCII / < > & are emitted verbatim
// rather than \uXXXX-escaped. The output is still valid JSON; "unsafe" only refers to
// embedding directly in HTML, which is the consuming application's concern, not ours.
private static readonly JsonSerializerOptions JsonOptions = new JsonSerializerOptions
private static readonly JsonWriterOptions WriterOptions = new JsonWriterOptions
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
};
Expand Down Expand Up @@ -139,16 +142,131 @@ private RecordBatch ConvertComplexColumns(RecordBatch batch)
private static StringArray SerializeToStringArray(IArrowArray array)
{
StringArray.Builder builder = new StringArray.Builder();

// Write each value with a manual Utf8JsonWriter rather than JsonSerializer.Serialize.
// The value graph (ToObject) is a closed set of Arrow scalar types, lists, and
// dictionaries, so we can emit it without reflection — keeping this trim- and
// NativeAOT-safe. A single stream/writer pair is reused across rows to avoid
// per-row allocations.
using MemoryStream stream = new MemoryStream();
using Utf8JsonWriter writer = new Utf8JsonWriter(stream, WriterOptions);
for (int i = 0; i < array.Length; i++)
{
if (array.IsNull(i))
{
builder.AppendNull();
else
builder.Append(JsonSerializer.Serialize(ToObject(array, i), JsonOptions));
continue;
}

stream.SetLength(0);
writer.Reset(stream);
WriteJsonValue(writer, ToObject(array, i));
writer.Flush();
builder.Append(Encoding.UTF8.GetString(stream.GetBuffer(), 0, (int)stream.Length));
}
return builder.Build();
}

/// <summary>
/// Writes a value from the <see cref="ToObject"/> graph to <paramref name="writer"/>.
/// The graph is a closed set of types: <see cref="JsonNode"/> (raw numbers), strings,
/// the boxed CLR primitives produced by <see cref="IArrowArrayExtensions.ValueAt"/>,
/// <see cref="IReadOnlyDictionary{TKey,TValue}"/> (struct/map), and lists. Output matches
/// what <c>System.Text.Json</c>'s default converters would have produced for these types.
/// </summary>
private static void WriteJsonValue(Utf8JsonWriter writer, object? value)
{
switch (value)
{
case null:
writer.WriteNullValue();
break;
case JsonNode node:
node.WriteTo(writer);
break;
case string s:
writer.WriteStringValue(s);
break;
case bool b:
writer.WriteBooleanValue(b);
break;
case sbyte v:
writer.WriteNumberValue(v);
break;
case byte v:
writer.WriteNumberValue(v);
break;
case short v:
writer.WriteNumberValue(v);
break;
case ushort v:
writer.WriteNumberValue(v);
break;
case int v:
writer.WriteNumberValue(v);
break;
case uint v:
writer.WriteNumberValue(v);
break;
case long v:
writer.WriteNumberValue(v);
break;
case ulong v:
writer.WriteNumberValue(v);
break;
case float v:
writer.WriteNumberValue(v);
break;
case double v:
writer.WriteNumberValue(v);
break;
case decimal v:
writer.WriteNumberValue(v);
break;
case DateTime v:
writer.WriteStringValue(v);
break;
case DateTimeOffset v:
writer.WriteStringValue(v);
break;
case Guid v:
writer.WriteStringValue(v);
break;
case byte[] bytes:
writer.WriteBase64StringValue(bytes);
break;
#if NET6_0_OR_GREATER
case TimeOnly t:
// System.Text.Json renders TimeOnly as an ISO 8601 time string.
writer.WriteStringValue(t.ToString("O", CultureInfo.InvariantCulture));
break;
#endif
case TimeSpan ts:
writer.WriteStringValue(ts.ToString("c", CultureInfo.InvariantCulture));
break;
case IReadOnlyDictionary<string, object?> dict:
writer.WriteStartObject();
foreach (KeyValuePair<string, object?> kvp in dict)
{
writer.WritePropertyName(kvp.Key);
WriteJsonValue(writer, kvp.Value);
}
writer.WriteEndObject();
break;
case IEnumerable<object?> list:
writer.WriteStartArray();
foreach (object? item in list)
WriteJsonValue(writer, item);
writer.WriteEndArray();
break;
default:
// Rare Arrow scalar types not enumerated above (e.g. native interval structs):
// fall back to their invariant string form to keep the output valid JSON.
writer.WriteStringValue(Convert.ToString(value, CultureInfo.InvariantCulture));
Comment thread
CurtHagenlocher marked this conversation as resolved.
Outdated
break;
}
}

/// <summary>
/// Detects complex columns by inspecting the <c>Spark:DataType:SqlName</c> metadata
/// on each field. This works for all result paths because they all expose the manifest
Expand Down
38 changes: 38 additions & 0 deletions csharp/src/DatabricksConfigJsonContext.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright (c) 2026 ADBC Drivers Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace AdbcDrivers.Databricks
{
/// <summary>
/// System.Text.Json source-generated metadata for the free-form configuration file, which is
/// read as a flat <see cref="Dictionary{TKey,TValue}"/> of string-to-string. Using the
/// generated context keeps the deserialization trim- and NativeAOT-safe. The options mirror
/// the prior runtime options: case-insensitive property names, comments skipped, and trailing
/// commas allowed.
/// </summary>
[JsonSourceGenerationOptions(
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true)]
[JsonSerializable(typeof(Dictionary<string, string>))]
internal partial class DatabricksConfigJsonContext : JsonSerializerContext
{
}
}
12 changes: 4 additions & 8 deletions csharp/src/DatabricksConfiguration.cs
Original file line number Diff line number Diff line change
Expand Up @@ -81,15 +81,11 @@ public static DatabricksConfiguration FromFile(string filePath)
try
{
string json = File.ReadAllText(filePath);
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true
};

// Deserialize as flat dictionary (free-form JSON)
var properties = JsonSerializer.Deserialize<Dictionary<string, string>>(json, options);
// Deserialize as flat dictionary (free-form JSON). The source-generated context
// (case-insensitive, comments skipped, trailing commas allowed) keeps this
// trim- and NativeAOT-safe.
var properties = JsonSerializer.Deserialize(json, DatabricksConfigJsonContext.Default.DictionaryStringString);
if (properties == null)
{
throw new InvalidOperationException($"Failed to deserialize configuration from {filePath}");
Expand Down
2 changes: 1 addition & 1 deletion csharp/src/DatabricksStatement.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1080,7 +1080,7 @@ protected override async Task<QueryResult> GetColumnsExtendedAsync(CancellationT
{
throw new FormatException($"Invalid json result of {query}: result is null or empty");
}
var result = JsonSerializer.Deserialize<DescTableExtendedResult>(resultJson!);
var result = JsonSerializer.Deserialize(resultJson!, DescTableJsonContext.Default.DescTableExtendedResult);
if (result == null)
{
throw new FormatException($"Invalid json result of {query}.Result={resultJson}");
Expand Down
2 changes: 1 addition & 1 deletion csharp/src/FeatureFlagContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ private void ProcessResponse(string content, Activity? activity)
{
try
{
var response = JsonSerializer.Deserialize<FeatureFlagsResponse>(content);
var response = JsonSerializer.Deserialize(content, FeatureFlagsJsonContext.Default.FeatureFlagsResponse);

if (response?.Flags != null)
{
Expand Down
29 changes: 29 additions & 0 deletions csharp/src/FeatureFlagsJsonContext.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (c) 2026 ADBC Drivers Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System.Text.Json.Serialization;

namespace AdbcDrivers.Databricks
{
/// <summary>
/// System.Text.Json source-generated metadata for <see cref="FeatureFlagsResponse"/> so the
/// feature-flag response is deserialized without reflection (trim- and NativeAOT-safe).
/// </summary>
[JsonSerializable(typeof(FeatureFlagsResponse))]
internal partial class FeatureFlagsJsonContext : JsonSerializerContext
{
}
}
30 changes: 30 additions & 0 deletions csharp/src/Result/DescTableJsonContext.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright (c) 2026 ADBC Drivers Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System.Text.Json.Serialization;

namespace AdbcDrivers.Databricks.Result
{
/// <summary>
/// System.Text.Json source-generated metadata for <see cref="DescTableExtendedResult"/>
/// (the <c>DESC EXTENDED … AS JSON</c> response). Used so the deserialization is trim- and
/// NativeAOT-safe. Default options match the prior parameterless deserialize calls.
/// </summary>
[JsonSerializable(typeof(DescTableExtendedResult))]
internal partial class DescTableJsonContext : JsonSerializerContext
{
}
}
Loading
Loading