Skip to content

Commit 4d3c00a

Browse files
alex-clickhouseclaude
alex-clickhouse
and
claude
committed
Add Dynamic support for the TCP client
Adds the ClickHouse `Dynamic` column type — a column whose per-row value type is discovered at runtime — over the native protocol, reading and writing only the FLATTENED serialization (wire version 3), gated on `output_format_native_use_flattened_dynamic_and_json_serialization = 1`. The leading version word is validated and any non-flat encoding rejected. - Dense `DynamicColumn` (runtime type-name list + widened discriminators + per-type child columns) as the zero-copy read/write source; an ergonomic `IColumn<object>` write source infers each value's ClickHouse type via a self-contained `DynamicTypeInference` (scalars, IPv4/IPv6 by address family, date-times, decimals, and Array/Map/Tuple recursion), normalizing a value to its codec's element type so it round-trips. - Introduces the per-operation write-state contract (`IColumnCodec.BeginWrite` + `IColumnWriteState` + state-aware `WriteStatePrefix`/`WriteColumn` overloads that default to the state-free ones, so leaf codecs are untouched). The block writer runs BeginWrite -> prefix -> body -> Dispose. This lets a data-dependent prefix (the Dynamic type list) and the element-flattening composites do their flatten/scatter once across both phases. - Full nesting both directions: the Array/Tuple/Map/Nested codecs now project their flattened inner sub-column into the child's prefix and body, so `Array(Dynamic)`, `Tuple(...Dynamic...)`, `Map(K, Dynamic)`, `Nested(...Dynamic...)` and composite values inside a Dynamic all round-trip. Variant rejects a Dynamic alternative (server-disallowed). - Like Variant, there is no `Nullable(Dynamic)` — NULL rides the discriminator (value `num_types`, not 255). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 63e0070 commit 4d3c00a

18 files changed

Lines changed: 2155 additions & 133 deletions

‎ClickHouse.Driver.Tcp.Tests/Types/DynamicColumnCodecTests.cs‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
using System;
2+
using System.Threading.Tasks;
3+
using ClickHouse.Driver.Tcp.Protocol;
4+
using ClickHouse.Driver.Tcp.Tests.Utilities;
5+
using ClickHouse.Driver.Tcp.Types;
6+
7+
namespace ClickHouse.Driver.Tcp.Tests.Types;
8+
9+
[TestFixture]
10+
public class DynamicColumnCodecTests
11+
{
12+
private static IColumnCodec Resolve(string type) => ColumnCodecRegistry.Default.Resolve(type, default);
13+
14+
// Captured verbatim from a ClickHouse server (FORMAT Native, flattened serialization) for a Dynamic column
15+
// holding [42::UInt64, 'hi'::String, NULL]. String sorts before UInt64, so discriminator 0 = String, 1 =
16+
// UInt64, and NULL is the discriminator equal to the type count (2). The version (3) and type list are the
17+
// state prefix; the discriminators and per-type runs are the body.
18+
private static readonly byte[] DocumentedBytes =
19+
{
20+
0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // state prefix: serialization version = 3 (flattened)
21+
0x02, // num_types = 2
22+
0x06, 0x53, 0x74, 0x72, 0x69, 0x6E, 0x67, // type[0] = "String"
23+
0x06, 0x55, 0x49, 0x6E, 0x74, 0x36, 0x34, // type[1] = "UInt64"
24+
0x01, 0x00, 0x02, // discriminators: 1 (UInt64), 0 (String), 2 (NULL)
25+
0x02, 0x68, 0x69, // String run (1 value): len = 2, "hi"
26+
0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // UInt64 run (1 value): 42
27+
};
28+
29+
[Test]
30+
public async Task WriteFull_ErgonomicColumn_ProducesTheDocumentedBytes()
31+
{
32+
IColumnCodec codec = Resolve("Dynamic");
33+
var column = new ArrayColumn<object>("d", "Dynamic", new object[] { 42UL, "hi", null });
34+
35+
// The inferred type list is name-sorted (String before UInt64), matching the server's canonicalization.
36+
byte[] bytes = await CodecTestHarness.WriteAsync(w => codec.WriteFull(w, column));
37+
38+
CollectionAssert.AreEqual(DocumentedBytes, bytes);
39+
}
40+
41+
[Test]
42+
public async Task WriteFull_DenseColumnReadBack_RoundTripsToIdenticalBytes()
43+
{
44+
IColumnCodec codec = Resolve("Dynamic");
45+
46+
using ClickHouseBinaryReader reader = CodecTestHarness.ReaderOver(DocumentedBytes);
47+
await codec.ReadStatePrefixAsync(reader, CodecTestHarness.None);
48+
using IColumn dense = await codec.ReadColumnAsync(reader, "d", "Dynamic", 3, CodecTestHarness.None);
49+
50+
// The read-back DynamicColumn is the zero-copy write source: writing it reproduces the exact bytes.
51+
byte[] bytes = await CodecTestHarness.WriteAsync(w => codec.WriteFull(w, dense));
52+
53+
CollectionAssert.AreEqual(DocumentedBytes, bytes);
54+
}
55+
56+
[Test]
57+
public async Task WriteStatePrefixThenColumn_SeparateStateFreeCalls_ProducesTheDocumentedBytes()
58+
{
59+
IColumnCodec codec = Resolve("Dynamic");
60+
var column = new ArrayColumn<object>("d", "Dynamic", new object[] { 42UL, "hi", null });
61+
62+
// The state-free prefix and body calls each recompute the (deterministic) type list independently; the
63+
// combined output must still match the shared-state path.
64+
byte[] bytes = await CodecTestHarness.WriteAsync(w =>
65+
{
66+
codec.WriteStatePrefix(w, column);
67+
codec.WriteColumn(w, column);
68+
});
69+
70+
CollectionAssert.AreEqual(DocumentedBytes, bytes);
71+
}
72+
73+
[Test]
74+
public async Task MeasureRowBytes_DenseColumn_PricesDiscriminatorPlusValue()
75+
{
76+
IColumnCodec codec = Resolve("Dynamic");
77+
78+
using ClickHouseBinaryReader reader = CodecTestHarness.ReaderOver(DocumentedBytes);
79+
await codec.ReadStatePrefixAsync(reader, CodecTestHarness.None);
80+
using IColumn dense = await codec.ReadColumnAsync(reader, "d", "Dynamic", 3, CodecTestHarness.None);
81+
82+
// Two runtime types, so the discriminator is one byte. Row 0 = UInt64 42: 1 + 8. Row 1 = String "hi":
83+
// 1 + (1 length varint + 2 bytes). Row 2 = NULL: just the discriminator.
84+
Assert.That(codec.MeasureRowBytes(dense, 0), Is.EqualTo(9));
85+
Assert.That(codec.MeasureRowBytes(dense, 1), Is.EqualTo(4));
86+
Assert.That(codec.MeasureRowBytes(dense, 2), Is.EqualTo(1));
87+
}
88+
89+
[Test]
90+
public async Task ReadColumn_DocumentedBytes_ReconstructsValuesAndNull()
91+
{
92+
IColumnCodec codec = Resolve("Dynamic");
93+
94+
using ClickHouseBinaryReader reader = CodecTestHarness.ReaderOver(DocumentedBytes);
95+
await codec.ReadStatePrefixAsync(reader, CodecTestHarness.None);
96+
using IColumn column = await codec.ReadColumnAsync(reader, "d", "Dynamic", 3, CodecTestHarness.None);
97+
98+
Assert.That(column.RowCount, Is.EqualTo(3));
99+
Assert.That(column.GetValue(0), Is.EqualTo(42UL));
100+
Assert.That(column.GetValue(1), Is.EqualTo("hi"));
101+
Assert.That(column.GetValue(2), Is.Null);
102+
}
103+
104+
[Test]
105+
public async Task ReadColumn_DocumentedBytes_SurfacesTheRuntimeTypeList()
106+
{
107+
IColumnCodec codec = Resolve("Dynamic");
108+
109+
using ClickHouseBinaryReader reader = CodecTestHarness.ReaderOver(DocumentedBytes);
110+
await codec.ReadStatePrefixAsync(reader, CodecTestHarness.None);
111+
using IColumn column = await codec.ReadColumnAsync(reader, "d", "Dynamic", 3, CodecTestHarness.None);
112+
113+
var dynamic = (IDynamicColumn)column;
114+
Assert.That(dynamic.TypeCount, Is.EqualTo(2));
115+
Assert.That(dynamic.TypeNames, Is.EqualTo(new[] { "String", "UInt64" }));
116+
Assert.That(dynamic.Discriminators.ToArray(), Is.EqualTo(new[] { 1, 0, 2 }));
117+
}
118+
119+
[Test]
120+
public void ReadStatePrefix_VersionNotFlattened_Throws()
121+
{
122+
IColumnCodec codec = Resolve("Dynamic");
123+
124+
// Version 2 is the non-flat native default, which this client does not decode.
125+
byte[] bytes = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
126+
using ClickHouseBinaryReader reader = CodecTestHarness.ReaderOver(bytes);
127+
128+
Assert.ThrowsAsync<ClickHouseProtocolException>(async () => await codec.ReadStatePrefixAsync(reader, CodecTestHarness.None));
129+
}
130+
131+
[Test]
132+
public async Task ReadColumn_DiscriminatorPastTypeCount_Throws()
133+
{
134+
IColumnCodec codec = Resolve("Dynamic");
135+
136+
// Prefix declares two types; a discriminator of 5 selects neither a type (0, 1) nor NULL (2).
137+
byte[] prefix =
138+
{
139+
0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
140+
0x02,
141+
0x06, 0x53, 0x74, 0x72, 0x69, 0x6E, 0x67,
142+
0x06, 0x55, 0x49, 0x6E, 0x74, 0x36, 0x34,
143+
0x05, // discriminator out of range
144+
};
145+
using ClickHouseBinaryReader reader = CodecTestHarness.ReaderOver(prefix);
146+
await codec.ReadStatePrefixAsync(reader, CodecTestHarness.None);
147+
148+
Assert.ThrowsAsync<FormatException>(async () => await codec.ReadColumnAsync(reader, "d", "Dynamic", 1, CodecTestHarness.None));
149+
}
150+
151+
[Test]
152+
public async Task ReadColumn_ZeroRows_ReturnsEmptyColumn()
153+
{
154+
IColumnCodec codec = Resolve("Dynamic");
155+
156+
// A zero-row block carries no prefix and no body, so read straight from an empty buffer.
157+
using ClickHouseBinaryReader reader = CodecTestHarness.ReaderOver(Array.Empty<byte>());
158+
using IColumn column = await codec.ReadColumnAsync(reader, "d", "Dynamic", 0, CodecTestHarness.None);
159+
160+
Assert.That(column.RowCount, Is.Zero);
161+
}
162+
163+
[Test]
164+
public void Create_MaxTypesArgument_IsAccepted()
165+
{
166+
IColumnCodec codec = Resolve("Dynamic(max_types=5)");
167+
Assert.That(codec.TypeName, Is.EqualTo("Dynamic(max_types=5)"));
168+
}
169+
170+
[Test]
171+
public void Create_UnknownArgument_Throws()
172+
=> Assert.Throws<FormatException>(() => Resolve("Dynamic(max_sizes=5)"));
173+
}

‎ClickHouse.Driver.Tcp.Tests/Types/DynamicTypeInferenceTests.cs‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Net;
4+
using ClickHouse.Driver.Tcp.Numerics;
5+
using ClickHouse.Driver.Tcp.Types.Codecs;
6+
7+
namespace ClickHouse.Driver.Tcp.Tests.Types;
8+
9+
[TestFixture]
10+
public class DynamicTypeInferenceTests
11+
{
12+
[Test]
13+
public void Infer_Null_Throws()
14+
=> Assert.Throws<ArgumentNullException>(() => DynamicTypeInference.Infer(null));
15+
16+
[TestCase((byte)1, "UInt8")]
17+
[TestCase((sbyte)-1, "Int8")]
18+
[TestCase((ushort)1, "UInt16")]
19+
[TestCase((short)-1, "Int16")]
20+
[TestCase(1u, "UInt32")]
21+
[TestCase(-1, "Int32")]
22+
[TestCase(1UL, "UInt64")]
23+
[TestCase(-1L, "Int64")]
24+
[TestCase(1.5f, "Float32")]
25+
[TestCase(1.5d, "Float64")]
26+
[TestCase(true, "Bool")]
27+
[TestCase("s", "String")]
28+
public void Infer_Scalar_MapsToClickHouseTypeAndKeepsValue(object value, string expected)
29+
{
30+
(string typeName, object canonical) = DynamicTypeInference.Infer(value);
31+
Assert.That(typeName, Is.EqualTo(expected));
32+
Assert.That(canonical, Is.EqualTo(value));
33+
}
34+
35+
[Test]
36+
public void Infer_WideIntegers_Map()
37+
{
38+
Assert.That(DynamicTypeInference.Infer(UInt128.One).TypeName, Is.EqualTo("UInt128"));
39+
Assert.That(DynamicTypeInference.Infer(Int128.MinValue).TypeName, Is.EqualTo("Int128"));
40+
Assert.That(DynamicTypeInference.Infer(UInt256.Zero).TypeName, Is.EqualTo("UInt256"));
41+
Assert.That(DynamicTypeInference.Infer(Int256.Zero).TypeName, Is.EqualTo("Int256"));
42+
}
43+
44+
[Test]
45+
public void Infer_Guid_MapsToUuid()
46+
=> Assert.That(DynamicTypeInference.Infer(Guid.NewGuid()).TypeName, Is.EqualTo("UUID"));
47+
48+
[Test]
49+
public void Infer_DateOnly_MapsToDate32()
50+
=> Assert.That(DynamicTypeInference.Infer(new DateOnly(2024, 1, 1)).TypeName, Is.EqualTo("Date32"));
51+
52+
[Test]
53+
public void Infer_IpAddress_DisambiguatesByFamily()
54+
{
55+
Assert.That(DynamicTypeInference.Infer(IPAddress.Parse("127.0.0.1")).TypeName, Is.EqualTo("IPv4"));
56+
Assert.That(DynamicTypeInference.Infer(IPAddress.Parse("::1")).TypeName, Is.EqualTo("IPv6"));
57+
}
58+
59+
[Test]
60+
public void Infer_DateTimeOffset_MapsToDateTime64AndCoercesToClickHouseDateTime64()
61+
{
62+
var value = new DateTimeOffset(2024, 1, 15, 10, 30, 0, TimeSpan.FromHours(5));
63+
(string typeName, object canonical) = DynamicTypeInference.Infer(value);
64+
65+
Assert.That(typeName, Is.EqualTo("DateTime64(9)"));
66+
Assert.That(canonical, Is.InstanceOf<ClickHouseDateTime64>());
67+
Assert.That(((ClickHouseDateTime64)canonical).ToDateTimeOffset(), Is.EqualTo(value));
68+
}
69+
70+
[Test]
71+
public void Infer_ClickHouseDateTime64_KeepsItsScale()
72+
=> Assert.That(DynamicTypeInference.Infer(new ClickHouseDateTime64(0, 3, TimeSpan.Zero)).TypeName, Is.EqualTo("DateTime64(3)"));
73+
74+
[Test]
75+
public void Infer_Decimal_MapsToDecimal128AtItsScaleAndCoerces()
76+
{
77+
(string typeName, object canonical) = DynamicTypeInference.Infer(12.340m);
78+
Assert.That(typeName, Is.EqualTo("Decimal(38, 3)"));
79+
Assert.That(canonical, Is.InstanceOf<ClickHouseDecimal>());
80+
}
81+
82+
[Test]
83+
public void Infer_ClickHouseDecimal_MapsToDecimal256AtItsScale()
84+
=> Assert.That(DynamicTypeInference.Infer(new ClickHouseDecimal(new System.Numerics.BigInteger(12345), 2)).TypeName, Is.EqualTo("Decimal(76, 2)"));
85+
86+
[Test]
87+
public void Infer_Array_RecursesIntoElementType()
88+
=> Assert.That(DynamicTypeInference.Infer(new ulong[] { 1, 2 }).TypeName, Is.EqualTo("Array(UInt64)"));
89+
90+
[Test]
91+
public void Infer_EmptyArray_UsesDeclaredElementType()
92+
=> Assert.That(DynamicTypeInference.Infer(Array.Empty<int>()).TypeName, Is.EqualTo("Array(Int32)"));
93+
94+
[Test]
95+
public void Infer_Map_MapsToMapOfKeyAndValue()
96+
=> Assert.That(DynamicTypeInference.Infer(new[] { new KeyValuePair<string, uint>("a", 1) }).TypeName, Is.EqualTo("Map(String, UInt32)"));
97+
98+
[Test]
99+
public void Infer_Tuple_MapsToTupleOfElements()
100+
=> Assert.That(DynamicTypeInference.Infer((1, "a")).TypeName, Is.EqualTo("Tuple(Int32, String)"));
101+
102+
[Test]
103+
public void Infer_UnsupportedType_Throws()
104+
=> Assert.Throws<NotSupportedException>(() => DynamicTypeInference.Infer(new object()));
105+
106+
[Test]
107+
public void Infer_ArrayOfCoercionNeedingElement_Throws()
108+
=> Assert.Throws<NotSupportedException>(() => DynamicTypeInference.Infer(new[] { DateTimeOffset.UnixEpoch }));
109+
110+
[Test]
111+
public void Infer_TupleWithCoercionNeedingElement_Throws()
112+
=> Assert.Throws<NotSupportedException>(() => DynamicTypeInference.Infer((1, 2.5m)));
113+
114+
[TestCase(1, 1)]
115+
[TestCase(255, 1)]
116+
[TestCase(256, 2)]
117+
[TestCase(65535, 2)]
118+
[TestCase(65536, 4)]
119+
public void DiscriminatorWidth_GrowsWithTypeCount(int typeCount, int expectedWidth)
120+
=> Assert.That(DynamicWire.DiscriminatorWidth(typeCount), Is.EqualTo(expectedWidth));
121+
}

‎ClickHouse.Driver.Tcp.Tests/Types/VariantColumnCodecTests.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ public void Create_NullableAlternative_Throws()
104104
public void Create_NoArguments_Throws()
105105
=> Assert.Throws<FormatException>(() => Resolve("Variant()"));
106106

107+
[Test]
108+
public void Create_DynamicAlternative_Throws()
109+
=> Assert.Throws<FormatException>(() => Resolve("Variant(String, Dynamic)"));
110+
107111
[Test]
108112
public void WriteColumn_ValueWithNoMatchingAlternative_Throws()
109113
{

0 commit comments

Comments
 (0)