Skip to content

Commit 16ac4fe

Browse files
rustyconoverclaude
andcommitted
release: 0.22.0 — VGI protocol 1.3.0, global functions, vgirpc 0.20.0
The community extension already serves protocol 1.3.0 (vgi commit e76ba86), so a worker emitting the old 15-field catalog_attach result fails ATTACH against what users install today. The new fields are non-nullable and the field count is pinned, so there is no compatibility window. Protocol 1.3.0: - CatalogAttachResult gains global_functions and global_function_prefix, positioned between supports_column_statistics and resolved_data_version. The record IS the schema here -- vgirpc derives it reflectively -- so the work was in the two producers in VgiServiceImpl: the main path and the extra-catalog short-circuit. Updating only one would have left MetaWorker-style auxiliary attaches broken. - VGI_PROTOCOL_VERSION 1.2.0 -> 1.3.0, enforced as an exact major+minor match at the dispatch boundary and required independently of the schema. Global-function probes: global_scalar, global_table, global_agg, global_buffered -- one per function kind, mirroring vgi-python's _test_fixtures/global_functions.py. Registered inside Main's existing example-catalog block, so the versioned/versioned_tables wrappers do not advertise functions they do not own under a prefix naming another catalog. Closes the three duckdb_functions() inventory tests. Advertisement is wired for real: Worker.registerGlobalFunctions / globalFunctionPrefix, and VgiServiceImpl.globalFunctionInfos(). Verified by decoding the payload with the independent vgi-python client -- 4 entries, kinds scalar/table/aggregate/table_buffering, all schema_name=main, names unprefixed, prefix carried separately -- and by querying duckdb_functions() per catalog: present on example, absent from both wrappers, with a guard that the wrappers still return a non-empty function set so the zeros are real absences rather than a dead ATTACH. The C++ consuming side is not written yet: duckdb_functions() matching vgi_example_global% is still 0 after ATTACH. Also bumps the vgirpc floor 0.17.0 -> 0.20.0 and the two workflow refs that build vgirpc from source, which would otherwise have tested 0.17 source against a 0.20.0 floor. Integration: launch 289/3 -> 292/0, http 287/3 -> 290/0. gradlew test holds at 271 / 0 failures / 2 skipped; javadoc unchanged at its 23 pre-existing warnings. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 1c2ddf2 commit 16ac4fe

12 files changed

Lines changed: 385 additions & 6 deletions

File tree

.github/workflows/integration.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ env:
3636
# The vgi-rpc-java commit built from source (composite include) for the worker's
3737
# HTTP features. Drop once a vgirpc release with these features is published and
3838
# pinned in vgi/build.gradle.kts.
39-
VGI_RPC_JAVA_REF: v0.17.0
39+
VGI_RPC_JAVA_REF: v0.20.0
4040

4141
jobs:
4242
integration:

.github/workflows/landing.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ env:
2929
# vgi-rpc-java commit built from source (composite include) for the worker's
3030
# HTTP landing surface. Keep in sync with integration.yml's VGI_RPC_JAVA_REF;
3131
# drop once a vgirpc release with the landing surface is published and pinned.
32-
VGI_RPC_JAVA_REF: v0.17.0
32+
VGI_RPC_JAVA_REF: v0.20.0
3333

3434
jobs:
3535
landing:

build.gradle.kts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ plugins {
88

99
allprojects {
1010
group = "farm.query"
11-
version = "0.21.0"
11+
version = "0.22.0"
1212

1313
repositories {
1414
mavenCentral()

vgi-example-worker/src/main/java/farm/query/vgi/example/Main.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import farm.query.vgi.example.aggregate.AvgFunction;
6868
import farm.query.vgi.example.aggregate.CountFunction;
6969
import farm.query.vgi.example.aggregate.GenericSumFunction;
70+
import farm.query.vgi.example.aggregate.GlobalAggFunction;
7071
import farm.query.vgi.example.aggregate.ListAggFunction;
7172
import farm.query.vgi.example.aggregate.PercentileFunction;
7273
import farm.query.vgi.example.aggregate.SecretTypedSumFunction;
@@ -331,6 +332,7 @@ public static void main(String[] args) {
331332
// The accumulate catalog rides only the default fixture worker —
332333
// the versioned/versioned_tables wrappers reuse this binary and
333334
// their vgi_catalogs() output must stay single-row.
335+
registerGlobalProbes(w);
334336
registerAccumulate(w);
335337
registerNarrowBind(w);
336338
registerTwinCatalogs(w);
@@ -1258,6 +1260,35 @@ private static void registerCopyTo(Worker w) {
12581260
new farm.query.vgi.example.copyto.SecretLinesCopyToFunction()));
12591261
}
12601262

1263+
/**
1264+
* Global-registration probes: one fixture per function kind, registered in
1265+
* {@code main} like any other AND advertised on the {@code catalog_attach}
1266+
* result so the client publishes them into its global namespace as
1267+
* {@code vgi_example_global_*}. Dedicated classes rather than reused
1268+
* fixtures — this catalog is a cross-language contract, so making e.g.
1269+
* {@code double} globally published would force the same semantic change on
1270+
* every other SDK's copy of it. See {@code scalar/GlobalScalarFunction}.
1271+
*
1272+
* <p>Scoped to the {@code example} catalog, unlike the four kind lists this
1273+
* binary shares with the versioned / versioned_tables wrappers. The probes
1274+
* belong to the cross-language example contract (vgi-python declares them on
1275+
* that catalog alone, as do the Go / TypeScript / Rust ports), and
1276+
* {@code vgi_example} names it — a wrapper catalog advertising them under
1277+
* that prefix would be claiming functions it does not own.
1278+
*/
1279+
private static void registerGlobalProbes(Worker w) {
1280+
var scalar = new farm.query.vgi.example.scalar.GlobalScalarFunction();
1281+
var table = new farm.query.vgi.example.table.GlobalTableFunction();
1282+
var agg = new GlobalAggFunction();
1283+
var buffered = new farm.query.vgi.example.buffering.GlobalBufferedFunction();
1284+
w.registerScalar(scalar)
1285+
.registerTable(table)
1286+
.registerAggregate(agg)
1287+
.registerTableBuffering(buffered)
1288+
.globalFunctionPrefix("vgi_example")
1289+
.registerGlobalFunctions(List.of(scalar, table, agg, buffered));
1290+
}
1291+
12611292
private static void registerBuffering(Worker w) {
12621293
w.registerTableBufferings(List.of(
12631294
new farm.query.vgi.example.buffering.BufferInputFunction(),
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright 2026 Query Farm LLC - https://query.farm
2+
3+
package farm.query.vgi.example.aggregate;
4+
5+
import farm.query.vgi.aggregate.AggregateFunction;
6+
import farm.query.vgi.function.FunctionMetadata;
7+
import farm.query.vgi.function.FunctionSpec;
8+
import farm.query.vgi.types.Schemas;
9+
import org.apache.arrow.vector.BigIntVector;
10+
import org.apache.arrow.vector.FieldVector;
11+
import org.apache.arrow.vector.VectorSchemaRoot;
12+
import org.apache.arrow.vector.types.pojo.Schema;
13+
14+
import java.io.Serializable;
15+
import java.util.List;
16+
import java.util.Map;
17+
18+
/**
19+
* {@code global_agg(value: int64) -> int64} — the aggregate member of the
20+
* global-registration probe family (see
21+
* {@code farm.query.vgi.example.scalar.GlobalScalarFunction} for the family's
22+
* rationale).
23+
*
24+
* <p>Sums its input per group; a group with no accumulated state finalizes to
25+
* NULL (the framework's {@code finalizeEmpty} default). Mirrors vgi-python's
26+
* {@code GlobalAggFunction}.
27+
*/
28+
public final class GlobalAggFunction implements AggregateFunction<GlobalAggFunction.State> {
29+
30+
/** Running total for one group. */
31+
public static final class State implements Serializable {
32+
private static final long serialVersionUID = 1L;
33+
long total;
34+
}
35+
36+
private static final Schema OUTPUT_SCHEMA = new Schema(List.of(
37+
Schemas.nullable("result", Schemas.INT64)));
38+
39+
private static final FunctionSpec SPEC = FunctionSpec.builder("global_agg")
40+
.metadata(FunctionMetadata.describe("Global-registration probe (aggregate)")
41+
.withCategories("test", "global"))
42+
.arg("value", Schemas.INT64)
43+
.build();
44+
45+
@Override public FunctionSpec spec() { return SPEC; }
46+
47+
@Override public Schema outputSchema() { return OUTPUT_SCHEMA; }
48+
49+
@Override public State newState() { return new State(); }
50+
51+
@Override
52+
public void update(Map<Long, State> states, long[] groupIds, VectorSchemaRoot input) {
53+
FieldVector v = input.getFieldVectors().get(0);
54+
if (!(v instanceof BigIntVector b)) return;
55+
int rows = input.getRowCount();
56+
for (int i = 0; i < rows; i++) {
57+
if (b.isNull(i)) continue;
58+
states.computeIfAbsent(groupIds[i], k -> new State()).total += b.get(i);
59+
}
60+
}
61+
62+
@Override
63+
public void combine(State target, State source) {
64+
target.total += source.total;
65+
}
66+
67+
@Override
68+
public void finalize(FieldVector result, int rowIndex, State state) {
69+
((BigIntVector) result).setSafe(rowIndex, state.total);
70+
}
71+
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// Copyright 2026 Query Farm LLC - https://query.farm
2+
3+
package farm.query.vgi.example.buffering;
4+
5+
import farm.query.vgi.function.FunctionMetadata;
6+
import farm.query.vgi.function.FunctionSpec;
7+
import farm.query.vgi.protocol.FunctionExample;
8+
9+
import java.util.List;
10+
11+
/**
12+
* {@code global_buffered(data TABLE) -> *} — the table-buffering member of the
13+
* global-registration probe family (see
14+
* {@code farm.query.vgi.example.scalar.GlobalScalarFunction} for the family's
15+
* rationale).
16+
*
17+
* <p>Buffers every input batch and replays them on finalize; the output schema
18+
* is the input schema. Mirrors vgi-python's {@code GlobalBufferedFunction}.
19+
*/
20+
public final class GlobalBufferedFunction extends AbstractBufferAndDrain {
21+
22+
private static final FunctionSpec SPEC = FunctionSpec.builder("global_buffered")
23+
.metadata(FunctionMetadata.describe("Global-registration probe (table-buffering)")
24+
.withCategories("test", "global")
25+
.withExamples(List.of(new FunctionExample(
26+
"SELECT * FROM vgi_example_global_buffered((SELECT 1 AS x))",
27+
"Buffering probe published into system.main", null))))
28+
.table("data")
29+
.build();
30+
31+
@Override public FunctionSpec spec() { return SPEC; }
32+
}
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright 2026 Query Farm LLC - https://query.farm
2+
3+
package farm.query.vgi.example.scalar;
4+
5+
import farm.query.vgi.function.FunctionMetadata;
6+
import farm.query.vgi.protocol.FunctionExample;
7+
import farm.query.vgi.scalar.ScalarFn;
8+
import farm.query.vgi.scalar.Vector;
9+
import org.apache.arrow.vector.BigIntVector;
10+
import org.apache.arrow.vector.VarCharVector;
11+
import org.apache.arrow.vector.util.Text;
12+
13+
import java.util.List;
14+
15+
/**
16+
* {@code global_scalar(value: int64) -> utf8} — the scalar member of the
17+
* global-registration probe family (the others are
18+
* {@code table/GlobalTableFunction}, {@code aggregate/GlobalAggFunction} and
19+
* {@code buffering/GlobalBufferedFunction}).
20+
*
21+
* <p>One probe per function kind, so a client publishing a worker's functions
22+
* into its <em>global</em> namespace exercises every registration path. They are
23+
* deliberately new rather than reused fixtures: the example catalog is a
24+
* cross-language contract, and reusing {@code double} / {@code ten_thousand} /
25+
* {@code vgi_sum} / {@code echo_buffering} would force the same semantic change
26+
* on every SDK's existing functions.
27+
*
28+
* <p>Each returns a value tagged with its own name, so a test can prove the
29+
* globally published name reached the function it was supposed to rather than a
30+
* same-named function belonging to another catalog. Mirrors vgi-python's
31+
* {@code _test_fixtures/global_functions.py}.
32+
*/
33+
public final class GlobalScalarFunction extends ScalarFn {
34+
35+
@Override public String name() { return "global_scalar"; }
36+
37+
@Override public String description() { return "Global-registration probe (scalar)"; }
38+
39+
@Override public FunctionMetadata metadata() {
40+
return FunctionMetadata.describe(description())
41+
.withCategories("test", "global")
42+
.withExamples(List.of(new FunctionExample(
43+
"SELECT vgi_example_global_scalar(7)",
44+
"Scalar probe published into system.main", null)));
45+
}
46+
47+
/**
48+
* Labels each row {@code global_scalar:<value>}; NULL in, NULL out.
49+
*
50+
* @param value the values to label
51+
* @param result framework-allocated output column
52+
*/
53+
public void compute(@Vector BigIntVector value, VarCharVector result) {
54+
int rows = value.getValueCount();
55+
for (int i = 0; i < rows; i++) {
56+
if (value.isNull(i)) {
57+
result.setNull(i);
58+
continue;
59+
}
60+
result.setSafe(i, new Text("global_scalar:" + value.get(i)));
61+
}
62+
}
63+
}
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// Copyright 2026 Query Farm LLC - https://query.farm
2+
3+
package farm.query.vgi.example.table;
4+
5+
import farm.query.vgi.function.FunctionMetadata;
6+
import farm.query.vgi.function.FunctionSpec;
7+
import farm.query.vgi.internal.BatchUtil;
8+
import farm.query.vgi.protocol.FunctionExample;
9+
import farm.query.vgi.table.SimpleTableFunction;
10+
import farm.query.vgi.table.TableBindParams;
11+
import farm.query.vgi.table.TableInitParams;
12+
import farm.query.vgi.table.TableProducerState;
13+
import farm.query.vgi.types.Schemas;
14+
import farm.query.vgirpc.CallContext;
15+
import farm.query.vgirpc.OutputCollector;
16+
import org.apache.arrow.vector.BigIntVector;
17+
import org.apache.arrow.vector.VarCharVector;
18+
import org.apache.arrow.vector.types.pojo.Schema;
19+
import org.apache.arrow.vector.util.Text;
20+
21+
import java.util.List;
22+
23+
/**
24+
* {@code global_table() -> (n int64, label utf8)} — the table member of the
25+
* global-registration probe family (see {@code scalar/GlobalScalarFunction} for
26+
* the family's rationale).
27+
*
28+
* <p>A fixed generator taking no arguments: three labelled rows, emitted once.
29+
* Mirrors vgi-python's {@code GlobalTableFunction}.
30+
*/
31+
public final class GlobalTableFunction extends SimpleTableFunction {
32+
33+
private static final Schema OUTPUT = Schemas.of(
34+
Schemas.nullable("n", Schemas.INT64),
35+
Schemas.nullable("label", Schemas.UTF8));
36+
37+
/** Exactly three rows, always. */
38+
private static final long ROWS = 3L;
39+
40+
private static final FunctionSpec SPEC = FunctionSpec.builder("global_table")
41+
.metadata(FunctionMetadata.describe("Global-registration probe (table)")
42+
.withCategories("test", "global")
43+
.withExamples(List.of(new FunctionExample(
44+
"SELECT * FROM vgi_example_global_table()",
45+
"Table probe published into system.main", null))))
46+
.build();
47+
48+
@Override public FunctionSpec spec() { return SPEC; }
49+
50+
@Override protected Schema outputSchema() { return OUTPUT; }
51+
52+
@Override public long cardinality(TableBindParams params) { return ROWS; }
53+
54+
@Override public TableProducerState createProducer(TableInitParams params) {
55+
return new State();
56+
}
57+
58+
/** One-shot emit latch for the three probe rows. */
59+
public static final class State extends TableProducerState {
60+
61+
/** Whether the single batch has been emitted. */
62+
public boolean emitted;
63+
64+
/** Required no-arg constructor for state deserialization. */
65+
public State() {}
66+
67+
@Override public void produceTick(OutputCollector out, CallContext ctx) {
68+
if (emitted) {
69+
out.finish();
70+
return;
71+
}
72+
emitted = true;
73+
BatchUtil.emit(OUTPUT, (int) ROWS, out, (root, rows, ignored) -> {
74+
BigIntVector n = (BigIntVector) root.getVector("n");
75+
VarCharVector label = (VarCharVector) root.getVector("label");
76+
for (int i = 0; i < rows; i++) {
77+
n.setSafe(i, i);
78+
label.setSafe(i, new Text("global_table:" + i));
79+
}
80+
});
81+
}
82+
}
83+
}

vgi/build.gradle.kts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ plugins {
55

66
dependencies {
77
// farm.query is the published group (see ../build.gradle.kts allprojects).
8-
api("farm.query:vgirpc:0.17.0")
8+
api("farm.query:vgirpc:0.20.0")
99
implementation("org.slf4j:slf4j-api:2.0.16")
1010
// Cross-process aggregate state store. DuckDB spawns multiple worker
1111
// subprocesses for parallel aggregation; SQLite's file locking gives us

0 commit comments

Comments
 (0)