release: stage Jina embedding migration for 0.2.5851

README.md

@@ -473,6 +473,16 @@ The local ANN is built only by an explicit command:
 codedb /path/to/repo semantic-index
 ```
 
+CodeDB 0.2.5851 stages the managed embedding migration without breaking older
+clients. Versions through 0.2.5850 continue to request
+`Qwen/Qwen3-Embedding-0.6B`; 0.2.5851 and later request
+`jinaai/jina-embeddings-v2-base-code`. The service keeps both routes live. An
+existing Qwen ANN sidecar is rejected by the new client's model/vector-space
+check and hybrid retrieval safely uses transient exact reranking until the user
+runs `semantic-index` once to replace it with a Jina sidecar. CodeDB never
+mixes vectors from the two models and never uploads a repository automatically
+during update or ordinary queries.
+
 It splits already-indexable files into bounded 832-byte source chunks, uses
 four concurrent 25-item requests by default (explicitly configurable from one
 to eight), and writes a

build.zig.zon

@@ -1,7 +1,7 @@
 .{
     .name = .codedb2,
     .fingerprint = 0x6e18d96ca2a31757,
-    .version = "0.2.5850",
+    .version = "0.2.5851",
     .minimum_zig_version = "0.17.0-dev.813+2153f8143",
     .dependencies = .{
         .nanoregex = .{

docs/cli.md

@@ -113,6 +113,23 @@ stays in `~/.codedb/credentials.json` (0600 on POSIX). Every cloud-bound path is
 the sensitive-file denylist; `.env`, `.env.*`, `.envrc`, credentials, private keys, and unsafe
 paths cannot enter a batch even from a stale index.
 
+### Managed model migration
+
+CodeDB 0.2.5851 changes the managed default to
+`jinaai/jina-embeddings-v2-base-code`; clients through 0.2.5850 continue to
+request `Qwen/Qwen3-Embedding-0.6B`, and the hosted service serves both model
+names during the migration. Sidecar metadata includes the model, dimensions,
+query encoding, document-card encoding, and a calibration vector. A new client
+therefore refuses an old Qwen sidecar and uses transient exact reranking until
+you explicitly replace it:
+
+```bash
+codedb /path/to/repo semantic-index
+```
+
+That command transactionally replaces the old generation. Updating the binary
+does not automatically send repository contents or build a semantic sidecar.
+
 ## Daemon Management
 
 ```bash

docs/releases/0.2.5851.md

@@ -0,0 +1,41 @@
+# CodeDB 0.2.5851
+
+CodeDB 0.2.5851 begins a backward-compatible managed embedding migration.
+Existing clients remain on Qwen while this release opts into the smaller,
+faster Jina code model. Both routes remain available on the hosted service.
+
+## Compatibility behavior
+
+- CodeDB 0.2.5850 and older continue to send the explicit model name
+  `Qwen/Qwen3-Embedding-0.6B`.
+- CodeDB 0.2.5851 sends
+  `jinaai/jina-embeddings-v2-base-code` at 512 dimensions.
+- Jina queries use its symmetric raw-text representation. Explicit Qwen and
+  custom-model overrides preserve the historical Qwen instruction prefix.
+- ANN sidecars persist the model and vector-space identity. A Qwen sidecar is
+  never searched with a Jina query vector.
+
+When a new client encounters an older Qwen sidecar, hybrid context remains
+available through transient exact reranking. Run the explicit command below
+once per previously indexed repository to build the Jina sidecar:
+
+```bash
+codedb /path/to/repo semantic-index
+```
+
+The rebuild is intentionally not automatic: updating CodeDB must not silently
+upload bounded source chunks from a repository. Rebuilding is transactional and
+replaces only the prior semantic generation.
+
+## Operational notes
+
+The hosted service runs Qwen and Jina concurrently and routes by the explicit
+`model` field already sent by every released client. No API token, endpoint, or
+configuration change is required. `CODEDB_EMBEDDINGS_MODEL` remains available
+for explicit custom-provider and rollback configurations.
+
+The Jina model is Apache-2.0 licensed. The pre-release evaluation found roughly
+2.0-2.3x higher indexing throughput and lower GPU memory use, while retrieval
+quality remained workload-dependent. Keeping both routes live makes the
+migration observable and reversible rather than forcing old installations onto
+a different vector space.

src/release_info.zig

@@ -1 +1 @@
-pub const semver = "0.2.5850";
+pub const semver = "0.2.5851";

src/semantic.zig

@@ -13,7 +13,8 @@ const snapshot = @import("snapshot.zig");
 const semantic_auth = @import("semantic_auth.zig");
 
 pub const default_url = "https://embeddings.wiki.codes/v1/codedb/embeddings";
-pub const default_model = "Qwen/Qwen3-Embedding-0.6B";
+pub const legacy_qwen_model = "Qwen/Qwen3-Embedding-0.6B";
+pub const default_model = "jinaai/jina-embeddings-v2-base-code";
 pub const default_dimensions: u16 = 512;
 pub const default_timeout_ms: u32 = 15_000;
 pub const min_timeout_ms: u32 = 10;
@@ -40,7 +41,8 @@ pub const default_semantic_weight: f32 = 0.05;
 // exact-fallback RRF so an ANN policy change cannot weaken the lexical floor.
 pub const default_ann_rrf_k: f32 = 20;
 pub const default_ann_semantic_weight: f32 = 1.5;
-pub const query_prefix_version = "qwen3-query-instruct-v1";
+pub const qwen_query_encoding_version = "qwen3-query-instruct-v1";
+pub const jina_query_encoding_version = "jina-v2-code-symmetric-raw-v1";
 pub const document_card_version = "codedb-code-chunk-v2";
 pub const calibration_text = "codedb vector-space calibration v1: deterministic code retrieval";
 pub const calibration_min_cosine: f32 = 0.9999;
@@ -137,13 +139,50 @@ pub const Config = struct {
         hash.update(self.model);
         hash.update(&.{0});
         hash.update(std.mem.asBytes(&self.dimensions));
-        hash.update(query_prefix_version);
+        hash.update(queryEncodingVersion(self.model));
         hash.update(&.{0});
         hash.update(document_card_version);
         return hash.final();
     }
 };
 
+fn queryEncodingVersion(model: []const u8) []const u8 {
+    return if (std.mem.eql(u8, model, default_model))
+        jina_query_encoding_version
+    else
+        qwen_query_encoding_version;
+}
+
+/// Jina v2 code is a symmetric embedding model, so queries and code cards
+/// share one raw-text vector space. Preserve the historical Qwen instruction
+/// for explicit Qwen/custom-model overrides so upgrading CodeDB does not
+/// silently change those deployments.
+fn formatQueryInput(allocator: std.mem.Allocator, model: []const u8, task: []const u8) ![]u8 {
+    if (std.mem.eql(u8, model, default_model)) return allocator.dupe(u8, task);
+    return std.fmt.allocPrint(
+        allocator,
+        "Instruct: Retrieve code relevant to the user request.\nQuery: {s}",
+        .{task},
+    );
+}
+
+test "semantic query encoding is raw for Jina and backward compatible for Qwen" {
+    const testing = std.testing;
+    const task = "find the authentication middleware";
+    const jina = try formatQueryInput(testing.allocator, default_model, task);
+    defer testing.allocator.free(jina);
+    try testing.expectEqualStrings(task, jina);
+
+    const qwen = try formatQueryInput(testing.allocator, legacy_qwen_model, task);
+    defer testing.allocator.free(qwen);
+    try testing.expectEqualStrings(
+        "Instruct: Retrieve code relevant to the user request.\nQuery: find the authentication middleware",
+        qwen,
+    );
+    try testing.expectEqualStrings(jina_query_encoding_version, queryEncodingVersion(default_model));
+    try testing.expectEqualStrings(qwen_query_encoding_version, queryEncodingVersion(legacy_qwen_model));
+}
+
 fn uriHost(uri: std.Uri, buffer: *[std.Io.net.HostName.max_len]u8) ![]const u8 {
     const component = uri.host orelse return error.UriMissingHost;
     const host = try component.toRaw(buffer);
@@ -468,11 +507,8 @@ pub fn embedQueryRemote(
     task: []const u8,
 ) !EmbeddingBatchResult {
     if (task.len < 3 or task.len > 1024) return error.InvalidEmbeddingQuery;
-    const query = try std.fmt.allocPrint(
-        allocator,
-        "Instruct: Retrieve code relevant to the user request.\nQuery: {s}",
-        .{task},
-    );
+    const config = Config.fromEnv();
+    const query = try formatQueryInput(allocator, config.model, task);
     defer allocator.free(query);
     return embedRemoteTexts(io, allocator, &.{query});
 }
@@ -486,11 +522,8 @@ pub fn embedQueryAndCalibrationRemote(
     task: []const u8,
 ) !EmbeddingBatchResult {
     if (task.len < 3 or task.len > 1024) return error.InvalidEmbeddingQuery;
-    const query = try std.fmt.allocPrint(
-        allocator,
-        "Instruct: Retrieve code relevant to the user request.\nQuery: {s}",
-        .{task},
-    );
+    const config = Config.fromEnv();
+    const query = try formatQueryInput(allocator, config.model, task);
     defer allocator.free(query);
     return embedRemoteTexts(io, allocator, &.{ query, calibration_text });
 }
@@ -561,8 +594,7 @@ pub fn scoreRemote(
     }
     if (actual_documents == 0) return error.NoEmbeddingCandidates;
 
-    const instruction = "Instruct: Retrieve code relevant to the user request.\nQuery: ";
-    const query = try std.fmt.allocPrint(allocator, "{s}{s}", .{ instruction, task });
+    const query = try formatQueryInput(allocator, config.model, task);
     defer allocator.free(query);
 
     const inputs = try allocator.alloc([]const u8, actual_documents + 1);

src/semantic_index.zig

@@ -1126,7 +1126,7 @@ pub fn search(
     return searchLoaded(io, allocator, &loaded, task, k, load_ns, false);
 }
 
-test "semantic ANN sidecar accepts the CodeDB Qwen 512D mapping out of the box" {
+test "semantic ANN sidecar accepts Jina 512D and rejects a legacy Qwen vector space" {
     const testing = std.testing;
     const io = testing.io;
     var tmp = testing.tmpDir(.{});
@@ -1148,7 +1148,15 @@ test "semantic ANN sidecar accepts the CodeDB Qwen 512D mapping out of the box"
     try source.writeSlabs(slab_path);
     var calibration: [ann.default_dimensions]f32 = @splat(0);
     calibration[3] = 1;
-    _ = try writeMetadata(io, testing.allocator, dir_path, semantic.default_model, ann.default_dimensions, 1234, 5678, &calibration, null, slab_name, &.{
+    const config = semantic.Config{
+        .url = semantic.default_url,
+        .model = semantic.default_model,
+        .token = null,
+        .dimensions = ann.default_dimensions,
+        .timeout_ms = semantic.default_timeout_ms,
+    };
+    const vector_space_id = config.vectorSpaceId();
+    _ = try writeMetadata(io, testing.allocator, dir_path, semantic.default_model, ann.default_dimensions, 1234, vector_space_id, &calibration, null, slab_name, &.{
         .{ .path = "src/a.zig", .line_start = 1, .line_end = 2 },
         .{ .path = "src/b.zig", .line_start = 8, .line_end = 12 },
     });
@@ -1160,10 +1168,14 @@ test "semantic ANN sidecar accepts the CodeDB Qwen 512D mapping out of the box"
     try testing.expectEqualStrings("src/b.zig", restored.records[1].path);
     try testing.expectEqual(@as(u32, 8), restored.records[1].line_start);
     try testing.expectEqual(@as(u64, 1234), restored.manifest);
-    try testing.expectEqual(@as(u64, 5678), restored.vector_space_id);
+    try testing.expectEqual(vector_space_id, restored.vector_space_id);
     try testing.expectEqual(ann.default_dimensions, restored.dimensions);
     try testing.expectEqualStrings(semantic.default_model, restored.model);
     try testing.expectEqualSlices(f32, &calibration, restored.calibration);
+    try validateLoadedConfig(&config, &restored);
+    var legacy_config = config;
+    legacy_config.model = semantic.legacy_qwen_model;
+    try testing.expectError(error.AnnModelMismatch, validateLoadedConfig(&legacy_config, &restored));
     const results = try restored.index.search(&b, 1, testing.allocator);
     defer testing.allocator.free(results);
     try testing.expectEqual(@as(u32, 1), results[0].id);